org.apache.flink.core.fs.Path Java Examples

The following examples show how to use org.apache.flink.core.fs.Path. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: NotifyCheckpointAbortedITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
	Configuration configuration = new Configuration();
	configuration.setBoolean(CheckpointingOptions.LOCAL_RECOVERY, true);
	configuration.setString(HighAvailabilityOptions.HA_MODE, TestingHAFactory.class.getName());

	checkpointPath = new Path(TEMPORARY_FOLDER.newFolder().toURI());
	cluster = new MiniClusterWithClientResource(
		new MiniClusterResourceConfiguration.Builder()
			.setConfiguration(configuration)
			.setNumberTaskManagers(1)
			.setNumberSlotsPerTaskManager(1).build());
	cluster.before();

	NormalMap.reset();
	DeclineSink.reset();
	TestingCompletedCheckpointStore.reset();
}
 
Example #2
Source File: OrcRowInputFormat.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an OrcRowInputFormat.
 *
 * @param path The path to read ORC files from.
 * @param orcSchema The schema of the ORC files as ORC TypeDescription.
 * @param orcConfig The configuration to read the ORC files with.
 * @param batchSize The number of Row objects to read in a batch.
 */
public OrcRowInputFormat(String path, TypeDescription orcSchema, Configuration orcConfig, int batchSize) {
	super(new Path(path));

	// configure OrcRowInputFormat
	this.schema = orcSchema;
	this.rowType = (RowTypeInfo) OrcBatchReader.schemaToTypeInfo(schema);
	this.conf = orcConfig;
	this.batchSize = batchSize;

	// set default selection mask, i.e., all fields.
	this.selectedFields = new int[this.schema.getChildren().size()];
	for (int i = 0; i < selectedFields.length; i++) {
		this.selectedFields[i] = i;
	}
}
 
Example #3
Source File: CheckpointStateOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path folder = new Path(tmp.newFolder().toURI());
	final String fileName = "nonCreativeTestFileName";
	final Path path = new Path(folder, fileName);

	// write some test data and close the stream
	try (FSDataOutputStream stream = createTestStream(fs, folder, fileName)) {
		Random rnd = new Random();
		for (int i = 0; i < rnd.nextInt(1000); i++) {
			stream.write(rnd.nextInt(100));
		}
		assertTrue(fs.exists(path));
	}

	assertFalse(fs.exists(path));
}
 
Example #4
Source File: LocalFileSystem.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public FileStatus[] listStatus(final Path f) throws IOException {

	final File localf = pathToFile(f);
	FileStatus[] results;

	if (!localf.exists()) {
		return null;
	}
	if (localf.isFile()) {
		return new FileStatus[] { new LocalFileStatus(localf, this) };
	}

	final String[] names = localf.list();
	if (names == null) {
		return null;
	}
	results = new FileStatus[names.length];
	for (int i = 0; i < names.length; i++) {
		results[i] = getFileStatus(new Path(f, names[i]));
	}

	return results;
}
 
Example #5
Source File: PythonDriverOptionsParserFactoryTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void verifyPythonDriverOptionsParsing(final String[] args) throws FlinkParseException {
	final PythonDriverOptions pythonCommandOptions = commandLineParser.parse(args);

	// verify the parsed python entrypoint module
	assertEquals("xxx", pythonCommandOptions.getEntrypointModule());

	// verify the parsed python library files
	final List<Path> pythonMainFile = pythonCommandOptions.getPythonLibFiles();
	assertNotNull(pythonMainFile);
	assertEquals(4, pythonMainFile.size());
	assertEquals(
		pythonMainFile.stream().map(Path::getName).collect(Collectors.joining(",")),
		"xxx.py,a.py,b.py,c.py");

	// verify the python program arguments
	final List<String> programArgs = pythonCommandOptions.getProgramArgs();
	assertEquals(2, programArgs.size());
	assertEquals("--input", programArgs.get(0));
	assertEquals("in.txt", programArgs.get(1));
}
 
Example #6
Source File: HadoopSwiftFileSystemITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void checkCredentialsAndSetup() throws IOException {
	// check whether credentials exist
	Assume.assumeTrue("Swift container not configured, skipping test...", CONTAINER != null);
	Assume.assumeTrue("Swift username not configured, skipping test...", USERNAME != null);
	Assume.assumeTrue("Swift password not configured, skipping test...", PASSWORD != null);
	Assume.assumeTrue("Swift tenant not configured, skipping test...", TENANT != null);
	Assume.assumeTrue("Swift region not configured, skipping test...", REGION != null);

	// initialize configuration with valid credentials
	final Configuration conf = createConfiguration();

	FileSystem.initialize(conf);

	// check for uniqueness of the test directory
	final Path directory = new Path("swift://" + CONTAINER + '.' + SERVICENAME + '/' + TEST_DATA_DIR);
	final FileSystem fs = directory.getFileSystem();

	// directory must not yet exist
	assertFalse(fs.exists(directory));

	// reset configuration
	FileSystem.initialize(new Configuration());

	skipTest = false;
}
 
Example #7
Source File: HadoopPathBasedBulkFormatBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
public HadoopPathBasedBulkFormatBuilder(
	org.apache.hadoop.fs.Path basePath,
	HadoopPathBasedBulkWriter.Factory<IN> writerFactory,
	HadoopFileCommitterFactory fileCommitterFactory,
	Configuration configuration,
	BucketAssigner<IN, BucketID> assigner,
	CheckpointRollingPolicy<IN, BucketID> policy,
	BucketFactory<IN, BucketID> bucketFactory,
	OutputFileConfig outputFileConfig) {

	this.basePath = new Path(Preconditions.checkNotNull(basePath).toString());
	this.writerFactory = writerFactory;
	this.fileCommitterFactory = fileCommitterFactory;
	this.serializableConfiguration = new SerializableConfiguration(configuration);
	this.bucketAssigner = Preconditions.checkNotNull(assigner);
	this.rollingPolicy = Preconditions.checkNotNull(policy);
	this.bucketFactory = Preconditions.checkNotNull(bucketFactory);
	this.outputFileConfig = Preconditions.checkNotNull(outputFileConfig);
}
 
Example #8
Source File: TestEnvironment.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	OptimizedPlan op = compileProgram(jobName);

	JobGraphGenerator jgg = new JobGraphGenerator();
	JobGraph jobGraph = jgg.compileJobGraph(op);

	for (Path jarFile: jarFiles) {
		jobGraph.addJar(jarFile);
	}

	jobGraph.setClasspaths(new ArrayList<>(classPaths));

	this.lastJobExecutionResult = jobExecutor.executeJobBlocking(jobGraph);
	return this.lastJobExecutionResult;
}
 
Example #9
Source File: DefaultBucketFactoryImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public Bucket<IN, BucketID> getNewBucket(
		final RecoverableWriter fsWriter,
		final int subtaskIndex,
		final BucketID bucketId,
		final Path bucketPath,
		final long initialPartCounter,
		final PartFileWriter.PartFileFactory<IN, BucketID> partFileWriterFactory,
		final RollingPolicy<IN, BucketID> rollingPolicy) {

	return Bucket.getNew(
			fsWriter,
			subtaskIndex,
			bucketId,
			bucketPath,
			initialPartCounter,
			partFileWriterFactory,
			rollingPolicy);
}
 
Example #10
Source File: ContinuousFileProcessingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
/** Create a file and fill it with content. */
private Tuple2<org.apache.hadoop.fs.Path, String> fillWithData(
	String base, String fileName, int fileIdx, String sampleLine) throws IOException, InterruptedException {

	assert (hdfs != null);

	org.apache.hadoop.fs.Path tmp =
		new org.apache.hadoop.fs.Path(base + "/." + fileName + fileIdx);

	FSDataOutputStream stream = hdfs.create(tmp);
	StringBuilder str = new StringBuilder();
	for (int i = 0; i < LINES_PER_FILE; i++) {
		String line = fileIdx + ": " + sampleLine + " " + i + "\n";
		str.append(line);
		stream.write(line.getBytes(ConfigConstants.DEFAULT_CHARSET));
	}
	stream.close();
	return new Tuple2<>(tmp, str.toString());
}
 
Example #11
Source File: FileCacheReadsFromBlobTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testFileDownloadedFromBlob() throws Exception {
	JobID jobID = new JobID();
	ExecutionAttemptID attemptID = new ExecutionAttemptID();

	final String fileName = "test_file";
	// copy / create the file
	final DistributedCache.DistributedCacheEntry entry = new DistributedCache.DistributedCacheEntry(
		fileName,
		false,
		InstantiationUtil.serializeObject(permanentBlobKey));
	Future<Path> copyResult = fileCache.createTmpFile(fileName, entry, jobID, attemptID);

	final Path dstPath = copyResult.get();
	final String actualContent = Files.toString(new File(dstPath.toUri()), StandardCharsets.UTF_8);
	assertTrue(dstPath.getFileSystem().exists(dstPath));
	assertEquals(testFileContent, actualContent);
}
 
Example #12
Source File: OuterJoinOperatorBaseTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"rawtypes", "unchecked"})
@Before
public void setup() {
	joiner = new MockRichFlatJoinFunction();

	baseOperator =
		new OuterJoinOperatorBase(joiner,
			new BinaryOperatorInformation(BasicTypeInfo.STRING_TYPE_INFO, BasicTypeInfo.STRING_TYPE_INFO,
				BasicTypeInfo.STRING_TYPE_INFO), new int[0], new int[0], "TestJoiner", null);

	executionConfig = new ExecutionConfig();

	String taskName = "Test rich outer join function";
	TaskInfo taskInfo = new TaskInfo(taskName, 1, 0, 1, 0);
	HashMap<String, Accumulator<?, ?>> accumulatorMap = new HashMap<>();
	HashMap<String, Future<Path>> cpTasks = new HashMap<>();

	runtimeContext = new RuntimeUDFContext(taskInfo, null, executionConfig, cpTasks,
		accumulatorMap, new UnregisteredMetricsGroup());
}
 
Example #13
Source File: BlobClient.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Uploads the JAR files to the {@link PermanentBlobService} of the {@link BlobServer} at the
 * given address with HA as configured.
 *
 * @param serverAddress
 * 		Server address of the {@link BlobServer}
 * @param clientConfig
 * 		Any additional configuration for the blob client
 * @param jobId
 * 		ID of the job this blob belongs to (or <tt>null</tt> if job-unrelated)
 * @param files
 * 		List of files to upload
 *
 * @throws IOException
 * 		if the upload fails
 */
public static List<PermanentBlobKey> uploadFiles(
		InetSocketAddress serverAddress, Configuration clientConfig, JobID jobId, List<Path> files)
		throws IOException {

	checkNotNull(jobId);

	if (files.isEmpty()) {
		return Collections.emptyList();
	} else {
		List<PermanentBlobKey> blobKeys = new ArrayList<>();

		try (BlobClient blobClient = new BlobClient(serverAddress, clientConfig)) {
			for (final Path file : files) {
				final PermanentBlobKey key = blobClient.uploadFile(jobId, file);
				blobKeys.add(key);
			}
		}

		return blobKeys;
	}
}
 
Example #14
Source File: FileSystemStateStorageHelper.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public RetrievableStateHandle<T> store(T state) throws Exception {
	Exception latestException = null;

	for (int attempt = 0; attempt < 10; attempt++) {
		Path filePath = getNewFilePath();

		try (FSDataOutputStream outStream = fs.create(filePath, FileSystem.WriteMode.NO_OVERWRITE)) {
			InstantiationUtil.serializeObject(outStream, state);
			return new RetrievableStreamStateHandle<T>(filePath, outStream.getPos());
		}
		catch (Exception e) {
			latestException = e;
		}
	}

	throw new Exception("Could not open output stream for state backend", latestException);
}
 
Example #15
Source File: ExecutionEnvironment.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public <X> DataSource<X> readFile(FileInputFormat<X> inputFormat, String filePath) {
	if (inputFormat == null) {
		throw new IllegalArgumentException("InputFormat must not be null.");
	}
	if (filePath == null) {
		throw new IllegalArgumentException("The file path must not be null.");
	}

	inputFormat.setFilePath(new Path(filePath));
	try {
		return createInput(inputFormat, TypeExtractor.getInputFormatTypes(inputFormat));
	}
	catch (Exception e) {
		throw new InvalidProgramException("The type returned by the input format could not be automatically determined. " +
				"Please specify the TypeInformation of the produced type explicitly by using the " +
				"'createInput(InputFormat, TypeInformation)' method instead.");
	}
}
 
Example #16
Source File: FsJobArchivist.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Reads the given archive file and returns a {@link Collection} of contained {@link ArchivedJson}.
 *
 * @param file archive to extract
 * @return collection of archived jsons
 * @throws IOException if the file can't be opened, read or doesn't contain valid json
 */
public static Collection<ArchivedJson> getArchivedJsons(Path file) throws IOException {
	try (FSDataInputStream input = file.getFileSystem().open(file);
		ByteArrayOutputStream output = new ByteArrayOutputStream()) {
		IOUtils.copyBytes(input, output);

		JsonNode archive = mapper.readTree(output.toByteArray());

		Collection<ArchivedJson> archives = new ArrayList<>();
		for (JsonNode archivePart : archive.get(ARCHIVE)) {
			String path = archivePart.get(PATH).asText();
			String json = archivePart.get(JSON).asText();
			archives.add(new ArchivedJson(path, json));
		}
		return archives;
	}
}
 
Example #17
Source File: ContinuousFileProcessingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 
Example #18
Source File: FsNegativeRunningJobsRegistryTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testSetFinishedAndRunning() throws Exception {
	final File folder = tempFolder.newFolder();
	final String uri = folder.toURI().toString();

	final JobID jid = new JobID();

	FsNegativeRunningJobsRegistry registry = new FsNegativeRunningJobsRegistry(new Path(uri));

	// set the job to finished and validate
	registry.setJobFinished(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// set the job to running does not overwrite the finished status
	registry.setJobRunning(jid);
	assertEquals(JobSchedulingStatus.DONE, registry.getJobSchedulingStatus(jid));

	// another registry should pick this up
	FsNegativeRunningJobsRegistry otherRegistry = new FsNegativeRunningJobsRegistry(new Path(uri));
	assertEquals(JobSchedulingStatus.DONE, otherRegistry.getJobSchedulingStatus(jid));

	// clear the running and finished marker, it will be pending
	otherRegistry.clearJob(jid);
	assertEquals(JobSchedulingStatus.PENDING, registry.getJobSchedulingStatus(jid));
	assertEquals(JobSchedulingStatus.PENDING, otherRegistry.getJobSchedulingStatus(jid));
}
 
Example #19
Source File: MemoryCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testParametrizationDirectories() throws Exception {
	final JobID jid = new JobID();
	final Path checkpointPath = new Path(tmp.newFolder().toURI().toString());
	final Path savepointPath = new Path(tmp.newFolder().toURI().toString());

	MemoryStateBackend backend = new MemoryStateBackend(
			checkpointPath.toString(), savepointPath.toString());

	MemoryBackendCheckpointStorage storage =
			(MemoryBackendCheckpointStorage) backend.createCheckpointStorage(jid);

	assertTrue(storage.supportsHighlyAvailableStorage());
	assertTrue(storage.hasDefaultSavepointLocation());
	assertNotNull(storage.getDefaultSavepointDirectory());

	assertEquals(savepointPath, storage.getDefaultSavepointDirectory());
}
 
Example #20
Source File: RocksDBIncrementalRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private IncrementalLocalKeyedStateHandle transferRemoteStateToLocalDirectory(
	Path temporaryRestoreInstancePath,
	IncrementalRemoteKeyedStateHandle restoreStateHandle) throws Exception {

	try (RocksDBStateDownloader rocksDBStateDownloader = new RocksDBStateDownloader(numberOfTransferringThreads)) {
		rocksDBStateDownloader.transferAllStateDataToDirectory(
			restoreStateHandle,
			temporaryRestoreInstancePath,
			cancelStreamRegistry);
	}

	// since we transferred all remote state to a local directory, we can use the same code as for
	// local recovery.
	return new IncrementalLocalKeyedStateHandle(
		restoreStateHandle.getBackendIdentifier(),
		restoreStateHandle.getCheckpointId(),
		new DirectoryStateHandle(temporaryRestoreInstancePath),
		restoreStateHandle.getKeyGroupRange(),
		restoreStateHandle.getMetaStateHandle(),
		restoreStateHandle.getSharedState().keySet());
}
 
Example #21
Source File: BootstrapTransformationTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testOperatorSpecificMaxParallelismRespected() {
	ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(4);

	DataSource<Integer> input = env.fromElements(0);

	BootstrapTransformation<Integer> transformation = OperatorTransformation
		.bootstrapWith(input)
		.setMaxParallelism(1)
		.transform(new ExampleStateBootstrapFunction());

	int maxParallelism = transformation.getMaxParallelism(4);
	DataSet<TaggedOperatorSubtaskState> result = transformation.writeOperatorSubtaskStates(
		OperatorIDGenerator.fromUid("uid"),
		new MemoryStateBackend(),
		new Path(),
		maxParallelism
	);

	Assert.assertEquals("The parallelism of a data set should be constrained my the savepoint max parallelism", 1, getParallelism(result));
}
 
Example #22
Source File: FileInputFormat.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 *
 * @return The path of the file to read.
 *
 * @deprecated Please use getFilePaths() instead.
 */
@Deprecated
public Path getFilePath() {

	if (supportsMultiPaths()) {
		if (this.filePaths == null || this.filePaths.length == 0) {
			return null;
		} else if (this.filePaths.length == 1) {
			return this.filePaths[0];
		} else {
			throw new UnsupportedOperationException(
				"FileInputFormat is configured with multiple paths. Use getFilePaths() instead.");
		}
	} else {
		return filePath;
	}
}
 
Example #23
Source File: ContinuousFileProcessingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void createHDFS() {
	Assume.assumeTrue("HDFS cluster cannot be start on Windows without extensions.", !OperatingSystem.isWindows());

	try {
		File hdfsDir = tempFolder.newFolder();

		org.apache.hadoop.conf.Configuration hdConf = new org.apache.hadoop.conf.Configuration();
		hdConf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, hdfsDir.getAbsolutePath());
		hdConf.set("dfs.block.size", String.valueOf(1048576)); // this is the minimum we can set.

		MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(hdConf);
		hdfsCluster = builder.build();

		hdfsURI = "hdfs://" + hdfsCluster.getURI().getHost() + ":" + hdfsCluster.getNameNodePort() + "/";
		hdfs = new org.apache.hadoop.fs.Path(hdfsURI).getFileSystem(hdConf);

	} catch (Throwable e) {
		e.printStackTrace();
		Assert.fail("Test failed " + e.getMessage());
	}
}
 
Example #24
Source File: ParquetPojoInputFormatTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testReadPojoFromSimpleRecord() throws IOException {
	Tuple3<Class<? extends SpecificRecord>, SpecificRecord, Row> simple = TestUtil.getSimpleRecordTestData();
	MessageType messageType = SCHEMA_CONVERTER.convert(TestUtil.SIMPLE_SCHEMA);
	Path path = TestUtil.createTempParquetFile(tempRoot.getRoot(), TestUtil.SIMPLE_SCHEMA, Collections.singletonList(simple.f1));

	ParquetPojoInputFormat<PojoSimpleRecord> inputFormat = new ParquetPojoInputFormat<>(
		path, messageType, (PojoTypeInfo<PojoSimpleRecord>) Types.POJO(PojoSimpleRecord.class));
	inputFormat.setRuntimeContext(TestUtil.getMockRuntimeContext());

	FileInputSplit[] splits = inputFormat.createInputSplits(1);
	assertEquals(1, splits.length);
	inputFormat.open(splits[0]);

	PojoSimpleRecord simpleRecord = inputFormat.nextRecord(null);
	assertEquals(simple.f2.getField(0), simpleRecord.getFoo());
	assertEquals(simple.f2.getField(1), simpleRecord.getBar());
	assertArrayEquals((Long[]) simple.f2.getField(2), simpleRecord.getArr());
}
 
Example #25
Source File: SnapshotUtilsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSnapshotUtilsLifecycle() throws Exception {
	StreamOperator<Void> operator 		= new LifecycleOperator();
	CheckpointStorageWorkerView storage = new MockStateBackend().createCheckpointStorage(new JobID());

	Path path = new Path(folder.newFolder().getAbsolutePath());

	SnapshotUtils.snapshot(operator, 0, 0L, storage, path);

	Assert.assertEquals(EXPECTED_CALL_OPERATOR_SNAPSHOT, ACTUAL_ORDER_TRACKING);
}
 
Example #26
Source File: FileInputFormatTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPathsWithoutSettingFirst() {
	final DummyFileInputFormat format = new DummyFileInputFormat();
	
	Path[] paths = format.getFilePaths();
	Assert.assertNotNull("Paths should not be null.", paths);
	Assert.assertEquals("Paths size should be 0.", 0, paths.length);
}
 
Example #27
Source File: BucketFactory.java    From flink with Apache License 2.0 5 votes vote down vote up
Bucket<IN, BucketID> getNewBucket(
final int subtaskIndex,
final BucketID bucketId,
final Path bucketPath,
final long initialPartCounter,
final BucketWriter<IN, BucketID> bucketWriter,
final RollingPolicy<IN, BucketID> rollingPolicy,
final OutputFileConfig outputFileConfig) throws IOException;
 
Example #28
Source File: TaskTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up
public void registerFileOutputTask(
	FileOutputFormat<Record> outputFormat,
	String outPath,
	Configuration formatParams) {

	outputFormat.setOutputFilePath(new Path(outPath));
	outputFormat.setWriteMode(WriteMode.OVERWRITE);

	OperatorID operatorID = new OperatorID();
	new InputOutputFormatContainer(Thread.currentThread().getContextClassLoader())
		.addOutputFormat(operatorID, outputFormat)
		.addParameters(operatorID, formatParams)
		.write(new TaskConfig(this.mockEnv.getTaskConfiguration()));
}
 
Example #29
Source File: AvroTypeExtractionTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSerializeWithAvro() throws Exception {
	final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
	env.getConfig().enableForceAvro();
	Path in = new Path(inFile.getAbsoluteFile().toURI());

	AvroInputFormat<User> users = new AvroInputFormat<>(in, User.class);
	DataSet<User> usersDS = env.createInput(users)
			.map((MapFunction<User, User>) value -> {
				Map<CharSequence, Long> ab = new HashMap<>(1);
				ab.put("hehe", 12L);
				value.setTypeMap(ab);
				return value;
			});

	usersDS.writeAsText(resultPath);

	env.execute("Simple Avro read job");

	expected = "{\"name\": \"Alyssa\", \"favorite_number\": 256, \"favorite_color\": null, \"type_long_test\": null, \"type_double_test\": 123.45, \"type_null_test\": null, " +
		"\"type_bool_test\": true, \"type_array_string\": [\"ELEMENT 1\", \"ELEMENT 2\"], \"type_array_boolean\": [true, false], \"type_nullable_array\": null, " +
		"\"type_enum\": \"GREEN\", \"type_map\": {\"hehe\": 12}, \"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", " +
		"\"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, \"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, " +
		"\"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, \"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, " +
		"\"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, \"type_decimal_fixed\": [7, -48]}\n" +
		"{\"name\": \"Charlie\", \"favorite_number\": null, \"favorite_color\": \"blue\", \"type_long_test\": 1337, \"type_double_test\": 1.337, \"type_null_test\": null, " +
		"\"type_bool_test\": false, \"type_array_string\": [], \"type_array_boolean\": [], \"type_nullable_array\": null, \"type_enum\": \"RED\", \"type_map\": {\"hehe\": 12}, " +
		"\"type_fixed\": null, \"type_union\": null, \"type_nested\": {\"num\": 239, \"street\": \"Baker Street\", \"city\": \"London\", \"state\": \"London\", \"zip\": \"NW1 6XE\"}, " +
		"\"type_bytes\": {\"bytes\": \"\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\\u0000\"}, \"type_date\": 2014-03-01, \"type_time_millis\": 12:12:12.000, " +
		"\"type_time_micros\": 123456, \"type_timestamp_millis\": 2014-03-01T12:12:12.321Z, \"type_timestamp_micros\": 123456, \"type_decimal_bytes\": {\"bytes\": \"\\u0007Ð\"}, " +
		"\"type_decimal_fixed\": [7, -48]}\n";

}
 
Example #30
Source File: JobGraph.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Adds the path of a JAR file required to run the job on a task manager.
 *
 * @param jar
 *        path of the JAR file required to run the job on a task manager
 */
public void addJar(Path jar) {
	if (jar == null) {
		throw new IllegalArgumentException();
	}

	if (!userJars.contains(jar)) {
		userJars.add(jar);
	}
}