Java Code Examples for org.apache.flink.core.fs.Path#fromLocalFile()

The following examples show how to use org.apache.flink.core.fs.Path#fromLocalFile() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 2
@Test
public void testSavepointsInOneDirectoryDefaultLocation() throws Exception {
	final Path defaultSavepointDir = Path.fromLocalFile(tmp.newFolder());

	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()), defaultSavepointDir, new JobID(), FILE_SIZE_THRESHOLD);

	final FsCheckpointStorageLocation savepointLocation = (FsCheckpointStorageLocation)
			storage.initializeLocationForSavepoint(52452L, null);

	// all state types should be in the expected location
	assertParent(defaultSavepointDir, savepointLocation.getCheckpointDirectory());
	assertParent(defaultSavepointDir, savepointLocation.getSharedStateDirectory());
	assertParent(defaultSavepointDir, savepointLocation.getTaskOwnedStateDirectory());

	// cleanup
	savepointLocation.disposeOnFailure();
}
 
Example 3
Source Project: flink   File: CompressionFactoryITCase.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testWriteCompressedFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<String> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(String.class)
	);

	stream.map(str -> str).addSink(
			StreamingFileSink.forBulkFormat(
					testPath,
					CompressWriters.forExtractor(new DefaultExtractor<String>()).withHadoopCompression(TEST_CODEC_NAME)
			).build());

	env.execute();

	validateResults(folder, testData, new CompressionCodecFactory(configuration).getCodecByName(TEST_CODEC_NAME));
}
 
Example 4
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 5
@Test
public void testWriteSequenceFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Tuple2<Long, String>> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {

			})
	);

	stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() {
		@Override
		public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception {
			return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1));
		}
	}).addSink(
		StreamingFileSink.forBulkFormat(
			testPath,
			new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2")
		).build());

	env.execute();

	validateResults(folder, testData);
}
 
Example 6
Source Project: Flink-CEPplus   File: FileCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Path call() throws IOException {
	final File file = blobService.getFile(jobID, blobKey);

	if (isDirectory) {
		Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target);
		return directory;
	} else {
		//noinspection ResultOfMethodCallIgnored
		file.setExecutable(isExecutable);
		return Path.fromLocalFile(file);
	}

}
 
Example 7
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 8
@Test
public void testWriteSequenceFile() throws Exception {
	final File folder = TEMPORARY_FOLDER.newFolder();
	final Path testPath = Path.fromLocalFile(folder);

	final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.setParallelism(1);
	env.enableCheckpointing(100);

	DataStream<Tuple2<Long, String>> stream = env.addSource(
			new FiniteTestSource<>(testData),
			TypeInformation.of(new TypeHint<Tuple2<Long, String>>() {

			})
	);

	stream.map(new MapFunction<Tuple2<Long, String>, Tuple2<LongWritable, Text>>() {
		@Override
		public Tuple2<LongWritable, Text> map(Tuple2<Long, String> value) throws Exception {
			return new Tuple2<>(new LongWritable(value.f0), new Text(value.f1));
		}
	}).addSink(
		StreamingFileSink.forBulkFormat(
			testPath,
			new SequenceFileWriterFactory<>(configuration, LongWritable.class, Text.class, "BZip2")
		).build());

	env.execute();

	validateResults(folder, testData);
}
 
Example 9
private void testSavepoint(
		@Nullable Path savepointDir,
		@Nullable Path customDir,
		Path expectedParent) throws Exception {

	final CheckpointStorage storage = savepointDir == null ?
			createCheckpointStorage(randomTempPath()) :
			createCheckpointStorageWithSavepointDir(randomTempPath(), savepointDir);

	final String customLocation = customDir == null ? null : customDir.toString();

	final CheckpointStorageLocation savepointLocation =
			storage.initializeLocationForSavepoint(52452L, customLocation);

	final byte[] data = {77, 66, 55, 99, 88};

	final CompletedCheckpointStorageLocation completed;
	try (CheckpointMetadataOutputStream out = savepointLocation.createMetadataOutputStream()) {
		out.write(data);
		completed = out.closeAndFinalizeCheckpoint();
	}

	// we need to do this step to make sure we have a slash (or not) in the same way as the
	// expected path has it
	final Path normalizedWithSlash = Path.fromLocalFile(new File(new Path(completed.getExternalPointer()).getParent().getPath()));

	assertEquals(expectedParent, normalizedWithSlash);
	validateContents(completed.getMetadataHandle(), data);

	// validate that the correct directory was used
	FileStateHandle fileStateHandle = (FileStateHandle) completed.getMetadataHandle();

	// we need to recreate that path in the same way as the "expected path" (via File and URI) to make
	// sure the either both use '/' suffixes, or neither uses them (a bit of an annoying ambiguity)
	Path usedSavepointDir = new Path(new File(fileStateHandle.getFilePath().getParent().getParent().getPath()).toURI());

	assertEquals(expectedParent, usedSavepointDir);
}
 
Example 10
@Test
public void testDisposeDeletesFile() throws Exception {
	File file = tempFolder.newFile();
	writeTestData(file);
	assertTrue(file.exists());

	FileStateHandle handle = new FileStateHandle(Path.fromLocalFile(file), file.length());
	handle.discardState();
	assertFalse(file.exists());
}
 
Example 11
@Test
public void testPointerPathResolution() throws Exception {
	final FileSystem fs = FileSystem.getLocalFileSystem();
	final Path metadataFile = new Path(Path.fromLocalFile(tmp.newFolder()), AbstractFsCheckpointStorage.METADATA_FILE_NAME);

	final String basePointer = metadataFile.getParent().toString();

	final String pointer1 = metadataFile.toString();
	final String pointer2 = metadataFile.getParent().toString();
	final String pointer3 = metadataFile.getParent().toString() + '/';

	// create the storage for some random checkpoint directory
	final CheckpointStorage storage = createCheckpointStorage(randomTempPath());

	final byte[] data = new byte[23686];
	new Random().nextBytes(data);
	try (FSDataOutputStream out = fs.create(metadataFile, WriteMode.NO_OVERWRITE)) {
		out.write(data);
	}

	CompletedCheckpointStorageLocation completed1 = storage.resolveCheckpoint(pointer1);
	CompletedCheckpointStorageLocation completed2 = storage.resolveCheckpoint(pointer2);
	CompletedCheckpointStorageLocation completed3 = storage.resolveCheckpoint(pointer3);

	assertEquals(basePointer, completed1.getExternalPointer());
	assertEquals(basePointer, completed2.getExternalPointer());
	assertEquals(basePointer, completed3.getExternalPointer());

	StreamStateHandle handle1 = completed1.getMetadataHandle();
	StreamStateHandle handle2 = completed2.getMetadataHandle();
	StreamStateHandle handle3 = completed3.getMetadataHandle();

	assertNotNull(handle1);
	assertNotNull(handle2);
	assertNotNull(handle3);

	validateContents(handle1, data);
	validateContents(handle2, data);
	validateContents(handle3, data);
}
 
Example 12
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertNull(handle);
}
 
Example 13
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 14
Source Project: flink   File: FileCache.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Path call() throws IOException {
	final File file = blobService.getFile(jobID, blobKey);

	if (isDirectory) {
		Path directory = FileUtils.expandDirectory(new Path(file.getAbsolutePath()), target);
		return directory;
	} else {
		//noinspection ResultOfMethodCallIgnored
		file.setExecutable(isExecutable);
		return Path.fromLocalFile(file);
	}

}
 
Example 15
/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 
Example 16
@Test(expected = IllegalArgumentException.class)
public void testWrongParameters() throws Exception {
	// this should fail
	new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000, relativePaths);
}
 
Example 17
@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 
Example 18
public Path randomTempPath() throws IOException {
	return Path.fromLocalFile(tmp.newFolder());
}
 
Example 19
@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 
Example 20
@Before
public void createStateDirectories() throws IOException {
	exclusiveStateDir = Path.fromLocalFile(TMP.newFolder("exclusive"));
	sharedStateDir = Path.fromLocalFile(TMP.newFolder("shared"));
}