Java Code Examples for org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream#closeAndGetHandle()

/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 

/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 

/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	// this operation does not work reliably on Windows, so we use an "assume" to skip the test
	// is this prerequisite operation is not supported.
	assumeTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 

@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
	final FileSystem fs = LocalFileSystem.getSharedInstance();
	final Path checkpointDir = randomTempPath();
	final Path sharedStateDir = randomTempPath();

	FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(
			fs,
			checkpointDir,
			sharedStateDir,
			randomTempPath(),
			CheckpointStorageLocationReference.getDefault(),
			FILE_SIZE_THRESHOLD,
			WRITE_BUFFER_SIZE);

	assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());

	assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create exclusive state

	FsCheckpointStateOutputStream exclusiveStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);

	exclusiveStream.write(42);
	exclusiveStream.flushToFile();
	StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create shared state

	FsCheckpointStateOutputStream sharedStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);

	sharedStream.write(42);
	sharedStream.flushToFile();
	StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// drop state

	exclusiveHandle.discardState();
	sharedHandle.discardState();
}