Java Code Examples for org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream

The following examples show how to use org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 2
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 3
Source Project: Flink-CEPplus   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() throws IOException {
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			FsCheckpointStreamFactory.DEFAULT_WRITE_BUFFER_SIZE,
			fileSizeThreshold);
}
 
Example 4
Source Project: Flink-CEPplus   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 5
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 
Example 6
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 7
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 8
/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 
Example 9
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() {
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			writeBufferSize,
			fileSizeThreshold);
}
 
Example 10
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 11
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 
Example 12
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 13
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 14
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	assertTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 
Example 15
Source Project: sylph   Source File: SylphFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream()
        throws IOException
{
    return new FsCheckpointStateOutputStream(
            taskOwnedStateDirectory,
            fileSystem,
            CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue(), //todo:FsStateBackend#getWriteBufferSize
            fileSizeThreshold);
}
 
Example 16
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() {
	// as the comment of CheckpointStorageWorkerView#createTaskOwnedStateStream said we may change into shared state,
	// so we use CheckpointedStateScope.SHARED here.
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			writeBufferSize,
			fileSizeThreshold);
}
 
Example 17
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 18
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertNull(handle);
}
 
Example 19
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 20
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 21
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 22
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that the stream does not check and clean the parent directory
 * when encountering a write error.
 */
@Test
public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception {
	final File directory = tempDir.newFolder();

	// prevent creation of files in that directory
	// this operation does not work reliably on Windows, so we use an "assume" to skip the test
	// is this prerequisite operation is not supported.
	assumeTrue(directory.setWritable(false, true));
	checkDirectoryNotWritable(directory);

	FileSystem fs = spy(FileSystem.getLocalFileSystem());

	FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);

	FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream(
			Path.fromLocalFile(directory), fs, 1024, 1, relativePaths);

	stream1.write(new byte[61]);
	stream2.write(new byte[61]);

	try {
		stream1.closeAndGetHandle();
		fail("this should fail with an exception");
	} catch (IOException ignored) {}

	stream2.close();

	// no delete call must have happened
	verify(fs, times(0)).delete(any(Path.class), anyBoolean());

	// the directory must still exist as a proper directory
	assertTrue(directory.exists());
	assertTrue(directory.isDirectory());
}
 
Example 23
@Test(expected = IllegalArgumentException.class)
public void testWrongParameters() throws Exception {
	// this should fail
	new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000);
}
 
Example 24
private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
		new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
				Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold);

	Random rnd = new Random();
	byte[] original = new byte[numBytes];
	byte[] bytes = new byte[original.length];

	rnd.nextBytes(original);
	System.arraycopy(original, 0, bytes, 0, original.length);

	// the test writes a mixture of writing individual bytes and byte arrays
	int pos = 0;
	while (pos < bytes.length) {
		boolean single = rnd.nextBoolean();
		if (single) {
			stream.write(bytes[pos++]);
		}
		else {
			int num = rnd.nextInt(Math.min(10, bytes.length - pos));
			stream.write(bytes, pos, num);
			pos += num;
		}
	}

	StreamStateHandle handle = stream.closeAndGetHandle();
	if (expectFile) {
		assertTrue(handle instanceof FileStateHandle);
	} else {
		assertTrue(handle instanceof ByteStreamStateHandle);
	}

	// make sure the writing process did not alter the original byte array
	assertArrayEquals(original, bytes);

	try (InputStream inStream = handle.openInputStream()) {
		byte[] validation = new byte[bytes.length];

		DataInputStream dataInputStream = new DataInputStream(inStream);
		dataInputStream.readFully(validation);

		assertArrayEquals(bytes, validation);
	}

	handle.discardState();
}
 
Example 25
@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 
Example 26
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testWrongParameters() throws Exception {
	// this should fail
	new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000);
}
 
Example 27
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 4 votes vote down vote up
private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
		new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
				Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold);

	Random rnd = new Random();
	byte[] original = new byte[numBytes];
	byte[] bytes = new byte[original.length];

	rnd.nextBytes(original);
	System.arraycopy(original, 0, bytes, 0, original.length);

	// the test writes a mixture of writing individual bytes and byte arrays
	int pos = 0;
	while (pos < bytes.length) {
		boolean single = rnd.nextBoolean();
		if (single) {
			stream.write(bytes[pos++]);
		}
		else {
			int num = rnd.nextBoolean() ?
				(bytes.length - pos) : rnd.nextInt(bytes.length - pos);
			stream.write(bytes, pos, num);
			pos += num;
		}
	}

	StreamStateHandle handle = stream.closeAndGetHandle();
	if (expectFile) {
		assertTrue(handle instanceof FileStateHandle);
	} else {
		assertTrue(handle instanceof ByteStreamStateHandle);
	}

	// make sure the writing process did not alter the original byte array
	assertArrayEquals(original, bytes);

	try (InputStream inStream = handle.openInputStream()) {
		byte[] validation = new byte[bytes.length];

		DataInputStream dataInputStream = new DataInputStream(inStream);
		dataInputStream.readFully(validation);

		assertArrayEquals(bytes, validation);
	}

	handle.discardState();
}
 
Example 28
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}
 
Example 29
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
	final FileSystem fs = LocalFileSystem.getSharedInstance();
	final Path checkpointDir = randomTempPath();
	final Path sharedStateDir = randomTempPath();

	FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(
			fs,
			checkpointDir,
			sharedStateDir,
			randomTempPath(),
			CheckpointStorageLocationReference.getDefault(),
			FILE_SIZE_THRESHOLD,
			WRITE_BUFFER_SIZE);

	assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());

	assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create exclusive state

	FsCheckpointStateOutputStream exclusiveStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);

	exclusiveStream.write(42);
	exclusiveStream.flushToFile();
	StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create shared state

	FsCheckpointStateOutputStream sharedStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);

	sharedStream.write(42);
	sharedStream.flushToFile();
	StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// drop state

	exclusiveHandle.discardState();
	sharedHandle.discardState();
}
 
Example 30
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test(expected = IllegalArgumentException.class)
public void testWrongParameters() throws Exception {
	// this should fail
	new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000, relativePaths);
}