Java Code Examples for org.apache.flink.runtime.state.CheckpointStreamFactory.CheckpointStateOutputStream#closeAndGetHandle()

The following examples show how to use org.apache.flink.runtime.state.CheckpointStreamFactory.CheckpointStateOutputStream#closeAndGetHandle() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: MemoryCheckpointOutputStreamTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}

Example 2

Source File: MemoryCheckpointOutputStreamTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}

Example 3

Source File: MemoryCheckpointOutputStreamTest.java From flink with Apache License 2.0

6 votes

@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}

Example 4

Source File: MemoryCheckpointOutputStreamTest.java From flink with Apache License 2.0

6 votes

@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}

Example 5

Source File: MemoryCheckpointOutputStreamTest.java From flink with Apache License 2.0

6 votes

@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}

Example 6

Source File: MemoryCheckpointOutputStreamTest.java From flink with Apache License 2.0

6 votes

@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}

Example 7

Source File: FsCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
	final FileSystem fs = LocalFileSystem.getSharedInstance();
	final Path checkpointDir = randomTempPath();
	final Path sharedStateDir = randomTempPath();

	FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(
			fs,
			checkpointDir,
			sharedStateDir,
			randomTempPath(),
			CheckpointStorageLocationReference.getDefault(),
			FILE_SIZE_THRESHOLD);

	assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());

	assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create exclusive state

	CheckpointStateOutputStream exclusiveStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);

	exclusiveStream.write(42);
	exclusiveStream.flush();
	StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create shared state

	CheckpointStateOutputStream sharedStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);

	sharedStream.write(42);
	sharedStream.flush();
	StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// drop state

	exclusiveHandle.discardState();
	sharedHandle.discardState();
}

Example 8

Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0

4 votes

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}

Example 9

Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0

4 votes

@Test
public void testDirectoriesForExclusiveAndSharedState() throws Exception {
	final FileSystem fs = LocalFileSystem.getSharedInstance();
	final Path checkpointDir = randomTempPath();
	final Path sharedStateDir = randomTempPath();

	FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation(
			fs,
			checkpointDir,
			sharedStateDir,
			randomTempPath(),
			CheckpointStorageLocationReference.getDefault(),
			FILE_SIZE_THRESHOLD,
			WRITE_BUFFER_SIZE);

	assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory());

	assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create exclusive state

	CheckpointStateOutputStream exclusiveStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);

	exclusiveStream.write(42);
	exclusiveStream.flush();
	StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// create shared state

	CheckpointStateOutputStream sharedStream =
			storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);

	sharedStream.write(42);
	sharedStream.flush();
	StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle();

	assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length);
	assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length);

	// drop state

	exclusiveHandle.discardState();
	sharedHandle.discardState();
}

Example 10

Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}

Example 11

Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0

4 votes

@Test
public void testMixedBelowAndAboveThreshold() throws Exception {
	final byte[] state1 = new byte[1274673];
	final byte[] state2 = new byte[1];
	final byte[] state3 = new byte[0];
	final byte[] state4 = new byte[177];

	final Random rnd = new Random();
	rnd.nextBytes(state1);
	rnd.nextBytes(state2);
	rnd.nextBytes(state3);
	rnd.nextBytes(state4);

	final File directory = tempDir.newFolder();
	final Path basePath = Path.fromLocalFile(directory);

	final Supplier<CheckpointStateOutputStream> factory = () ->
			new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths);

	CheckpointStateOutputStream stream1 = factory.get();
	CheckpointStateOutputStream stream2 = factory.get();
	CheckpointStateOutputStream stream3 = factory.get();

	stream1.write(state1);
	stream2.write(state2);
	stream3.write(state3);

	FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
	ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
	ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();

	// use with try-with-resources
	StreamStateHandle handle4;
	try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) {
		stream4.write(state4);
		handle4 = stream4.closeAndGetHandle();
	}

	// close before accessing handle
	CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get();
	stream5.write(state4);
	stream5.close();
	try {
		stream5.closeAndGetHandle();
		fail();
	} catch (IOException e) {
		// uh-huh
	}

	validateBytesInStream(handle1.openInputStream(), state1);
	handle1.discardState();
	assertFalse(isDirectoryEmpty(directory));
	ensureLocalFileDeleted(handle1.getFilePath());

	validateBytesInStream(handle2.openInputStream(), state2);
	handle2.discardState();
	assertFalse(isDirectoryEmpty(directory));

	// nothing was written to the stream, so it will return nothing
	assertNull(handle3);
	assertFalse(isDirectoryEmpty(directory));

	validateBytesInStream(handle4.openInputStream(), state4);
	handle4.discardState();
	assertTrue(isDirectoryEmpty(directory));
}