Java Code Examples for org.apache.flink.runtime.state.CheckpointStreamFactory.CheckpointStateOutputStream

The following examples show how to use org.apache.flink.runtime.state.CheckpointStreamFactory.CheckpointStateOutputStream. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 2
Source Project: Flink-CEPplus   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof MemoryCheckpointOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 3
@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}
 
Example 4
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example 5
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 6
Source Project: flink   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof MemoryCheckpointOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 7
Source Project: flink   Source File: MemoryCheckpointOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}
 
Example 8
Source Project: flink   Source File: MemoryCheckpointOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example 9
Source Project: flink   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof MemoryCheckpointOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 10
Source Project: flink   Source File: MemoryCheckpointOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testOversizedState() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(10);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);

	oos.writeObject(state);
	oos.flush();

	try {
		outStream.closeAndGetHandle();
		fail("this should cause an exception");
	}
	catch (IOException e) {
		// that's what we expect
	}
}
 
Example 11
Source Project: flink   Source File: MemoryCheckpointOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() throws IOException {
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			FsCheckpointStreamFactory.DEFAULT_WRITE_BUFFER_SIZE,
			fileSizeThreshold);
}
 
Example 13
Source Project: Flink-CEPplus   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 14
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 
Example 15
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 16
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 17
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() {
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			writeBufferSize,
			fileSizeThreshold);
}
 
Example 18
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 19
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertTrue(handle == null);
}
 
Example 20
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 21
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 22
Source Project: sylph   Source File: SylphFsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream()
        throws IOException
{
    return new FsCheckpointStateOutputStream(
            taskOwnedStateDirectory,
            fileSystem,
            CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue(), //todo:FsStateBackend#getWriteBufferSize
            fileSizeThreshold);
}
 
Example 23
Source Project: flink   Source File: ChannelStateCheckpointWriter.java    License: Apache License 2.0 5 votes vote down vote up
ChannelStateCheckpointWriter(
		long checkpointId,
		ChannelStateWriteResult result,
		CheckpointStateOutputStream stream,
		ChannelStateSerializer serializer,
		RunnableWithException onComplete) throws Exception {
	this(checkpointId, result, serializer, onComplete, stream, new DataOutputStream(stream));
}
 
Example 24
Source Project: flink   Source File: ChannelStateCheckpointWriter.java    License: Apache License 2.0 5 votes vote down vote up
ChannelStateCheckpointWriter(
		long checkpointId,
		ChannelStateWriteResult result,
		ChannelStateSerializer serializer,
		RunnableWithException onComplete,
		CheckpointStateOutputStream checkpointStateOutputStream,
		DataOutputStream dataStream) throws Exception {
	this.checkpointId = checkpointId;
	this.result = checkNotNull(result);
	this.checkpointStream = checkNotNull(checkpointStateOutputStream);
	this.serializer = checkNotNull(serializer);
	this.dataStream = checkNotNull(dataStream);
	this.onComplete = checkNotNull(onComplete);
	runWithChecks(() -> serializer.writeHeader(dataStream));
}
 
Example 25
Source Project: flink   Source File: FsCheckpointStorage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public CheckpointStateOutputStream createTaskOwnedStateStream() {
	// as the comment of CheckpointStorageWorkerView#createTaskOwnedStateStream said we may change into shared state,
	// so we use CheckpointedStateScope.SHARED here.
	return new FsCheckpointStateOutputStream(
			taskOwnedStateDirectory,
			fileSystem,
			writeBufferSize,
			fileSizeThreshold);
}
 
Example 26
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 27
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyState() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths);

	StreamStateHandle handle = stream.closeAndGetHandle();
	assertNull(handle);
}
 
Example 28
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testGetPos() throws Exception {
	FsCheckpointStreamFactory.CheckpointStateOutputStream stream =
			new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
					Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);

	for (int i = 0; i < 64; ++i) {
		Assert.assertEquals(i, stream.getPos());
		stream.write(0x42);
	}

	stream.closeAndGetHandle();

	// ----------------------------------------------------

	stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
			Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths);

	byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET);

	for (int i = 0; i < 7; ++i) {
		Assert.assertEquals(i * (1 + data.length), stream.getPos());
		stream.write(0x42);
		stream.write(data);
	}

	stream.closeAndGetHandle();
}
 
Example 29
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted upon calling close.
 */
@Test
public void testCleanupWhenClosingStream() throws IOException {

	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	stream.close();

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}
 
Example 30
Source Project: flink   Source File: FsCheckpointStateOutputStreamTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails.
 */
@Test
public void testCleanupWhenFailingCloseAndGetHandle() throws IOException {
	final FileSystem fs = mock(FileSystem.class);
	final FSDataOutputStream outputStream = mock(FSDataOutputStream.class);

	final ArgumentCaptor<Path>  pathCaptor = ArgumentCaptor.forClass(Path.class);

	when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream);
	doThrow(new IOException("Test IOException.")).when(outputStream).close();

	CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(
		Path.fromLocalFile(tempDir.newFolder()),
		fs,
		4,
		0,
		relativePaths);

	// this should create the underlying file stream
	stream.write(new byte[] {1, 2, 3, 4, 5});

	verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class));

	try {
		stream.closeAndGetHandle();
		fail("Expected IOException");
	} catch (IOException ioE) {
		// expected exception
	}

	verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean());
}