org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream Java Examples
The following examples show how to use
org.apache.flink.runtime.state.filesystem.FsCheckpointStreamFactory.FsCheckpointStateOutputStream.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #2
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 6 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #3
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails. */ @Test public void testCleanupWhenFailingCloseAndGetHandle() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); doThrow(new IOException("Test IOException.")).when(outputStream).close(); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); try { stream.closeAndGetHandle(); fail("Expected IOException"); } catch (IOException ioE) { // expected exception } verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #4
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * This test checks that the stream does not check and clean the parent directory * when encountering a write error. */ @Test public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception { final File directory = tempDir.newFolder(); // prevent creation of files in that directory // this operation does not work reliably on Windows, so we use an "assume" to skip the test // is this prerequisite operation is not supported. assumeTrue(directory.setWritable(false, true)); checkDirectoryNotWritable(directory); FileSystem fs = spy(FileSystem.getLocalFileSystem()); FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1, relativePaths); FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1, relativePaths); stream1.write(new byte[61]); stream2.write(new byte[61]); try { stream1.closeAndGetHandle(); fail("this should fail with an exception"); } catch (IOException ignored) {} stream2.close(); // no delete call must have happened verify(fs, times(0)).delete(any(Path.class), anyBoolean()); // the directory must still exist as a proper directory assertTrue(directory.exists()); assertTrue(directory.isDirectory()); }
Example #5
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails. */ @Test public void testCleanupWhenFailingCloseAndGetHandle() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); doThrow(new IOException("Test IOException.")).when(outputStream).close(); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); try { stream.closeAndGetHandle(); fail("Expected IOException"); } catch (IOException ioE) { // expected exception } verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #6
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * Tests that the underlying stream file is deleted upon calling close. */ @Test public void testCleanupWhenClosingStream() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0, relativePaths); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); stream.close(); verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #7
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetPos() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths); for (int i = 0; i < 64; ++i) { Assert.assertEquals(i, stream.getPos()); stream.write(0x42); } stream.closeAndGetHandle(); // ---------------------------------------------------- stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17, relativePaths); byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET); for (int i = 0; i < 7; ++i) { Assert.assertEquals(i * (1 + data.length), stream.getPos()); stream.write(0x42); stream.write(data); } stream.closeAndGetHandle(); }
Example #8
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testEmptyState() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512, relativePaths); StreamStateHandle handle = stream.closeAndGetHandle(); assertNull(handle); }
Example #9
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTaskOwnedStateStream() throws Exception { final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter"); // we chose a small size threshold here to force the state to disk final FsCheckpointStorage storage = new FsCheckpointStorage( Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10, WRITE_BUFFER_SIZE); final StreamStateHandle stateHandle; try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) { assertTrue(stream instanceof FsCheckpointStateOutputStream); new ObjectOutputStream(stream).writeObject(state); stateHandle = stream.closeAndGetHandle(); } // the state must have gone to disk FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; // check that the state is in the correct directory String parentDirName = fileStateHandle.getFilePath().getParent().getName(); assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName); // validate the contents try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) { assertEquals(state, in.readObject()); } }
Example #10
Source File: FsCheckpointStorage.java From flink with Apache License 2.0 | 5 votes |
@Override public CheckpointStateOutputStream createTaskOwnedStateStream() { // as the comment of CheckpointStorageWorkerView#createTaskOwnedStateStream said we may change into shared state, // so we use CheckpointedStateScope.SHARED here. return new FsCheckpointStateOutputStream( taskOwnedStateDirectory, fileSystem, writeBufferSize, fileSizeThreshold); }
Example #11
Source File: SylphFsCheckpointStorage.java From sylph with Apache License 2.0 | 5 votes |
@Override public CheckpointStateOutputStream createTaskOwnedStateStream() throws IOException { return new FsCheckpointStateOutputStream( taskOwnedStateDirectory, fileSystem, CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue(), //todo:FsStateBackend#getWriteBufferSize fileSizeThreshold); }
Example #12
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
/** * This test checks that the stream does not check and clean the parent directory * when encountering a write error. */ @Test public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception { final File directory = tempDir.newFolder(); // prevent creation of files in that directory assertTrue(directory.setWritable(false, true)); checkDirectoryNotWritable(directory); FileSystem fs = spy(FileSystem.getLocalFileSystem()); FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); stream1.write(new byte[61]); stream2.write(new byte[61]); try { stream1.closeAndGetHandle(); fail("this should fail with an exception"); } catch (IOException ignored) {} stream2.close(); // no delete call must have happened verify(fs, times(0)).delete(any(Path.class), anyBoolean()); // the directory must still exist as a proper directory assertTrue(directory.exists()); assertTrue(directory.isDirectory()); }
Example #13
Source File: FsCheckpointStorage.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Override public CheckpointStateOutputStream createTaskOwnedStateStream() throws IOException { return new FsCheckpointStateOutputStream( taskOwnedStateDirectory, fileSystem, FsCheckpointStreamFactory.DEFAULT_WRITE_BUFFER_SIZE, fileSizeThreshold); }
Example #14
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testGetPos() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17); for (int i = 0; i < 64; ++i) { Assert.assertEquals(i, stream.getPos()); stream.write(0x42); } stream.closeAndGetHandle(); // ---------------------------------------------------- stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17); byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET); for (int i = 0; i < 7; ++i) { Assert.assertEquals(i * (1 + data.length), stream.getPos()); stream.write(0x42); stream.write(data); } stream.closeAndGetHandle(); }
Example #15
Source File: FsCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testTaskOwnedStateStream() throws Exception { final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter"); // we chose a small size threshold here to force the state to disk final FsCheckpointStorage storage = new FsCheckpointStorage( Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10); final StreamStateHandle stateHandle; try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) { assertTrue(stream instanceof FsCheckpointStateOutputStream); new ObjectOutputStream(stream).writeObject(state); stateHandle = stream.closeAndGetHandle(); } // the state must have gone to disk FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; // check that the state is in the correct directory String parentDirName = fileStateHandle.getFilePath().getParent().getName(); assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName); // validate the contents try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) { assertEquals(state, in.readObject()); } }
Example #16
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTaskOwnedStateStream() throws Exception { final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter"); // we chose a small size threshold here to force the state to disk final FsCheckpointStorage storage = new FsCheckpointStorage( Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10, WRITE_BUFFER_SIZE); final StreamStateHandle stateHandle; try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) { assertTrue(stream instanceof FsCheckpointStateOutputStream); new ObjectOutputStream(stream).writeObject(state); stateHandle = stream.closeAndGetHandle(); } // the state must have gone to disk FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; // check that the state is in the correct directory String parentDirName = fileStateHandle.getFilePath().getParent().getName(); assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName); // validate the contents try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) { assertEquals(state, in.readObject()); } }
Example #17
Source File: FsCheckpointStorage.java From flink with Apache License 2.0 | 5 votes |
@Override public CheckpointStateOutputStream createTaskOwnedStateStream() { return new FsCheckpointStateOutputStream( taskOwnedStateDirectory, fileSystem, writeBufferSize, fileSizeThreshold); }
Example #18
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This test checks that the stream does not check and clean the parent directory * when encountering a write error. */ @Test public void testStreamDoesNotTryToCleanUpParentOnError() throws Exception { final File directory = tempDir.newFolder(); // prevent creation of files in that directory assertTrue(directory.setWritable(false, true)); checkDirectoryNotWritable(directory); FileSystem fs = spy(FileSystem.getLocalFileSystem()); FsCheckpointStateOutputStream stream1 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); FsCheckpointStateOutputStream stream2 = new FsCheckpointStateOutputStream( Path.fromLocalFile(directory), fs, 1024, 1); stream1.write(new byte[61]); stream2.write(new byte[61]); try { stream1.closeAndGetHandle(); fail("this should fail with an exception"); } catch (IOException ignored) {} stream2.close(); // no delete call must have happened verify(fs, times(0)).delete(any(Path.class), anyBoolean()); // the directory must still exist as a proper directory assertTrue(directory.exists()); assertTrue(directory.isDirectory()); }
Example #19
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testEmptyState() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512); StreamStateHandle handle = stream.closeAndGetHandle(); assertTrue(handle == null); }
Example #20
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testEmptyState() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 1024, 512); StreamStateHandle handle = stream.closeAndGetHandle(); assertTrue(handle == null); }
Example #21
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Test public void testGetPos() throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17); for (int i = 0; i < 64; ++i) { Assert.assertEquals(i, stream.getPos()); stream.write(0x42); } stream.closeAndGetHandle(); // ---------------------------------------------------- stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 31, 17); byte[] data = "testme!".getBytes(ConfigConstants.DEFAULT_CHARSET); for (int i = 0; i < 7; ++i) { Assert.assertEquals(i * (1 + data.length), stream.getPos()); stream.write(0x42); stream.write(data); } stream.closeAndGetHandle(); }
Example #22
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Tests that the underlying stream file is deleted if the closeAndGetHandle method fails. */ @Test public void testCleanupWhenFailingCloseAndGetHandle() throws IOException { final FileSystem fs = mock(FileSystem.class); final FSDataOutputStream outputStream = mock(FSDataOutputStream.class); final ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class); when(fs.create(pathCaptor.capture(), any(FileSystem.WriteMode.class))).thenReturn(outputStream); doThrow(new IOException("Test IOException.")).when(outputStream).close(); CheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), fs, 4, 0); // this should create the underlying file stream stream.write(new byte[] {1, 2, 3, 4, 5}); verify(fs).create(any(Path.class), any(FileSystem.WriteMode.class)); try { stream.closeAndGetHandle(); fail("Expected IOException"); } catch (IOException ioE) { // expected exception } verify(fs).delete(eq(pathCaptor.getValue()), anyBoolean()); }
Example #23
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testDirectoriesForExclusiveAndSharedState() throws Exception { final FileSystem fs = LocalFileSystem.getSharedInstance(); final Path checkpointDir = randomTempPath(); final Path sharedStateDir = randomTempPath(); FsCheckpointStorageLocation storageLocation = new FsCheckpointStorageLocation( fs, checkpointDir, sharedStateDir, randomTempPath(), CheckpointStorageLocationReference.getDefault(), FILE_SIZE_THRESHOLD, WRITE_BUFFER_SIZE); assertNotEquals(storageLocation.getCheckpointDirectory(), storageLocation.getSharedStateDirectory()); assertEquals(0, fs.listStatus(storageLocation.getCheckpointDirectory()).length); assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length); // create exclusive state FsCheckpointStateOutputStream exclusiveStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); exclusiveStream.write(42); exclusiveStream.flushToFile(); StreamStateHandle exclusiveHandle = exclusiveStream.closeAndGetHandle(); assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length); assertEquals(0, fs.listStatus(storageLocation.getSharedStateDirectory()).length); // create shared state FsCheckpointStateOutputStream sharedStream = storageLocation.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED); sharedStream.write(42); sharedStream.flushToFile(); StreamStateHandle sharedHandle = sharedStream.closeAndGetHandle(); assertEquals(1, fs.listStatus(storageLocation.getCheckpointDirectory()).length); assertEquals(1, fs.listStatus(storageLocation.getSharedStateDirectory()).length); // drop state exclusiveHandle.discardState(); sharedHandle.discardState(); }
Example #24
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testMixedBelowAndAboveThreshold() throws Exception { final byte[] state1 = new byte[1274673]; final byte[] state2 = new byte[1]; final byte[] state3 = new byte[0]; final byte[] state4 = new byte[177]; final Random rnd = new Random(); rnd.nextBytes(state1); rnd.nextBytes(state2); rnd.nextBytes(state3); rnd.nextBytes(state4); final File directory = tempDir.newFolder(); final Path basePath = Path.fromLocalFile(directory); final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15, relativePaths); CheckpointStateOutputStream stream1 = factory.get(); CheckpointStateOutputStream stream2 = factory.get(); CheckpointStateOutputStream stream3 = factory.get(); stream1.write(state1); stream2.write(state2); stream3.write(state3); FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle(); ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle(); ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle(); // use with try-with-resources StreamStateHandle handle4; try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get(); stream5.write(state4); stream5.close(); try { stream5.closeAndGetHandle(); fail(); } catch (IOException e) { // uh-huh } validateBytesInStream(handle1.openInputStream(), state1); handle1.discardState(); assertFalse(isDirectoryEmpty(directory)); ensureLocalFileDeleted(handle1.getFilePath()); validateBytesInStream(handle2.openInputStream(), state2); handle2.discardState(); assertFalse(isDirectoryEmpty(directory)); // nothing was written to the stream, so it will return nothing assertNull(handle3); assertFalse(isDirectoryEmpty(directory)); validateBytesInStream(handle4.openInputStream(), state4); handle4.discardState(); assertTrue(isDirectoryEmpty(directory)); }
Example #25
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold, relativePaths); Random rnd = new Random(); byte[] original = new byte[numBytes]; byte[] bytes = new byte[original.length]; rnd.nextBytes(original); System.arraycopy(original, 0, bytes, 0, original.length); // the test writes a mixture of writing individual bytes and byte arrays int pos = 0; while (pos < bytes.length) { boolean single = rnd.nextBoolean(); if (single) { stream.write(bytes[pos++]); } else { int num = rnd.nextBoolean() ? (bytes.length - pos) : rnd.nextInt(bytes.length - pos); stream.write(bytes, pos, num); pos += num; } } StreamStateHandle handle = stream.closeAndGetHandle(); if (expectFile) { assertTrue(handle instanceof FileStateHandle); } else { assertTrue(handle instanceof ByteStreamStateHandle); } // make sure the writing process did not alter the original byte array assertArrayEquals(original, bytes); try (InputStream inStream = handle.openInputStream()) { byte[] validation = new byte[bytes.length]; DataInputStream dataInputStream = new DataInputStream(inStream); dataInputStream.readFully(validation); assertArrayEquals(bytes, validation); } handle.discardState(); }
Example #26
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test(expected = IllegalArgumentException.class) public void testWrongParameters() throws Exception { // this should fail new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000); }
Example #27
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
@Test(expected = IllegalArgumentException.class) public void testWrongParameters() throws Exception { // this should fail new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), 4000, 5000, relativePaths); }
Example #28
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception { FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream( Path.fromLocalFile(tempDir.newFolder()), FileSystem.getLocalFileSystem(), bufferSize, threshold); Random rnd = new Random(); byte[] original = new byte[numBytes]; byte[] bytes = new byte[original.length]; rnd.nextBytes(original); System.arraycopy(original, 0, bytes, 0, original.length); // the test writes a mixture of writing individual bytes and byte arrays int pos = 0; while (pos < bytes.length) { boolean single = rnd.nextBoolean(); if (single) { stream.write(bytes[pos++]); } else { int num = rnd.nextInt(Math.min(10, bytes.length - pos)); stream.write(bytes, pos, num); pos += num; } } StreamStateHandle handle = stream.closeAndGetHandle(); if (expectFile) { assertTrue(handle instanceof FileStateHandle); } else { assertTrue(handle instanceof ByteStreamStateHandle); } // make sure the writing process did not alter the original byte array assertArrayEquals(original, bytes); try (InputStream inStream = handle.openInputStream()) { byte[] validation = new byte[bytes.length]; DataInputStream dataInputStream = new DataInputStream(inStream); dataInputStream.readFully(validation); assertArrayEquals(bytes, validation); } handle.discardState(); }
Example #29
Source File: FsCheckpointStateOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Test public void testMixedBelowAndAboveThreshold() throws Exception { final byte[] state1 = new byte[1274673]; final byte[] state2 = new byte[1]; final byte[] state3 = new byte[0]; final byte[] state4 = new byte[177]; final Random rnd = new Random(); rnd.nextBytes(state1); rnd.nextBytes(state2); rnd.nextBytes(state3); rnd.nextBytes(state4); final File directory = tempDir.newFolder(); final Path basePath = Path.fromLocalFile(directory); final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15); CheckpointStateOutputStream stream1 = factory.get(); CheckpointStateOutputStream stream2 = factory.get(); CheckpointStateOutputStream stream3 = factory.get(); stream1.write(state1); stream2.write(state2); stream3.write(state3); FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle(); ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle(); ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle(); // use with try-with-resources StreamStateHandle handle4; try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get(); stream5.write(state4); stream5.close(); try { stream5.closeAndGetHandle(); fail(); } catch (IOException e) { // uh-huh } validateBytesInStream(handle1.openInputStream(), state1); handle1.discardState(); assertFalse(isDirectoryEmpty(directory)); ensureLocalFileDeleted(handle1.getFilePath()); validateBytesInStream(handle2.openInputStream(), state2); handle2.discardState(); assertFalse(isDirectoryEmpty(directory)); // nothing was written to the stream, so it will return nothing assertNull(handle3); assertFalse(isDirectoryEmpty(directory)); validateBytesInStream(handle4.openInputStream(), state4); handle4.discardState(); assertTrue(isDirectoryEmpty(directory)); }
Example #30
Source File: FsCheckpointStateOutputStreamTest.java From flink with Apache License 2.0 | 4 votes |
@Test public void testMixedBelowAndAboveThreshold() throws Exception { final byte[] state1 = new byte[1274673]; final byte[] state2 = new byte[1]; final byte[] state3 = new byte[0]; final byte[] state4 = new byte[177]; final Random rnd = new Random(); rnd.nextBytes(state1); rnd.nextBytes(state2); rnd.nextBytes(state3); rnd.nextBytes(state4); final File directory = tempDir.newFolder(); final Path basePath = Path.fromLocalFile(directory); final Supplier<CheckpointStateOutputStream> factory = () -> new FsCheckpointStateOutputStream(basePath, FileSystem.getLocalFileSystem(), 1024, 15); CheckpointStateOutputStream stream1 = factory.get(); CheckpointStateOutputStream stream2 = factory.get(); CheckpointStateOutputStream stream3 = factory.get(); stream1.write(state1); stream2.write(state2); stream3.write(state3); FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle(); ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle(); ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle(); // use with try-with-resources StreamStateHandle handle4; try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = factory.get()) { stream4.write(state4); handle4 = stream4.closeAndGetHandle(); } // close before accessing handle CheckpointStreamFactory.CheckpointStateOutputStream stream5 = factory.get(); stream5.write(state4); stream5.close(); try { stream5.closeAndGetHandle(); fail(); } catch (IOException e) { // uh-huh } validateBytesInStream(handle1.openInputStream(), state1); handle1.discardState(); assertFalse(isDirectoryEmpty(directory)); ensureLocalFileDeleted(handle1.getFilePath()); validateBytesInStream(handle2.openInputStream(), state2); handle2.discardState(); assertFalse(isDirectoryEmpty(directory)); // nothing was written to the stream, so it will return nothing assertNull(handle3); assertFalse(isDirectoryEmpty(directory)); validateBytesInStream(handle4.openInputStream(), state4); handle4.discardState(); assertTrue(isDirectoryEmpty(directory)); }