org.apache.flink.runtime.state.StreamStateHandle Java Examples
The following examples show how to use
org.apache.flink.runtime.state.StreamStateHandle.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MemoryCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testTaskOwnedStateStream() throws Exception { final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter"); final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage( new JobID(), null, null, DEFAULT_MAX_STATE_SIZE); StreamStateHandle stateHandle; try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) { assertTrue(stream instanceof MemoryCheckpointOutputStream); new ObjectOutputStream(stream).writeObject(state); stateHandle = stream.closeAndGetHandle(); } try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) { assertEquals(state, in.readObject()); } }
Example #2
Source File: MemoryCheckpointStorageTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testNonPersistentCheckpointLocation() throws Exception { MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage( new JobID(), null, null, DEFAULT_MAX_STATE_SIZE); CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9); CheckpointMetadataOutputStream stream = location.createMetadataOutputStream(); stream.write(99); CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint(); StreamStateHandle handle = completed.getMetadataHandle(); assertTrue(handle instanceof ByteStreamStateHandle); // the reference is not valid in that case try { storage.resolveCheckpoint(completed.getExternalPointer()); fail("should fail with an exception"); } catch (Exception e) { // expected } }
Example #3
Source File: BlockingCheckpointOutputStream.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Nullable @Override public StreamStateHandle closeAndGetHandle() throws IOException { if (!closed.compareAndSet(false, true)) { throw new IOException("Stream was already closed!"); } if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) { StreamStateHandle streamStateHandle = ((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle(); unblockAll(); return streamStateHandle; } else { unblockAll(); throw new IOException("Delegate is not a CheckpointStateOutputStream!"); } }
Example #4
Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@VisibleForTesting public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type); } }
Example #5
Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException { final int type = dis.read(); if (NULL_HANDLE == type) { return null; } else if (FILE_STREAM_STATE_HANDLE == type) { long size = dis.readLong(); String pathString = dis.readUTF(); return new FileStateHandle(new Path(pathString), size); } else if (BYTE_STREAM_STATE_HANDLE == type) { String handleName = dis.readUTF(); int numBytes = dis.readInt(); byte[] data = new byte[numBytes]; dis.readFully(data); return new ByteStreamStateHandle(handleName, data); } else { throw new IOException("Unknown implementation of StreamStateHandle, code: " + type); } }
Example #6
Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@VisibleForTesting public static void serializeStreamStateHandle( StreamStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof FileStateHandle) { dos.writeByte(FILE_STREAM_STATE_HANDLE); FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; dos.writeLong(stateHandle.getStateSize()); dos.writeUTF(fileStateHandle.getFilePath().toString()); } else if (stateHandle instanceof ByteStreamStateHandle) { dos.writeByte(BYTE_STREAM_STATE_HANDLE); ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle; dos.writeUTF(byteStreamStateHandle.getHandleName()); byte[] internalData = byteStreamStateHandle.getData(); dos.writeInt(internalData.length); dos.write(byteStreamStateHandle.getData()); } else { throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass()); } dos.flush(); }
Example #7
Source File: RocksDBStateDownloader.java From flink with Apache License 2.0 | 6 votes |
/** * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their * {@link StateHandleID}. */ private void downloadDataForAllStateHandles( Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) throws Exception { try { List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry); List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size()); for (Runnable runnable : runnables) { futures.add(CompletableFuture.runAsync(runnable, executorService)); } FutureUtils.waitForAll(futures).get(); } catch (ExecutionException e) { Throwable throwable = ExceptionUtils.stripExecutionException(e); throwable = ExceptionUtils.stripException(throwable, RuntimeException.class); if (throwable instanceof IOException) { throw (IOException) throwable; } else { throw new FlinkRuntimeException("Failed to download data for state handles.", e); } } }
Example #8
Source File: MemoryCheckpointOutputStreamTest.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Test public void testStateStream() throws Exception { HashMap<String, Integer> state = new HashMap<>(); state.put("hey there", 2); state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77); CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE); ObjectOutputStream oos = new ObjectOutputStream(outStream); oos.writeObject(state); oos.flush(); StreamStateHandle handle = outStream.closeAndGetHandle(); assertNotNull(handle); try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) { assertEquals(state, ois.readObject()); assertTrue(ois.available() <= 0); } }
Example #9
Source File: ChannelStateCheckpointWriter.java From flink with Apache License 2.0 | 6 votes |
private <I, H extends AbstractChannelStateHandle<I>> H createHandle( HandleFactory<I, H> handleFactory, StreamStateHandle underlying, I channelInfo, StateContentMetaInfo contentMetaInfo) throws IOException { Optional<byte[]> bytes = underlying.asBytesIfInMemory(); // todo: consider restructuring channel state and removing this method: https://issues.apache.org/jira/browse/FLINK-17972 if (bytes.isPresent()) { StreamStateHandle extracted = new ByteStreamStateHandle( randomUUID().toString(), serializer.extractAndMerge(bytes.get(), contentMetaInfo.getOffsets())); return handleFactory.create( channelInfo, extracted, singletonList(serializer.getHeaderLength()), extracted.getStateSize()); } else { return handleFactory.create(channelInfo, underlying, contentMetaInfo.getOffsets(), contentMetaInfo.getSize()); } }
Example #10
Source File: RocksDBStateDownloader.java From flink with Apache License 2.0 | 6 votes |
/** * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their * {@link StateHandleID}. */ private void downloadDataForAllStateHandles( Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) throws Exception { try { List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry); List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size()); for (Runnable runnable : runnables) { futures.add(CompletableFuture.runAsync(runnable, executorService)); } FutureUtils.waitForAll(futures).get(); } catch (ExecutionException e) { Throwable throwable = ExceptionUtils.stripExecutionException(e); throwable = ExceptionUtils.stripException(throwable, RuntimeException.class); if (throwable instanceof IOException) { throw (IOException) throwable; } else { throw new FlinkRuntimeException("Failed to download data for state handles.", e); } } }
Example #11
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 6 votes |
/** * Reads Flink's state meta data file from the state handle. */ private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(inputStream); DataInputView in = new DataInputViewStreamWrapper(inputStream); return readMetaData(in); } finally { if (cancelStreamRegistry.unregisterCloseable(inputStream)) { inputStream.close(); } } }
Example #12
Source File: RocksDBStateDownloader.java From flink with Apache License 2.0 | 6 votes |
private List<Runnable> createDownloadRunnables( Map<StateHandleID, StreamStateHandle> stateHandleMap, Path restoreInstancePath, CloseableRegistry closeableRegistry) { List<Runnable> runnables = new ArrayList<>(stateHandleMap.size()); for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) { StateHandleID stateHandleID = entry.getKey(); StreamStateHandle remoteFileHandle = entry.getValue(); Path path = restoreInstancePath.resolve(stateHandleID.toString()); runnables.add(ThrowingRunnable.unchecked( () -> downloadDataForStateHandle(path, remoteFileHandle, closeableRegistry))); } return runnables; }
Example #13
Source File: BlockingCheckpointOutputStream.java From flink with Apache License 2.0 | 6 votes |
@Nullable @Override public StreamStateHandle closeAndGetHandle() throws IOException { if (!closed.compareAndSet(false, true)) { throw new IOException("Stream was already closed!"); } if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) { StreamStateHandle streamStateHandle = ((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle(); unblockAll(); return streamStateHandle; } else { unblockAll(); throw new IOException("Delegate is not a CheckpointStateOutputStream!"); } }
Example #14
Source File: SavepointV2Serializer.java From flink with Apache License 2.0 | 6 votes |
@VisibleForTesting public static void serializeStreamStateHandle( StreamStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof FileStateHandle) { dos.writeByte(FILE_STREAM_STATE_HANDLE); FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; dos.writeLong(stateHandle.getStateSize()); dos.writeUTF(fileStateHandle.getFilePath().toString()); } else if (stateHandle instanceof ByteStreamStateHandle) { dos.writeByte(BYTE_STREAM_STATE_HANDLE); ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle; dos.writeUTF(byteStreamStateHandle.getHandleName()); byte[] internalData = byteStreamStateHandle.getData(); dos.writeInt(internalData.length); dos.write(byteStreamStateHandle.getData()); } else { throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass()); } dos.flush(); }
Example #15
Source File: RocksIncrementalSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void createUploadFilePaths( FileStatus[] fileStatuses, Map<StateHandleID, StreamStateHandle> sstFiles, Map<StateHandleID, Path> sstFilePaths, Map<StateHandleID, Path> miscFilePaths) { for (FileStatus fileStatus : fileStatuses) { final Path filePath = fileStatus.getPath(); final String fileName = filePath.getName(); final StateHandleID stateHandleID = new StateHandleID(fileName); if (fileName.endsWith(SST_FILE_SUFFIX)) { final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID); if (existsAlready) { // we introduce a placeholder state handle, that is replaced with the // original from the shared state registry (created from a previous checkpoint) sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle()); } else { sstFilePaths.put(stateHandleID, filePath); } } else { miscFilePaths.put(stateHandleID, filePath); } } }
Example #16
Source File: RocksIncrementalSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void uploadSstFiles( @Nonnull Map<StateHandleID, StreamStateHandle> sstFiles, @Nonnull Map<StateHandleID, StreamStateHandle> miscFiles) throws Exception { // write state data Preconditions.checkState(localBackupDirectory.exists()); Map<StateHandleID, Path> sstFilePaths = new HashMap<>(); Map<StateHandleID, Path> miscFilePaths = new HashMap<>(); FileStatus[] fileStatuses = localBackupDirectory.listStatus(); if (fileStatuses != null) { createUploadFilePaths(fileStatuses, sstFiles, sstFilePaths, miscFilePaths); sstFiles.putAll(stateUploader.uploadFilesToCheckpointFs( sstFilePaths, checkpointStreamFactory, snapshotCloseableRegistry)); miscFiles.putAll(stateUploader.uploadFilesToCheckpointFs( miscFilePaths, checkpointStreamFactory, snapshotCloseableRegistry)); } }
Example #17
Source File: GenericWriteAheadSink.java From flink with Apache License 2.0 | 6 votes |
/** * Called when a checkpoint barrier arrives. It closes any open streams to the backend * and marks them as pending for committing to the external, third-party storage system. * * @param checkpointId the id of the latest received checkpoint. * @throws IOException in case something went wrong when handling the stream to the backend. */ private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception { //only add handle if a new OperatorState was created since the last snapshot if (out != null) { int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask(); StreamStateHandle handle = out.closeAndGetHandle(); PendingCheckpoint pendingCheckpoint = new PendingCheckpoint( checkpointId, subtaskIdx, timestamp, handle); if (pendingCheckpoints.contains(pendingCheckpoint)) { //we already have a checkpoint stored for that ID that may have been partially written, //so we discard this "alternate version" and use the stored checkpoint handle.discardState(); } else { pendingCheckpoints.add(pendingCheckpoint); } out = null; } }
Example #18
Source File: RocksIncrementalSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void createUploadFilePaths( FileStatus[] fileStatuses, Map<StateHandleID, StreamStateHandle> sstFiles, Map<StateHandleID, Path> sstFilePaths, Map<StateHandleID, Path> miscFilePaths) { for (FileStatus fileStatus : fileStatuses) { final Path filePath = fileStatus.getPath(); final String fileName = filePath.getName(); final StateHandleID stateHandleID = new StateHandleID(fileName); if (fileName.endsWith(SST_FILE_SUFFIX)) { final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID); if (existsAlready) { // we introduce a placeholder state handle, that is replaced with the // original from the shared state registry (created from a previous checkpoint) sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle()); } else { sstFilePaths.put(stateHandleID, filePath); } } else { miscFilePaths.put(stateHandleID, filePath); } } }
Example #19
Source File: RocksIncrementalSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void createUploadFilePaths( Path[] files, Map<StateHandleID, StreamStateHandle> sstFiles, Map<StateHandleID, Path> sstFilePaths, Map<StateHandleID, Path> miscFilePaths) { for (Path filePath : files) { final String fileName = filePath.getFileName().toString(); final StateHandleID stateHandleID = new StateHandleID(fileName); if (fileName.endsWith(SST_FILE_SUFFIX)) { final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID); if (existsAlready) { // we introduce a placeholder state handle, that is replaced with the // original from the shared state registry (created from a previous checkpoint) sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle()); } else { sstFilePaths.put(stateHandleID, filePath); } } else { miscFilePaths.put(stateHandleID, filePath); } } }
Example #20
Source File: MemoryCheckpointOutputStreamTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testStateStream() throws Exception { HashMap<String, Integer> state = new HashMap<>(); state.put("hey there", 2); state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77); CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE); ObjectOutputStream oos = new ObjectOutputStream(outStream); oos.writeObject(state); oos.flush(); StreamStateHandle handle = outStream.closeAndGetHandle(); assertNotNull(handle); try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) { assertEquals(state, ois.readObject()); assertTrue(ois.available() <= 0); } }
Example #21
Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
/** * Reads Flink's state meta data file from the state handle. */ private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception { FSDataInputStream inputStream = null; try { inputStream = metaStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(inputStream); DataInputView in = new DataInputViewStreamWrapper(inputStream); return readMetaData(in); } finally { if (cancelStreamRegistry.unregisterCloseable(inputStream)) { inputStream.close(); } } }
Example #22
Source File: MemoryCheckpointStorageTest.java From flink with Apache License 2.0 | 6 votes |
@Test public void testNonPersistentCheckpointLocation() throws Exception { MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage( new JobID(), null, null, DEFAULT_MAX_STATE_SIZE); CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9); CheckpointMetadataOutputStream stream = location.createMetadataOutputStream(); stream.write(99); CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint(); StreamStateHandle handle = completed.getMetadataHandle(); assertTrue(handle instanceof ByteStreamStateHandle); // the reference is not valid in that case try { storage.resolveCheckpoint(completed.getExternalPointer()); fail("should fail with an exception"); } catch (Exception e) { // expected } }
Example #23
Source File: RocksDBStateUploaderTest.java From flink with Apache License 2.0 | 5 votes |
private CheckpointStreamFactory.CheckpointStateOutputStream createFailingCheckpointStateOutputStream( IOException failureException) { return new CheckpointStreamFactory.CheckpointStateOutputStream() { @Nullable @Override public StreamStateHandle closeAndGetHandle() { return new ByteStreamStateHandle("testHandle", "testHandle".getBytes()); } @Override public void close() { } @Override public long getPos() { return 0; } @Override public void flush() { } @Override public void sync() { } @Override public void write(int b) throws IOException { throw failureException; } }; }
Example #24
Source File: RocksDBStateUploader.java From flink with Apache License 2.0 | 5 votes |
/** * Upload all the files to checkpoint fileSystem using specified number of threads. * * @param files The files will be uploaded to checkpoint filesystem. * @param checkpointStreamFactory The checkpoint streamFactory used to create outputstream. * * @throws Exception Thrown if can not upload all the files. */ public Map<StateHandleID, StreamStateHandle> uploadFilesToCheckpointFs( @Nonnull Map<StateHandleID, Path> files, CheckpointStreamFactory checkpointStreamFactory, CloseableRegistry closeableRegistry) throws Exception { Map<StateHandleID, StreamStateHandle> handles = new HashMap<>(); Map<StateHandleID, CompletableFuture<StreamStateHandle>> futures = createUploadFutures(files, checkpointStreamFactory, closeableRegistry); try { FutureUtils.waitForAll(futures.values()).get(); for (Map.Entry<StateHandleID, CompletableFuture<StreamStateHandle>> entry : futures.entrySet()) { handles.put(entry.getKey(), entry.getValue().get()); } } catch (ExecutionException e) { Throwable throwable = ExceptionUtils.stripExecutionException(e); throwable = ExceptionUtils.stripException(throwable, RuntimeException.class); if (throwable instanceof IOException) { throw (IOException) throwable; } else { throw new FlinkRuntimeException("Failed to download data for state handles.", e); } } return handles; }
Example #25
Source File: SavepointV2Serializer.java From flink with Apache License 2.0 | 5 votes |
private static void serializeStreamStateHandleMap( Map<StateHandleID, StreamStateHandle> map, DataOutputStream dos) throws IOException { dos.writeInt(map.size()); for (Map.Entry<StateHandleID, StreamStateHandle> entry : map.entrySet()) { dos.writeUTF(entry.getKey().toString()); serializeStreamStateHandle(entry.getValue(), dos); } }
Example #26
Source File: FsCheckpointStreamFactoryTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testSharedStateHasAbsolutePathHandles() throws IOException { final FsCheckpointStreamFactory factory = createFactory(FileSystem.getLocalFileSystem(), 0); final FsCheckpointStreamFactory.FsCheckpointStateOutputStream stream = factory.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED); stream.write(0); final StreamStateHandle handle = stream.closeAndGetHandle(); assertThat(handle, instanceOf(FileStateHandle.class)); assertThat(handle, not(instanceOf(RelativeFileStateHandle.class))); assertPathsEqual(sharedStateDir, ((FileStateHandle) handle).getFilePath().getParent()); }
Example #27
Source File: RocksDBStateUploaderTest.java From flink with Apache License 2.0 | 5 votes |
private CheckpointStreamFactory.CheckpointStateOutputStream createFailingCheckpointStateOutputStream( IOException failureException) { return new CheckpointStreamFactory.CheckpointStateOutputStream() { @Nullable @Override public StreamStateHandle closeAndGetHandle() { return new ByteStreamStateHandle("testHandle", "testHandle".getBytes()); } @Override public void close() { } @Override public long getPos() { return 0; } @Override public void flush() { } @Override public void sync() { } @Override public void write(int b) throws IOException { throw failureException; } }; }
Example #28
Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0 | 5 votes |
private static Map<StateHandleID, StreamStateHandle> deserializeStreamStateHandleMap( DataInputStream dis, @Nullable DeserializationContext context) throws IOException { final int size = dis.readInt(); Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size); for (int i = 0; i < size; ++i) { StateHandleID stateHandleID = new StateHandleID(dis.readUTF()); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context); result.put(stateHandleID, stateHandle); } return result; }
Example #29
Source File: MemCheckpointStreamFactory.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@Nullable @Override public StreamStateHandle closeAndGetHandle() throws IOException { if (isEmpty) { return null; } return new ByteStreamStateHandle(String.valueOf(UUID.randomUUID()), closeAndGetBytes()); }
Example #30
Source File: FsCheckpointStorageTest.java From flink with Apache License 2.0 | 5 votes |
@Test public void testTaskOwnedStateStream() throws Exception { final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter"); // we chose a small size threshold here to force the state to disk final FsCheckpointStorage storage = new FsCheckpointStorage( Path.fromLocalFile(tmp.newFolder()), null, new JobID(), 10, WRITE_BUFFER_SIZE); final StreamStateHandle stateHandle; try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) { assertTrue(stream instanceof FsCheckpointStateOutputStream); new ObjectOutputStream(stream).writeObject(state); stateHandle = stream.closeAndGetHandle(); } // the state must have gone to disk FileStateHandle fileStateHandle = (FileStateHandle) stateHandle; // check that the state is in the correct directory String parentDirName = fileStateHandle.getFilePath().getParent().getName(); assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName); // validate the contents try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) { assertEquals(state, in.readObject()); } }