org.apache.flink.runtime.state.StreamStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.StreamStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MemoryCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof MemoryCheckpointOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example #2
Source File: MemoryCheckpointStorageTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #3
Source File: BlockingCheckpointOutputStream.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {

	if (!closed.compareAndSet(false, true)) {
		throw new IOException("Stream was already closed!");
	}

	if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) {
		StreamStateHandle streamStateHandle =
			((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle();
		unblockAll();
		return streamStateHandle;
	} else {
		unblockAll();
		throw new IOException("Delegate is not a CheckpointStateOutputStream!");
	}
}
 
Example #4
Source File: SavepointV1Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}
 
Example #5
Source File: SavepointV2Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example #6
Source File: SavepointV2Serializer.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example #7
Source File: RocksDBStateDownloader.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their
 * {@link StateHandleID}.
 */
private void downloadDataForAllStateHandles(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) throws Exception {

	try {
		List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry);
		List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size());
		for (Runnable runnable : runnables) {
			futures.add(CompletableFuture.runAsync(runnable, executorService));
		}
		FutureUtils.waitForAll(futures).get();
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}
}
 
Example #8
Source File: MemoryCheckpointOutputStreamTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example #9
Source File: ChannelStateCheckpointWriter.java    From flink with Apache License 2.0 6 votes vote down vote up
private <I, H extends AbstractChannelStateHandle<I>> H createHandle(
		HandleFactory<I, H> handleFactory,
		StreamStateHandle underlying,
		I channelInfo,
		StateContentMetaInfo contentMetaInfo) throws IOException {
	Optional<byte[]> bytes = underlying.asBytesIfInMemory(); // todo: consider restructuring channel state and removing this method: https://issues.apache.org/jira/browse/FLINK-17972
	if (bytes.isPresent()) {
		StreamStateHandle extracted = new ByteStreamStateHandle(
			randomUUID().toString(),
			serializer.extractAndMerge(bytes.get(), contentMetaInfo.getOffsets()));
		return handleFactory.create(
			channelInfo,
			extracted,
			singletonList(serializer.getHeaderLength()),
			extracted.getStateSize());
	} else {
		return handleFactory.create(channelInfo, underlying, contentMetaInfo.getOffsets(), contentMetaInfo.getSize());
	}
}
 
Example #10
Source File: RocksDBStateDownloader.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their
 * {@link StateHandleID}.
 */
private void downloadDataForAllStateHandles(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) throws Exception {

	try {
		List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry);
		List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size());
		for (Runnable runnable : runnables) {
			futures.add(CompletableFuture.runAsync(runnable, executorService));
		}
		FutureUtils.waitForAll(futures).get();
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}
}
 
Example #11
Source File: RocksDBIncrementalRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example #12
Source File: RocksDBStateDownloader.java    From flink with Apache License 2.0 6 votes vote down vote up
private List<Runnable> createDownloadRunnables(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) {
	List<Runnable> runnables = new ArrayList<>(stateHandleMap.size());
	for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
		StateHandleID stateHandleID = entry.getKey();
		StreamStateHandle remoteFileHandle = entry.getValue();

		Path path = restoreInstancePath.resolve(stateHandleID.toString());

		runnables.add(ThrowingRunnable.unchecked(
			() -> downloadDataForStateHandle(path, remoteFileHandle, closeableRegistry)));
	}
	return runnables;
}
 
Example #13
Source File: BlockingCheckpointOutputStream.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {

	if (!closed.compareAndSet(false, true)) {
		throw new IOException("Stream was already closed!");
	}

	if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) {
		StreamStateHandle streamStateHandle =
			((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle();
		unblockAll();
		return streamStateHandle;
	} else {
		unblockAll();
		throw new IOException("Delegate is not a CheckpointStateOutputStream!");
	}
}
 
Example #14
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example #15
Source File: RocksIncrementalSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example #16
Source File: RocksIncrementalSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private void uploadSstFiles(
	@Nonnull Map<StateHandleID, StreamStateHandle> sstFiles,
	@Nonnull Map<StateHandleID, StreamStateHandle> miscFiles) throws Exception {

	// write state data
	Preconditions.checkState(localBackupDirectory.exists());

	Map<StateHandleID, Path> sstFilePaths = new HashMap<>();
	Map<StateHandleID, Path> miscFilePaths = new HashMap<>();

	FileStatus[] fileStatuses = localBackupDirectory.listStatus();
	if (fileStatuses != null) {
		createUploadFilePaths(fileStatuses, sstFiles, sstFilePaths, miscFilePaths);

		sstFiles.putAll(stateUploader.uploadFilesToCheckpointFs(
			sstFilePaths,
			checkpointStreamFactory,
			snapshotCloseableRegistry));
		miscFiles.putAll(stateUploader.uploadFilesToCheckpointFs(
			miscFilePaths,
			checkpointStreamFactory,
			snapshotCloseableRegistry));
	}
}
 
Example #17
Source File: GenericWriteAheadSink.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Called when a checkpoint barrier arrives. It closes any open streams to the backend
 * and marks them as pending for committing to the external, third-party storage system.
 *
 * @param checkpointId the id of the latest received checkpoint.
 * @throws IOException in case something went wrong when handling the stream to the backend.
 */
private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception {

	//only add handle if a new OperatorState was created since the last snapshot
	if (out != null) {
		int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
		StreamStateHandle handle = out.closeAndGetHandle();

		PendingCheckpoint pendingCheckpoint = new PendingCheckpoint(
			checkpointId, subtaskIdx, timestamp, handle);

		if (pendingCheckpoints.contains(pendingCheckpoint)) {
			//we already have a checkpoint stored for that ID that may have been partially written,
			//so we discard this "alternate version" and use the stored checkpoint
			handle.discardState();
		} else {
			pendingCheckpoints.add(pendingCheckpoint);
		}
		out = null;
	}
}
 
Example #18
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example #19
Source File: RocksIncrementalSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
private void createUploadFilePaths(
	Path[] files,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (Path filePath : files) {
		final String fileName = filePath.getFileName().toString();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example #20
Source File: MemoryCheckpointOutputStreamTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example #21
Source File: RocksDBIncrementalRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example #22
Source File: MemoryCheckpointStorageTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example #23
Source File: RocksDBStateUploaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private CheckpointStreamFactory.CheckpointStateOutputStream createFailingCheckpointStateOutputStream(
	IOException failureException) {
	return new CheckpointStreamFactory.CheckpointStateOutputStream() {
		@Nullable
		@Override
		public StreamStateHandle closeAndGetHandle() {
			return new ByteStreamStateHandle("testHandle", "testHandle".getBytes());
		}

		@Override
		public void close() {
		}

		@Override
		public long getPos() {
			return 0;
		}

		@Override
		public void flush() {
		}

		@Override
		public void sync() {
		}

		@Override
		public void write(int b) throws IOException {
			throw failureException;
		}
	};
}
 
Example #24
Source File: RocksDBStateUploader.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Upload all the files to checkpoint fileSystem using specified number of threads.
 *
 * @param files The files will be uploaded to checkpoint filesystem.
 * @param checkpointStreamFactory The checkpoint streamFactory used to create outputstream.
 *
 * @throws Exception Thrown if can not upload all the files.
 */
public Map<StateHandleID, StreamStateHandle> uploadFilesToCheckpointFs(
	@Nonnull Map<StateHandleID, Path> files,
	CheckpointStreamFactory checkpointStreamFactory,
	CloseableRegistry closeableRegistry) throws Exception {

	Map<StateHandleID, StreamStateHandle> handles = new HashMap<>();

	Map<StateHandleID, CompletableFuture<StreamStateHandle>> futures =
		createUploadFutures(files, checkpointStreamFactory, closeableRegistry);

	try {
		FutureUtils.waitForAll(futures.values()).get();

		for (Map.Entry<StateHandleID, CompletableFuture<StreamStateHandle>> entry : futures.entrySet()) {
			handles.put(entry.getKey(), entry.getValue().get());
		}
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}

	return handles;
}
 
Example #25
Source File: SavepointV2Serializer.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void serializeStreamStateHandleMap(
	Map<StateHandleID, StreamStateHandle> map,
	DataOutputStream dos) throws IOException {
	dos.writeInt(map.size());
	for (Map.Entry<StateHandleID, StreamStateHandle> entry : map.entrySet()) {
		dos.writeUTF(entry.getKey().toString());
		serializeStreamStateHandle(entry.getValue(), dos);
	}
}
 
Example #26
Source File: FsCheckpointStreamFactoryTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSharedStateHasAbsolutePathHandles() throws IOException {
	final FsCheckpointStreamFactory factory = createFactory(FileSystem.getLocalFileSystem(), 0);

	final FsCheckpointStreamFactory.FsCheckpointStateOutputStream stream =
		factory.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED);
	stream.write(0);
	final StreamStateHandle handle = stream.closeAndGetHandle();

	assertThat(handle, instanceOf(FileStateHandle.class));
	assertThat(handle, not(instanceOf(RelativeFileStateHandle.class)));
	assertPathsEqual(sharedStateDir, ((FileStateHandle) handle).getFilePath().getParent());
}
 
Example #27
Source File: RocksDBStateUploaderTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private CheckpointStreamFactory.CheckpointStateOutputStream createFailingCheckpointStateOutputStream(
	IOException failureException) {
	return new CheckpointStreamFactory.CheckpointStateOutputStream() {
		@Nullable
		@Override
		public StreamStateHandle closeAndGetHandle() {
			return new ByteStreamStateHandle("testHandle", "testHandle".getBytes());
		}

		@Override
		public void close() {
		}

		@Override
		public long getPos() {
			return 0;
		}

		@Override
		public void flush() {
		}

		@Override
		public void sync() {
		}

		@Override
		public void write(int b) throws IOException {
			throw failureException;
		}
	};
}
 
Example #28
Source File: MetadataV2V3SerializerBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Map<StateHandleID, StreamStateHandle> deserializeStreamStateHandleMap(
	DataInputStream dis,
	@Nullable DeserializationContext context) throws IOException {

	final int size = dis.readInt();
	Map<StateHandleID, StreamStateHandle> result = new HashMap<>(size);

	for (int i = 0; i < size; ++i) {
		StateHandleID stateHandleID = new StateHandleID(dis.readUTF());
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis, context);
		result.put(stateHandleID, stateHandle);
	}

	return result;
}
 
Example #29
Source File: MemCheckpointStreamFactory.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {
	if (isEmpty) {
		return null;
	}
	return new ByteStreamStateHandle(String.valueOf(UUID.randomUUID()), closeAndGetBytes());
}
 
Example #30
Source File: FsCheckpointStorageTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}