Java Code Examples for org.apache.flink.runtime.state.StreamStateHandle

The following examples show how to use org.apache.flink.runtime.state.StreamStateHandle. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example 2
Source Project: flink   Source File: MemoryCheckpointOutputStreamTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example 3
Source Project: flink   Source File: RocksIncrementalSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void createUploadFilePaths(
	Path[] files,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (Path filePath : files) {
		final String fileName = filePath.getFileName().toString();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example 4
private void uploadSstFiles(
	@Nonnull Map<StateHandleID, StreamStateHandle> sstFiles,
	@Nonnull Map<StateHandleID, StreamStateHandle> miscFiles) throws Exception {

	// write state data
	Preconditions.checkState(localBackupDirectory.exists());

	Map<StateHandleID, Path> sstFilePaths = new HashMap<>();
	Map<StateHandleID, Path> miscFilePaths = new HashMap<>();

	FileStatus[] fileStatuses = localBackupDirectory.listStatus();
	if (fileStatuses != null) {
		createUploadFilePaths(fileStatuses, sstFiles, sstFilePaths, miscFilePaths);

		sstFiles.putAll(stateUploader.uploadFilesToCheckpointFs(
			sstFilePaths,
			checkpointStreamFactory,
			snapshotCloseableRegistry));
		miscFiles.putAll(stateUploader.uploadFilesToCheckpointFs(
			miscFilePaths,
			checkpointStreamFactory,
			snapshotCloseableRegistry));
	}
}
 
Example 5
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example 6
Source Project: flink   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 7
Source Project: flink   Source File: BlockingCheckpointOutputStream.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {

	if (!closed.compareAndSet(false, true)) {
		throw new IOException("Stream was already closed!");
	}

	if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) {
		StreamStateHandle streamStateHandle =
			((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle();
		unblockAll();
		return streamStateHandle;
	} else {
		unblockAll();
		throw new IOException("Delegate is not a CheckpointStateOutputStream!");
	}
}
 
Example 8
Source Project: flink   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	final MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof MemoryCheckpointOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 9
Source Project: Flink-CEPplus   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static void serializeStreamStateHandle(
		StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);

	} else if (stateHandle instanceof FileStateHandle) {
		dos.writeByte(FILE_STREAM_STATE_HANDLE);
		FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
		dos.writeLong(stateHandle.getStateSize());
		dos.writeUTF(fileStateHandle.getFilePath().toString());

	} else if (stateHandle instanceof ByteStreamStateHandle) {
		dos.writeByte(BYTE_STREAM_STATE_HANDLE);
		ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
		dos.writeUTF(byteStreamStateHandle.getHandleName());
		byte[] internalData = byteStreamStateHandle.getData();
		dos.writeInt(internalData.length);
		dos.write(byteStreamStateHandle.getData());
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
	}

	dos.flush();
}
 
Example 10
Source Project: Flink-CEPplus   Source File: SavepointV2Serializer.java    License: Apache License 2.0 6 votes vote down vote up
public static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.read();
	if (NULL_HANDLE == type) {
		return null;
	} else if (FILE_STREAM_STATE_HANDLE == type) {
		long size = dis.readLong();
		String pathString = dis.readUTF();
		return new FileStateHandle(new Path(pathString), size);
	} else if (BYTE_STREAM_STATE_HANDLE == type) {
		String handleName = dis.readUTF();
		int numBytes = dis.readInt();
		byte[] data = new byte[numBytes];
		dis.readFully(data);
		return new ByteStreamStateHandle(handleName, data);
	} else {
		throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
	}
}
 
Example 11
Source Project: Flink-CEPplus   Source File: SavepointV1Serializer.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}
 
Example 12
Source Project: flink   Source File: RocksDBStateDownloader.java    License: Apache License 2.0 6 votes vote down vote up
private List<Runnable> createDownloadRunnables(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) {
	List<Runnable> runnables = new ArrayList<>(stateHandleMap.size());
	for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
		StateHandleID stateHandleID = entry.getKey();
		StreamStateHandle remoteFileHandle = entry.getValue();

		Path path = restoreInstancePath.resolve(stateHandleID.toString());

		runnables.add(ThrowingRunnable.unchecked(
			() -> downloadDataForStateHandle(path, remoteFileHandle, closeableRegistry)));
	}
	return runnables;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example 14
@Test
public void testStateStream() throws Exception {
	HashMap<String, Integer> state = new HashMap<>();
	state.put("hey there", 2);
	state.put("the crazy brown fox stumbles over a sentence that does not contain every letter", 77);

	CheckpointStateOutputStream outStream = new MemoryCheckpointOutputStream(MemoryStateBackend.DEFAULT_MAX_STATE_SIZE);
	ObjectOutputStream oos = new ObjectOutputStream(outStream);
	oos.writeObject(state);
	oos.flush();

	StreamStateHandle handle = outStream.closeAndGetHandle();
	assertNotNull(handle);

	try (ObjectInputStream ois = new ObjectInputStream(handle.openInputStream())) {
		assertEquals(state, ois.readObject());
		assertTrue(ois.available() <= 0);
	}
}
 
Example 15
@Nullable
@Override
public StreamStateHandle closeAndGetHandle() throws IOException {

	if (!closed.compareAndSet(false, true)) {
		throw new IOException("Stream was already closed!");
	}

	if (delegate instanceof CheckpointStreamFactory.CheckpointStateOutputStream) {
		StreamStateHandle streamStateHandle =
			((CheckpointStreamFactory.CheckpointStateOutputStream) delegate).closeAndGetHandle();
		unblockAll();
		return streamStateHandle;
	} else {
		unblockAll();
		throw new IOException("Delegate is not a CheckpointStateOutputStream!");
	}
}
 
Example 16
Source Project: flink   Source File: RocksDBStateDownloader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their
 * {@link StateHandleID}.
 */
private void downloadDataForAllStateHandles(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) throws Exception {

	try {
		List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry);
		List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size());
		for (Runnable runnable : runnables) {
			futures.add(CompletableFuture.runAsync(runnable, executorService));
		}
		FutureUtils.waitForAll(futures).get();
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}
}
 
Example 17
Source Project: flink   Source File: ChannelStateCheckpointWriter.java    License: Apache License 2.0 6 votes vote down vote up
private <I, H extends AbstractChannelStateHandle<I>> H createHandle(
		HandleFactory<I, H> handleFactory,
		StreamStateHandle underlying,
		I channelInfo,
		StateContentMetaInfo contentMetaInfo) throws IOException {
	Optional<byte[]> bytes = underlying.asBytesIfInMemory(); // todo: consider restructuring channel state and removing this method: https://issues.apache.org/jira/browse/FLINK-17972
	if (bytes.isPresent()) {
		StreamStateHandle extracted = new ByteStreamStateHandle(
			randomUUID().toString(),
			serializer.extractAndMerge(bytes.get(), contentMetaInfo.getOffsets()));
		return handleFactory.create(
			channelInfo,
			extracted,
			singletonList(serializer.getHeaderLength()),
			extracted.getStateSize());
	} else {
		return handleFactory.create(channelInfo, underlying, contentMetaInfo.getOffsets(), contentMetaInfo.getSize());
	}
}
 
Example 18
Source Project: flink   Source File: RocksDBStateDownloader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Copies all the files from the given stream state handles to the given path, renaming the files w.r.t. their
 * {@link StateHandleID}.
 */
private void downloadDataForAllStateHandles(
	Map<StateHandleID, StreamStateHandle> stateHandleMap,
	Path restoreInstancePath,
	CloseableRegistry closeableRegistry) throws Exception {

	try {
		List<Runnable> runnables = createDownloadRunnables(stateHandleMap, restoreInstancePath, closeableRegistry);
		List<CompletableFuture<Void>> futures = new ArrayList<>(runnables.size());
		for (Runnable runnable : runnables) {
			futures.add(CompletableFuture.runAsync(runnable, executorService));
		}
		FutureUtils.waitForAll(futures).get();
	} catch (ExecutionException e) {
		Throwable throwable = ExceptionUtils.stripExecutionException(e);
		throwable = ExceptionUtils.stripException(throwable, RuntimeException.class);
		if (throwable instanceof IOException) {
			throw (IOException) throwable;
		} else {
			throw new FlinkRuntimeException("Failed to download data for state handles.", e);
		}
	}
}
 
Example 19
Source Project: flink   Source File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {

	FSDataInputStream inputStream = null;

	try {
		inputStream = metaStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(inputStream);
		DataInputView in = new DataInputViewStreamWrapper(inputStream);
		return readMetaData(in);
	} finally {
		if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}
	}
}
 
Example 20
Source Project: flink   Source File: GenericWriteAheadSink.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Called when a checkpoint barrier arrives. It closes any open streams to the backend
 * and marks them as pending for committing to the external, third-party storage system.
 *
 * @param checkpointId the id of the latest received checkpoint.
 * @throws IOException in case something went wrong when handling the stream to the backend.
 */
private void saveHandleInState(final long checkpointId, final long timestamp) throws Exception {

	//only add handle if a new OperatorState was created since the last snapshot
	if (out != null) {
		int subtaskIdx = getRuntimeContext().getIndexOfThisSubtask();
		StreamStateHandle handle = out.closeAndGetHandle();

		PendingCheckpoint pendingCheckpoint = new PendingCheckpoint(
			checkpointId, subtaskIdx, timestamp, handle);

		if (pendingCheckpoints.contains(pendingCheckpoint)) {
			//we already have a checkpoint stored for that ID that may have been partially written,
			//so we discard this "alternate version" and use the stored checkpoint
			handle.discardState();
		} else {
			pendingCheckpoints.add(pendingCheckpoint);
		}
		out = null;
	}
}
 
Example 21
Source Project: flink   Source File: RocksIncrementalSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void createUploadFilePaths(
	FileStatus[] fileStatuses,
	Map<StateHandleID, StreamStateHandle> sstFiles,
	Map<StateHandleID, Path> sstFilePaths,
	Map<StateHandleID, Path> miscFilePaths) {
	for (FileStatus fileStatus : fileStatuses) {
		final Path filePath = fileStatus.getPath();
		final String fileName = filePath.getName();
		final StateHandleID stateHandleID = new StateHandleID(fileName);

		if (fileName.endsWith(SST_FILE_SUFFIX)) {
			final boolean existsAlready = baseSstFiles != null && baseSstFiles.contains(stateHandleID);

			if (existsAlready) {
				// we introduce a placeholder state handle, that is replaced with the
				// original from the shared state registry (created from a previous checkpoint)
				sstFiles.put(stateHandleID, new PlaceholderStreamStateHandle());
			} else {
				sstFilePaths.put(stateHandleID, filePath);
			}
		} else {
			miscFilePaths.put(stateHandleID, filePath);
		}
	}
}
 
Example 22
Source Project: flink   Source File: MemoryCheckpointStorageTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNonPersistentCheckpointLocation() throws Exception {
	MemoryBackendCheckpointStorage storage = new MemoryBackendCheckpointStorage(
			new JobID(), null, null, DEFAULT_MAX_STATE_SIZE);

	CheckpointStorageLocation location = storage.initializeLocationForCheckpoint(9);

	CheckpointMetadataOutputStream stream = location.createMetadataOutputStream();
	stream.write(99);

	CompletedCheckpointStorageLocation completed = stream.closeAndFinalizeCheckpoint();
	StreamStateHandle handle = completed.getMetadataHandle();
	assertTrue(handle instanceof ByteStreamStateHandle);

	// the reference is not valid in that case
	try {
		storage.resolveCheckpoint(completed.getExternalPointer());
		fail("should fail with an exception");
	} catch (Exception e) {
		// expected
	}
}
 
Example 23
Source Project: bravo   Source File: RocksDBCheckpointIterator.java    License: Apache License 2.0 5 votes vote down vote up
private void transferAllDataFromStateHandles(
		Map<StateHandleID, StreamStateHandle> stateHandleMap,
		Path restoreInstancePath) throws IOException {

	for (Map.Entry<StateHandleID, StreamStateHandle> entry : stateHandleMap.entrySet()) {
		StateHandleID stateHandleID = entry.getKey();
		StreamStateHandle remoteFileHandle = entry.getValue();
		copyStateDataHandleData(new Path(restoreInstancePath, stateHandleID.toString()), remoteFileHandle);
	}
}
 
Example 24
Source Project: flink   Source File: FsCheckpointStorageTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testTaskOwnedStateStream() throws Exception {
	final List<String> state = Arrays.asList("Flopsy", "Mopsy", "Cotton Tail", "Peter");

	// we chose a small size threshold here to force the state to disk
	final FsCheckpointStorage storage = new FsCheckpointStorage(
			Path.fromLocalFile(tmp.newFolder()),  null, new JobID(), 10, WRITE_BUFFER_SIZE);

	final StreamStateHandle stateHandle;

	try (CheckpointStateOutputStream stream = storage.createTaskOwnedStateStream()) {
		assertTrue(stream instanceof FsCheckpointStateOutputStream);

		new ObjectOutputStream(stream).writeObject(state);
		stateHandle = stream.closeAndGetHandle();
	}

	// the state must have gone to disk
	FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;

	// check that the state is in the correct directory
	String parentDirName = fileStateHandle.getFilePath().getParent().getName();
	assertEquals(FsCheckpointStorage.CHECKPOINT_TASK_OWNED_STATE_DIR, parentDirName);

	// validate the contents
	try (ObjectInputStream in = new ObjectInputStream(stateHandle.openInputStream())) {
		assertEquals(state, in.readObject());
	}
}
 
Example 25
Source Project: flink   Source File: ChannelStateCheckpointWriter.java    License: Apache License 2.0 5 votes vote down vote up
private void finishWriteAndResult() throws IOException {
	if (inputChannelOffsets.isEmpty() && resultSubpartitionOffsets.isEmpty()) {
		dataStream.close();
		result.inputChannelStateHandles.complete(emptyList());
		result.resultSubpartitionStateHandles.complete(emptyList());
		return;
	}
	dataStream.flush();
	StreamStateHandle underlying = checkpointStream.closeAndGetHandle();
	complete(underlying, result.inputChannelStateHandles, inputChannelOffsets, HandleFactory.INPUT_CHANNEL);
	complete(underlying, result.resultSubpartitionStateHandles, resultSubpartitionOffsets, HandleFactory.RESULT_SUBPARTITION);
}
 
Example 26
Source Project: Flink-CEPplus   Source File: RocksDBStateDownloader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Copies the file from a single state handle to the given path.
 */
private void downloadDataForStateHandle(
	Path restoreFilePath,
	StreamStateHandle remoteFileHandle,
	CloseableRegistry closeableRegistry) throws IOException {

	FSDataInputStream inputStream = null;
	FSDataOutputStream outputStream = null;

	try {
		FileSystem restoreFileSystem = restoreFilePath.getFileSystem();
		inputStream = remoteFileHandle.openInputStream();
		closeableRegistry.registerCloseable(inputStream);

		outputStream = restoreFileSystem.create(restoreFilePath, FileSystem.WriteMode.OVERWRITE);
		closeableRegistry.registerCloseable(outputStream);

		byte[] buffer = new byte[8 * 1024];
		while (true) {
			int numBytes = inputStream.read(buffer);
			if (numBytes == -1) {
				break;
			}

			outputStream.write(buffer, 0, numBytes);
		}
	} finally {
		if (closeableRegistry.unregisterCloseable(inputStream)) {
			inputStream.close();
		}

		if (closeableRegistry.unregisterCloseable(outputStream)) {
			outputStream.close();
		}
	}
}
 
Example 27
Source Project: flink   Source File: InterruptSensitiveRestoreTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testRestoreWithInterrupt(int mode) throws Exception {

		IN_RESTORE_LATCH.reset();
		Configuration taskConfig = new Configuration();
		StreamConfig cfg = new StreamConfig(taskConfig);
		cfg.setTimeCharacteristic(TimeCharacteristic.ProcessingTime);
		switch (mode) {
			case OPERATOR_MANAGED:
			case OPERATOR_RAW:
			case KEYED_MANAGED:
			case KEYED_RAW:
				cfg.setStateKeySerializer(IntSerializer.INSTANCE);
				cfg.setStreamOperator(new StreamSource<>(new TestSource(mode)));
				break;
			default:
				throw new IllegalArgumentException();
		}

		StreamStateHandle lockingHandle = new InterruptLockingStateHandle();

		Task task = createTask(cfg, taskConfig, lockingHandle, mode);

		// start the task and wait until it is in "restore"
		task.startTaskThread();
		IN_RESTORE_LATCH.await();

		// trigger cancellation and signal to continue
		task.cancelExecution();

		task.getExecutingThread().join(30000);

		if (task.getExecutionState() == ExecutionState.CANCELING) {
			fail("Task is stuck and not canceling");
		}

		assertEquals(ExecutionState.CANCELED, task.getExecutionState());
		assertNull(task.getFailureCause());
	}
 
Example 28
Source Project: flink   Source File: RocksDBStateUploader.java    License: Apache License 2.0 5 votes vote down vote up
private Map<StateHandleID, CompletableFuture<StreamStateHandle>> createUploadFutures(
	Map<StateHandleID, Path> files,
	CheckpointStreamFactory checkpointStreamFactory,
	CloseableRegistry closeableRegistry) {
	Map<StateHandleID, CompletableFuture<StreamStateHandle>> futures = new HashMap<>(files.size());

	for (Map.Entry<StateHandleID, Path> entry : files.entrySet()) {
		final Supplier<StreamStateHandle> supplier =
			CheckedSupplier.unchecked(() -> uploadLocalFileToCheckpointFs(entry.getValue(), checkpointStreamFactory, closeableRegistry));
		futures.put(entry.getKey(), CompletableFuture.supplyAsync(supplier, executorService));
	}

	return futures;
}
 
Example 29
Source Project: flink   Source File: ChannelStateCheckpointWriterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testEmptyState() throws Exception {
	MemoryCheckpointOutputStream stream = new MemoryCheckpointOutputStream(1000) {
		@Override
		public StreamStateHandle closeAndGetHandle() {
			fail("closeAndGetHandle shouldn't be called for empty channel state");
			return null;
		}
	};
	ChannelStateCheckpointWriter writer = createWriter(new ChannelStateWriteResult(), stream);
	writer.completeOutput();
	writer.completeInput();
	assertTrue(stream.isClosed());
}
 
Example 30
Source Project: Flink-CEPplus   Source File: RocksDBStateUploaderTest.java    License: Apache License 2.0 5 votes vote down vote up
private CheckpointStreamFactory.CheckpointStateOutputStream createFailingCheckpointStateOutputStream(
	IOException failureException) {
	return new CheckpointStreamFactory.CheckpointStateOutputStream() {
		@Nullable
		@Override
		public StreamStateHandle closeAndGetHandle() {
			return new ByteStreamStateHandle("testHandle", "testHandle".getBytes());
		}

		@Override
		public void close() {
		}

		@Override
		public long getPos() {
			return 0;
		}

		@Override
		public void flush() {
		}

		@Override
		public void sync() {
		}

		@Override
		public void write(int b) throws IOException {
			throw failureException;
		}
	};
}