org.apache.flink.runtime.state.filesystem.AbstractFsCheckpointStorage Java Examples

The following examples show how to use org.apache.flink.runtime.state.filesystem.AbstractFsCheckpointStorage. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BravoTestPipeline.java    From bravo with Apache License 2.0 6 votes vote down vote up
public static Savepoint loadSavepoint(String checkpointPointer) {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				BravoTestPipeline.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 
Example #2
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 6 votes vote down vote up
/**
 * Load the Savepoint metadata object from the given path
 */
public static Savepoint loadSavepoint(String checkpointPointer) throws IOException {
	try {
		Method resolveCheckpointPointer = AbstractFsCheckpointStorage.class.getDeclaredMethod(
				"resolveCheckpointPointer",
				String.class);
		resolveCheckpointPointer.setAccessible(true);
		CompletedCheckpointStorageLocation loc = (CompletedCheckpointStorageLocation) resolveCheckpointPointer
				.invoke(null, checkpointPointer);

		return Checkpoints.loadCheckpointMetadata(new DataInputStream(loc.getMetadataHandle().openInputStream()),
				StateMetadataUtils.class.getClassLoader());
	} catch (Exception e) {
		throw new RuntimeException(e);
	}

}
 
Example #3
Source File: PersistentMetadataCheckpointStorageLocation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 
Example #4
Source File: SavepointOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 
Example #5
Source File: SnapshotUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(
	OP operator,
	int index,
	long timestamp,
	CheckpointStorageWorkerView checkpointStorage,
	Path savepointPath) throws Exception {

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.SAVEPOINT,
		AbstractFsCheckpointStorage.encodePathAsReference(savepointPath));

	operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);

	CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation(
		CHECKPOINT_ID,
		options.getTargetLocation());

	OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(
		CHECKPOINT_ID,
		timestamp,
		options,
		storage);

	OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();

	operator.notifyCheckpointComplete(CHECKPOINT_ID);
	return new TaggedOperatorSubtaskState(index, state);
}
 
Example #6
Source File: PersistentMetadataCheckpointStorageLocation.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 
Example #7
Source File: SavepointOutputFormat.java    From flink with Apache License 2.0 5 votes vote down vote up
private static CheckpointStorageLocation createSavepointLocation(Path location) throws IOException {
	final CheckpointStorageLocationReference reference = AbstractFsCheckpointStorage.encodePathAsReference(location);
	return new FsCheckpointStorageLocation(
		location.getFileSystem(),
		location,
		location,
		location,
		reference,
		(int) CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue().getBytes(),
		CheckpointingOptions.FS_WRITE_BUFFER_SIZE.defaultValue());
}
 
Example #8
Source File: SnapshotUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static <OUT, OP extends StreamOperator<OUT>> TaggedOperatorSubtaskState snapshot(
	OP operator,
	int index,
	long timestamp,
	boolean isExactlyOnceMode,
	boolean isUnalignedCheckpoint,
	CheckpointStorageWorkerView checkpointStorage,
	Path savepointPath) throws Exception {

	CheckpointOptions options = new CheckpointOptions(
		CheckpointType.SAVEPOINT,
		AbstractFsCheckpointStorage.encodePathAsReference(savepointPath),
		isExactlyOnceMode,
		isUnalignedCheckpoint);

	operator.prepareSnapshotPreBarrier(CHECKPOINT_ID);

	CheckpointStreamFactory storage = checkpointStorage.resolveCheckpointStorageLocation(
		CHECKPOINT_ID,
		options.getTargetLocation());

	OperatorSnapshotFutures snapshotInProgress = operator.snapshotState(
		CHECKPOINT_ID,
		timestamp,
		options,
		storage);

	OperatorSubtaskState state = new OperatorSnapshotFinalizer(snapshotInProgress).getJobManagerOwnedState();

	operator.notifyCheckpointComplete(CHECKPOINT_ID);
	return new TaggedOperatorSubtaskState(index, state);
}
 
Example #9
Source File: MetadataV2V3SerializerBase.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Path createExclusiveDirPath(String externalPointer) throws IOException {
	try {
		return AbstractFsCheckpointStorage.resolveCheckpointPointer(externalPointer).getExclusiveCheckpointDir();
	} catch (IOException e) {
		throw new IOException("Could not parse external pointer as state base path", e);
	}
}
 
Example #10
Source File: PersistentMetadataCheckpointStorageLocation.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a checkpoint storage persists metadata to a file system and stores state
 * in line in state handles with the metadata.
 *
 * @param fileSystem The file system to which the metadata will be written.
 * @param checkpointDir The directory where the checkpoint metadata will be written.
 */
public PersistentMetadataCheckpointStorageLocation(
		FileSystem fileSystem,
		Path checkpointDir,
		int maxStateSize) {

	super(maxStateSize);

	this.fileSystem = checkNotNull(fileSystem);
	this.checkpointDirectory = checkNotNull(checkpointDir);
	this.metadataFilePath = new Path(checkpointDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
}
 
Example #11
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 4 votes vote down vote up
public static Path writeSavepointMetadata(Path newCheckpointBasePath, Savepoint savepoint) throws IOException {
	Path p = new Path(newCheckpointBasePath, AbstractFsCheckpointStorage.METADATA_FILE_NAME);
	Checkpoints.storeCheckpointMetadata(savepoint,
			newCheckpointBasePath.getFileSystem().create(p, WriteMode.NO_OVERWRITE));
	return p;
}
 
Example #12
Source File: SavepointLoader.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static Savepoint loadSavepoint(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader());
	}
}
 
Example #13
Source File: SavepointLoader.java    From flink with Apache License 2.0 3 votes vote down vote up
/**
 * Takes the given string (representing a pointer to a checkpoint) and resolves it to a file
 * status for the checkpoint's metadata file.
 *
 *<p>This should only be used when the user code class loader is the current classloader for
 * the thread.
 * @param savepointPath The path to an external savepoint.
 * @return A state handle to savepoint's metadata.
 * @throws IOException Thrown, if the path cannot be resolved, the file system not accessed, or
 *     the path points to a location that does not seem to be a savepoint.
 */
public static CheckpointMetadata loadSavepointMetadata(String savepointPath) throws IOException {
	CompletedCheckpointStorageLocation location = AbstractFsCheckpointStorage
		.resolveCheckpointPointer(savepointPath);

	try (DataInputStream stream = new DataInputStream(location.getMetadataHandle().openInputStream())) {
		return Checkpoints.loadCheckpointMetadata(stream, Thread.currentThread().getContextClassLoader(), savepointPath);
	}
}