org.apache.flink.contrib.streaming.state.snapshot.RocksDBSnapshotStrategyBase Java Examples

/**
 * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and
 * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always
 * be called by the same thread.
 *
 * @param checkpointId  The Id of the checkpoint.
 * @param timestamp     The timestamp of the checkpoint.
 * @param streamFactory The factory that we can use for writing our state to streams.
 * @param checkpointOptions Options for how to perform this checkpoint.
 * @return Future to the state handle of the snapshot data.
 * @throws Exception indicating a problem in the synchronous part of the checkpoint.
 */
@Nonnull
@Override
public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
	final long checkpointId,
	final long timestamp,
	@Nonnull final CheckpointStreamFactory streamFactory,
	@Nonnull CheckpointOptions checkpointOptions) throws Exception {

	long startTime = System.currentTimeMillis();

	// flush everything into db before taking a snapshot
	writeBatchWrapper.flush();

	RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy =
		CheckpointType.SAVEPOINT == checkpointOptions.getCheckpointType() ?
			savepointSnapshotStrategy : checkpointSnapshotStrategy;

	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner =
		chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);

	chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime);

	return snapshotRunner;
}
 

/**
 * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and
 * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always
 * be called by the same thread.
 *
 * @param checkpointId  The Id of the checkpoint.
 * @param timestamp     The timestamp of the checkpoint.
 * @param streamFactory The factory that we can use for writing our state to streams.
 * @param checkpointOptions Options for how to perform this checkpoint.
 * @return Future to the state handle of the snapshot data.
 * @throws Exception indicating a problem in the synchronous part of the checkpoint.
 */
@Nonnull
@Override
public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
	final long checkpointId,
	final long timestamp,
	@Nonnull final CheckpointStreamFactory streamFactory,
	@Nonnull CheckpointOptions checkpointOptions) throws Exception {

	long startTime = System.currentTimeMillis();

	// flush everything into db before taking a snapshot
	writeBatchWrapper.flush();

	RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy =
			checkpointOptions.getCheckpointType().isSavepoint() ? savepointSnapshotStrategy : checkpointSnapshotStrategy;

	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner =
		chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);

	chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime);

	return snapshotRunner;
}
 

/**
 * Triggers an asynchronous snapshot of the keyed state backend from RocksDB. This snapshot can be canceled and
 * is also stopped when the backend is closed through {@link #dispose()}. For each backend, this method must always
 * be called by the same thread.
 *
 * @param checkpointId  The Id of the checkpoint.
 * @param timestamp     The timestamp of the checkpoint.
 * @param streamFactory The factory that we can use for writing our state to streams.
 * @param checkpointOptions Options for how to perform this checkpoint.
 * @return Future to the state handle of the snapshot data.
 * @throws Exception indicating a problem in the synchronous part of the checkpoint.
 */
@Nonnull
@Override
public RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshot(
	final long checkpointId,
	final long timestamp,
	@Nonnull final CheckpointStreamFactory streamFactory,
	@Nonnull CheckpointOptions checkpointOptions) throws Exception {

	long startTime = System.currentTimeMillis();

	// flush everything into db before taking a snapshot
	writeBatchWrapper.flush();

	RocksDBSnapshotStrategyBase<K> chosenSnapshotStrategy =
			checkpointOptions.getCheckpointType().isSavepoint() ? savepointSnapshotStrategy : checkpointSnapshotStrategy;

	RunnableFuture<SnapshotResult<KeyedStateHandle>> snapshotRunner =
		chosenSnapshotStrategy.snapshot(checkpointId, timestamp, streamFactory, checkpointOptions);

	chosenSnapshotStrategy.logSyncCompleted(streamFactory, startTime);

	return snapshotRunner;
}
 

public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager) {

	super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
		keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.dbOptions = Preconditions.checkNotNull(dbOptions);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = new WriteOptions().setDisableWAL(true);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
 

private SnapshotStrategy<K> initializeSavepointAndCheckpointStrategies(
	CloseableRegistry cancelStreamRegistry,
	ResourceGuard rocksDBResourceGuard,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	RocksDB db,
	UUID backendUID,
	SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId) {
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy = new RocksFullSnapshotStrategy<>(
		db,
		rocksDBResourceGuard,
		keySerializerProvider.currentSchemaSerializer(),
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry,
		keyGroupCompressionDecorator);
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
	if (enableIncrementalCheckpointing) {
		// TODO eventually we might want to separate savepoint and snapshot strategy, i.e. having 2 strategies.
		checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
			db,
			rocksDBResourceGuard,
			keySerializerProvider.currentSchemaSerializer(),
			kvStateInformation,
			keyGroupRange,
			keyGroupPrefixBytes,
			localRecoveryConfig,
			cancelStreamRegistry,
			instanceBasePath,
			backendUID,
			materializedSstFiles,
			lastCompletedCheckpointId,
			numberOfTransferingThreads);
	} else {
		checkpointSnapshotStrategy = savepointSnapshotStrategy;
	}
	return new SnapshotStrategy<>(checkpointSnapshotStrategy, savepointSnapshotStrategy);
}
 

SnapshotStrategy(RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
				RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy) {
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
}
 

public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.dbOptions = Preconditions.checkNotNull(dbOptions);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = new WriteOptions().setDisableWAL(true);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
 

private SnapshotStrategy<K> initializeSavepointAndCheckpointStrategies(
	CloseableRegistry cancelStreamRegistry,
	ResourceGuard rocksDBResourceGuard,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	RocksDB db,
	UUID backendUID,
	SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId) {
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy = new RocksFullSnapshotStrategy<>(
		db,
		rocksDBResourceGuard,
		keySerializerProvider.currentSchemaSerializer(),
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry,
		keyGroupCompressionDecorator);
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
	if (enableIncrementalCheckpointing) {
		// TODO eventually we might want to separate savepoint and snapshot strategy, i.e. having 2 strategies.
		checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
			db,
			rocksDBResourceGuard,
			keySerializerProvider.currentSchemaSerializer(),
			kvStateInformation,
			keyGroupRange,
			keyGroupPrefixBytes,
			localRecoveryConfig,
			cancelStreamRegistry,
			instanceBasePath,
			backendUID,
			materializedSstFiles,
			lastCompletedCheckpointId,
			numberOfTransferingThreads);
	} else {
		checkpointSnapshotStrategy = savepointSnapshotStrategy;
	}
	return new SnapshotStrategy<>(checkpointSnapshotStrategy, savepointSnapshotStrategy);
}
 

SnapshotStrategy(RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
				RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy) {
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
}
 

public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	RocksDBResourceContainer optionsContainer,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext,
	@Nonnegative long writeBatchSize) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.optionsContainer = Preconditions.checkNotNull(optionsContainer);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = optionsContainer.getWriteOptions();
	this.readOptions = optionsContainer.getReadOptions();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative value.");
	this.writeBatchSize = writeBatchSize;
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}
 

private SnapshotStrategy<K> initializeSavepointAndCheckpointStrategies(
	CloseableRegistry cancelStreamRegistry,
	ResourceGuard rocksDBResourceGuard,
	LinkedHashMap<String, RocksDBKeyedStateBackend.RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	RocksDB db,
	UUID backendUID,
	SortedMap<Long, Set<StateHandleID>> materializedSstFiles,
	long lastCompletedCheckpointId) {
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy = new RocksFullSnapshotStrategy<>(
		db,
		rocksDBResourceGuard,
		keySerializerProvider.currentSchemaSerializer(),
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry,
		keyGroupCompressionDecorator);
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy;
	if (enableIncrementalCheckpointing) {
		// TODO eventually we might want to separate savepoint and snapshot strategy, i.e. having 2 strategies.
		checkpointSnapshotStrategy = new RocksIncrementalSnapshotStrategy<>(
			db,
			rocksDBResourceGuard,
			keySerializerProvider.currentSchemaSerializer(),
			kvStateInformation,
			keyGroupRange,
			keyGroupPrefixBytes,
			localRecoveryConfig,
			cancelStreamRegistry,
			instanceBasePath,
			backendUID,
			materializedSstFiles,
			lastCompletedCheckpointId,
			numberOfTransferingThreads);
	} else {
		checkpointSnapshotStrategy = savepointSnapshotStrategy;
	}
	return new SnapshotStrategy<>(checkpointSnapshotStrategy, savepointSnapshotStrategy);
}
 

SnapshotStrategy(RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
				RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy) {
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
}