org.apache.flink.runtime.state.StreamCompressionDecorator Java Examples

The following examples show how to use org.apache.flink.runtime.state.StreamCompressionDecorator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: MockKeyedStateBackendBuilder.java From flink with Apache License 2.0

6 votes

public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}

Example #2

Source File: HeapSnapshotStrategy.java From flink with Apache License 2.0

6 votes

HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}

Example #3

Source File: RocksFullSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0

6 votes

public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}

Example #4

Source File: HeapKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0

6 votes

public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	StateSerializerProvider<K> keySerializerProvider,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy
) {
	super(kvStateRegistry, keySerializerProvider, userCodeClassLoader, numberOfKeyGroups,
		keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry, keyGroupCompressionDecorator);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}

Example #5

Source File: HeapSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0

6 votes

HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}

Example #6

Source File: MockKeyedStateBackendBuilder.java From Flink-CEPplus with Apache License 2.0

6 votes

public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}

Example #7

Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0

6 votes

public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}

Example #8

Source File: MockKeyedStateBackendBuilder.java From flink with Apache License 2.0

6 votes

public MockKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
}

Example #9

Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0

6 votes

public RocksFullSnapshotStrategy(
	@Nonnull RocksDB db,
	@Nonnull ResourceGuard rocksDBResourceGuard,
	@Nonnull TypeSerializer<K> keySerializer,
	@Nonnull LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	@Nonnull KeyGroupRange keyGroupRange,
	@Nonnegative int keyGroupPrefixBytes,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull CloseableRegistry cancelStreamRegistry,
	@Nonnull StreamCompressionDecorator keyGroupCompressionDecorator) {
	super(
		DESCRIPTION,
		db,
		rocksDBResourceGuard,
		keySerializer,
		kvStateInformation,
		keyGroupRange,
		keyGroupPrefixBytes,
		localRecoveryConfig,
		cancelStreamRegistry);

	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
}

Example #10

Source File: HeapSnapshotStrategy.java From flink with Apache License 2.0

6 votes

HeapSnapshotStrategy(
	SnapshotStrategySynchronicityBehavior<K> snapshotStrategySynchronicityTrait,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	KeyGroupRange keyGroupRange,
	CloseableRegistry cancelStreamRegistry,
	StateSerializerProvider<K> keySerializerProvider) {
	super("Heap backend snapshot");
	this.snapshotStrategySynchronicityTrait = snapshotStrategySynchronicityTrait;
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.keyGroupCompressionDecorator = keyGroupCompressionDecorator;
	this.localRecoveryConfig = localRecoveryConfig;
	this.keyGroupRange = keyGroupRange;
	this.cancelStreamRegistry = cancelStreamRegistry;
	this.keySerializerProvider = keySerializerProvider;
}

Example #11

Source File: HeapKeyedStateBackend.java From flink with Apache License 2.0

5 votes

public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy,
	InternalKeyContext<K> keyContext) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}

Example #12

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

5 votes

private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}

Example #13

Source File: HeapKeyedStateBackend.java From flink with Apache License 2.0

5 votes

public HeapKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	Map<String, StateTable<K, ?, ?>> registeredKVStates,
	Map<String, HeapPriorityQueueSnapshotRestoreWrapper> registeredPQStates,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	HeapSnapshotStrategy<K> snapshotStrategy,
	InternalKeyContext<K> keyContext) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);
	this.registeredKVStates = registeredKVStates;
	this.registeredPQStates = registeredPQStates;
	this.localRecoveryConfig = localRecoveryConfig;
	LOG.info("Initializing heap keyed state backend with stream factory.");
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.snapshotStrategy = snapshotStrategy;
}

Example #14

Source File: HeapKeyedStateBackendBuilder.java From flink with Apache License 2.0

5 votes

public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}

Example #15

Source File: RocksDBSavepointIterator.java From bravo with Apache License 2.0

5 votes

private boolean openIfNeeded() throws Exception {
	if (stateHandleInStream == null) {
		LOGGER.debug("Opening {}", keyGroupsStateHandle.getDelegateStateHandle());
		stateHandleInStream = keyGroupsStateHandle.openInputStream();

		final KeyedBackendSerializationProxy<?> serializationProxy = StateMetadataUtils
				.getKeyedBackendSerializationProxy(keyGroupsStateHandle);

		this.stateIdMapping = StateMetadataUtils.getStateIdMapping(serializationProxy);
		final StreamCompressionDecorator streamCompressionDecorator = StateMetadataUtils
				.getCompressionDecorator(serializationProxy);

		final KeyGroupRangeOffsets rangeOffsets = keyGroupsStateHandle.getGroupRangeOffsets();
		LOGGER.debug("{}", rangeOffsets);

		offsetsIt = new ValidOffsetsIterator(rangeOffsets);

		hasNext = seekNextOffset();

		if (hasNext) {
			final InputStream compressedInputStream = streamCompressionDecorator
					.decorateWithCompression(stateHandleInStream);
			compressedInputView = new DataInputViewStreamWrapper(compressedInputStream);
			seekNextStateId(false);
		}
	}

	return hasNext;
}

Example #16

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

5 votes

private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}

Example #17

Source File: HeapKeyedStateBackendBuilder.java From flink with Apache License 2.0

5 votes

public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}

Example #18

Source File: HeapRestoreOperation.java From Flink-CEPplus with Apache License 2.0

5 votes

private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}

Example #19

Source File: HeapKeyedStateBackendBuilder.java From Flink-CEPplus with Apache License 2.0

5 votes

public HeapKeyedStateBackendBuilder(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	LocalRecoveryConfig localRecoveryConfig,
	HeapPriorityQueueSetFactory priorityQueueSetFactory,
	boolean asynchronousSnapshots,
	CloseableRegistry cancelStreamRegistry) {
	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);
	this.localRecoveryConfig = localRecoveryConfig;
	this.priorityQueueSetFactory = priorityQueueSetFactory;
	this.asynchronousSnapshots = asynchronousSnapshots;
}

Example #20

Source File: RocksDBStateBackend.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {

	// first, make sure that the RocksDB JNI library is loaded
	// we do this explicitly here to have better error handling
	String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0];
	ensureRocksDBIsLoaded(tempDir);

	// replace all characters that are not legal for filenames with underscore
	String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");

	lazyInitializeForJob(env, fileCompatibleIdentifier);

	File instanceBasePath = new File(
		getNextStoragePath(),
		"job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());

	LocalRecoveryConfig localRecoveryConfig =
		env.getTaskStateManager().createLocalRecoveryConfig();

	ExecutionConfig executionConfig = env.getExecutionConfig();
	StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
	RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(
		operatorIdentifier,
		env.getUserClassLoader(),
		instanceBasePath,
		getDbOptions(),
		stateName -> getColumnOptions(),
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	).setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled())
		.setEnableTtlCompactionFilter(isTtlCompactionFilterEnabled())
		.setNumberOfTransferingThreads(getNumberOfTransferingThreads())
		.setNativeMetricOptions(getMemoryWatcherOptions());
	return builder.build();
}

Example #21

Source File: RocksDBKeyedStateBackendBuilder.java From flink with Apache License 2.0

4 votes

@VisibleForTesting
RocksDBKeyedStateBackendBuilder(
	String operatorIdentifier,
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	LocalRecoveryConfig localRecoveryConfig,
	RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	RocksDB injectedTestDB,
	ColumnFamilyHandle injectedDefaultColumnFamilyHandle,
	CloseableRegistry cancelStreamRegistry) {
	this(
		operatorIdentifier,
		userCodeClassLoader,
		instanceBasePath,
		dbOptions,
		columnFamilyOptionsFactory,
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	);
	this.injectedTestDB = injectedTestDB;
	this.injectedDefaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
}

Example #22

Source File: RocksDBKeyedStateBackendBuilder.java From flink with Apache License 2.0

4 votes

public RocksDBKeyedStateBackendBuilder(
	String operatorIdentifier,
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	LocalRecoveryConfig localRecoveryConfig,
	RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);

	this.operatorIdentifier = operatorIdentifier;
	this.priorityQueueStateType = priorityQueueStateType;
	this.localRecoveryConfig = localRecoveryConfig;
	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
	this.dbOptions = dbOptions;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = new File(instanceBasePath, DB_INSTANCE_DIR_STRING);
	this.metricGroup = metricGroup;
	this.enableIncrementalCheckpointing = false;
	this.nativeMetricOptions = new RocksDBNativeMetricOptions();
	this.numberOfTransferingThreads = RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue();
}

Example #23

Source File: StateMetadataUtils.java From bravo with Apache License 2.0

4 votes

public static StreamCompressionDecorator getCompressionDecorator(KeyedBackendSerializationProxy<?> proxy) {
	return proxy.isUsingKeyGroupCompression()
			? SnappyStreamCompressionDecorator.INSTANCE
			: UncompressedStreamCompressionDecorator.INSTANCE;
}

Example #24

Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0

4 votes

public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.dbOptions = Preconditions.checkNotNull(dbOptions);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = new WriteOptions().setDisableWAL(true);
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}

Example #25

Source File: RocksDBStateBackend.java From flink with Apache License 2.0

4 votes

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {

	// first, make sure that the RocksDB JNI library is loaded
	// we do this explicitly here to have better error handling
	String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0];
	ensureRocksDBIsLoaded(tempDir);

	// replace all characters that are not legal for filenames with underscore
	String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");

	lazyInitializeForJob(env, fileCompatibleIdentifier);

	File instanceBasePath = new File(
		getNextStoragePath(),
		"job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());

	LocalRecoveryConfig localRecoveryConfig =
		env.getTaskStateManager().createLocalRecoveryConfig();

	final OpaqueMemoryResource<RocksDBSharedResources> sharedResources = RocksDBOperationUtils
			.allocateSharedCachesIfConfigured(memoryConfiguration, env.getMemoryManager(), LOG);
	if (sharedResources != null) {
		LOG.info("Obtained shared RocksDB cache of size {} bytes", sharedResources.getSize());
	}
	final RocksDBResourceContainer resourceContainer = createOptionsAndResourceContainer(sharedResources);

	ExecutionConfig executionConfig = env.getExecutionConfig();
	StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
	RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(
		operatorIdentifier,
		env.getUserClassLoader(),
		instanceBasePath,
		resourceContainer,
		stateName -> resourceContainer.getColumnOptions(),
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		getPriorityQueueStateType(),
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	)
		.setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled())
		.setNumberOfTransferingThreads(getNumberOfTransferThreads())
		.setNativeMetricOptions(resourceContainer.getMemoryWatcherOptions(defaultMetricOptions))
		.setWriteBatchSize(getWriteBatchSize());
	return builder.build();
}

Example #26

Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0

4 votes

public RocksDBKeyedStateBackend(
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	RocksDBResourceContainer optionsContainer,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	RocksDB db,
	LinkedHashMap<String, RocksDbKvStateInfo> kvStateInformation,
	int keyGroupPrefixBytes,
	CloseableRegistry cancelStreamRegistry,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	ResourceGuard rocksDBResourceGuard,
	RocksDBSnapshotStrategyBase<K> checkpointSnapshotStrategy,
	RocksDBSnapshotStrategyBase<K> savepointSnapshotStrategy,
	RocksDBWriteBatchWrapper writeBatchWrapper,
	ColumnFamilyHandle defaultColumnFamilyHandle,
	RocksDBNativeMetricMonitor nativeMetricMonitor,
	RocksDBSerializedCompositeKeyBuilder<K> sharedRocksKeyBuilder,
	PriorityQueueSetFactory priorityQueueFactory,
	RocksDbTtlCompactFiltersManager ttlCompactFiltersManager,
	InternalKeyContext<K> keyContext,
	@Nonnegative long writeBatchSize) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		cancelStreamRegistry,
		keyGroupCompressionDecorator,
		keyContext);

	this.ttlCompactFiltersManager = ttlCompactFiltersManager;

	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);

	this.optionsContainer = Preconditions.checkNotNull(optionsContainer);

	this.instanceBasePath = Preconditions.checkNotNull(instanceBasePath);

	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.kvStateInformation = kvStateInformation;

	this.writeOptions = optionsContainer.getWriteOptions();
	this.readOptions = optionsContainer.getReadOptions();
	checkArgument(writeBatchSize >= 0, "Write batch size have to be no negative value.");
	this.writeBatchSize = writeBatchSize;
	this.db = db;
	this.rocksDBResourceGuard = rocksDBResourceGuard;
	this.checkpointSnapshotStrategy = checkpointSnapshotStrategy;
	this.savepointSnapshotStrategy = savepointSnapshotStrategy;
	this.writeBatchWrapper = writeBatchWrapper;
	this.defaultColumnFamily = defaultColumnFamilyHandle;
	this.nativeMetricMonitor = nativeMetricMonitor;
	this.sharedRocksKeyBuilder = sharedRocksKeyBuilder;
	this.priorityQueueFactory = priorityQueueFactory;
}

Example #27

Source File: RocksDBKeyedStateBackendBuilder.java From flink with Apache License 2.0

4 votes

public RocksDBKeyedStateBackendBuilder(
	String operatorIdentifier,
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	RocksDBResourceContainer optionsContainer,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	LocalRecoveryConfig localRecoveryConfig,
	RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	CloseableRegistry cancelStreamRegistry) {

	super(
		kvStateRegistry,
		keySerializer,
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry);

	this.operatorIdentifier = operatorIdentifier;
	this.priorityQueueStateType = priorityQueueStateType;
	this.localRecoveryConfig = localRecoveryConfig;
	// ensure that we use the right merge operator, because other code relies on this
	this.columnFamilyOptionsFactory = Preconditions.checkNotNull(columnFamilyOptionsFactory);
	this.optionsContainer = optionsContainer;
	this.instanceBasePath = instanceBasePath;
	this.instanceRocksDBPath = new File(instanceBasePath, DB_INSTANCE_DIR_STRING);
	this.metricGroup = metricGroup;
	this.enableIncrementalCheckpointing = false;
	this.nativeMetricOptions = new RocksDBNativeMetricOptions();
	this.numberOfTransferingThreads = RocksDBOptions.CHECKPOINT_TRANSFER_THREAD_NUM.defaultValue();
}

Example #28

Source File: RocksDBKeyedStateBackendBuilder.java From flink with Apache License 2.0

4 votes

@VisibleForTesting
RocksDBKeyedStateBackendBuilder(
	String operatorIdentifier,
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	RocksDBResourceContainer optionsContainer,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	LocalRecoveryConfig localRecoveryConfig,
	RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	RocksDB injectedTestDB,
	ColumnFamilyHandle injectedDefaultColumnFamilyHandle,
	CloseableRegistry cancelStreamRegistry) {
	this(
		operatorIdentifier,
		userCodeClassLoader,
		instanceBasePath,
		optionsContainer,
		columnFamilyOptionsFactory,
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	);
	this.injectedTestDB = injectedTestDB;
	this.injectedDefaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
}

Example #29

Source File: RocksDBStateBackend.java From flink with Apache License 2.0

4 votes

@Override
public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend(
	Environment env,
	JobID jobID,
	String operatorIdentifier,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	TaskKvStateRegistry kvStateRegistry,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	CloseableRegistry cancelStreamRegistry) throws IOException {

	// first, make sure that the RocksDB JNI library is loaded
	// we do this explicitly here to have better error handling
	String tempDir = env.getTaskManagerInfo().getTmpDirectories()[0];
	ensureRocksDBIsLoaded(tempDir);

	// replace all characters that are not legal for filenames with underscore
	String fileCompatibleIdentifier = operatorIdentifier.replaceAll("[^a-zA-Z0-9\\-]", "_");

	lazyInitializeForJob(env, fileCompatibleIdentifier);

	File instanceBasePath = new File(
		getNextStoragePath(),
		"job_" + jobId + "_op_" + fileCompatibleIdentifier + "_uuid_" + UUID.randomUUID());

	LocalRecoveryConfig localRecoveryConfig =
		env.getTaskStateManager().createLocalRecoveryConfig();

	ExecutionConfig executionConfig = env.getExecutionConfig();
	StreamCompressionDecorator keyGroupCompressionDecorator = getCompressionDecorator(executionConfig);
	RocksDBKeyedStateBackendBuilder<K> builder = new RocksDBKeyedStateBackendBuilder<>(
		operatorIdentifier,
		env.getUserClassLoader(),
		instanceBasePath,
		getDbOptions(),
		stateName -> getColumnOptions(),
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	).setEnableIncrementalCheckpointing(isIncrementalCheckpointsEnabled())
		.setEnableTtlCompactionFilter(isTtlCompactionFilterEnabled())
		.setNumberOfTransferingThreads(getNumberOfTransferingThreads())
		.setNativeMetricOptions(getMemoryWatcherOptions());
	return builder.build();
}

Example #30

Source File: RocksDBKeyedStateBackendBuilder.java From Flink-CEPplus with Apache License 2.0

4 votes

@VisibleForTesting
RocksDBKeyedStateBackendBuilder(
	String operatorIdentifier,
	ClassLoader userCodeClassLoader,
	File instanceBasePath,
	DBOptions dbOptions,
	Function<String, ColumnFamilyOptions> columnFamilyOptionsFactory,
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	LocalRecoveryConfig localRecoveryConfig,
	RocksDBStateBackend.PriorityQueueStateType priorityQueueStateType,
	TtlTimeProvider ttlTimeProvider,
	MetricGroup metricGroup,
	@Nonnull Collection<KeyedStateHandle> stateHandles,
	StreamCompressionDecorator keyGroupCompressionDecorator,
	RocksDB injectedTestDB,
	ColumnFamilyHandle injectedDefaultColumnFamilyHandle,
	CloseableRegistry cancelStreamRegistry) {
	this(
		operatorIdentifier,
		userCodeClassLoader,
		instanceBasePath,
		dbOptions,
		columnFamilyOptionsFactory,
		kvStateRegistry,
		keySerializer,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		localRecoveryConfig,
		priorityQueueStateType,
		ttlTimeProvider,
		metricGroup,
		stateHandles,
		keyGroupCompressionDecorator,
		cancelStreamRegistry
	);
	this.injectedTestDB = injectedTestDB;
	this.injectedDefaultColumnFamilyHandle = injectedDefaultColumnFamilyHandle;
}