org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator Java Examples

The following examples show how to use org.apache.flink.runtime.state.UncompressedStreamCompressionDecorator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #2
Source File: RocksFullSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
Example #3
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #4
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
Example #5
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #6
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
Example #7
Source File: RocksDBTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
public static <K> RocksDBKeyedStateBackendBuilder<K> builderForTestDefaults(
		File instanceBasePath,
		TypeSerializer<K> keySerializer) {

	final RocksDBResourceContainer optionsContainer = new RocksDBResourceContainer();

	return new RocksDBKeyedStateBackendBuilder<>(
		"no-op",
		ClassLoader.getSystemClassLoader(),
		instanceBasePath,
		optionsContainer,
		stateName -> optionsContainer.getColumnOptions(),
		new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
		keySerializer,
		2,
		new KeyGroupRange(0, 1),
		new ExecutionConfig(),
		TestLocalRecoveryConfig.disabled(),
		RocksDBStateBackend.PriorityQueueStateType.HEAP,
		TtlTimeProvider.DEFAULT,
		new UnregisteredMetricsGroup(),
		Collections.emptyList(),
		UncompressedStreamCompressionDecorator.INSTANCE,
		new CloseableRegistry());
}
Example #8
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #9
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #10
Source File: RocksDBSavepointWriter.java    From bravo with Apache License 2.0 5 votes vote down vote up 
public RocksDBSavepointWriter(int maxParallelism, int parallelism, Map<String, Integer> stateIdMapping,
		boolean compression, Path opStateDir, byte[] metaBytes) {

	this.maxParallelism = maxParallelism;
	this.parallelism = parallelism;
	this.stateIdMapping = stateIdMapping;
	this.opStateDir = opStateDir;
	this.metaBytes = metaBytes;
	this.keyGroupCompressionDecorator = compression ? SnappyStreamCompressionDecorator.INSTANCE
			: UncompressedStreamCompressionDecorator.INSTANCE;
}
Example #11
Source File: RocksDBTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up 
public static <K> RocksDBKeyedStateBackendBuilder<K> builderForTestDB(
		File instanceBasePath,
		TypeSerializer<K> keySerializer,
		RocksDB db,
		ColumnFamilyHandle defaultCFHandle,
		ColumnFamilyOptions columnFamilyOptions) {

	final RocksDBResourceContainer optionsContainer = new RocksDBResourceContainer();

	return new RocksDBKeyedStateBackendBuilder<>(
			"no-op",
			ClassLoader.getSystemClassLoader(),
			instanceBasePath,
			optionsContainer,
			stateName -> columnFamilyOptions,
			new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID()),
			keySerializer,
			2,
			new KeyGroupRange(0, 1),
			new ExecutionConfig(),
			TestLocalRecoveryConfig.disabled(),
			RocksDBStateBackend.PriorityQueueStateType.HEAP,
			TtlTimeProvider.DEFAULT,
			new UnregisteredMetricsGroup(),
			Collections.emptyList(),
			UncompressedStreamCompressionDecorator.INSTANCE,
			db,
			defaultCFHandle,
			new CloseableRegistry());
}
Example #12
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #13
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 4 votes vote down vote up 
public static StreamCompressionDecorator getCompressionDecorator(KeyedBackendSerializationProxy<?> proxy) {
	return proxy.isUsingKeyGroupCompression()
			? SnappyStreamCompressionDecorator.INSTANCE
			: UncompressedStreamCompressionDecorator.INSTANCE;
}