org.apache.flink.runtime.state.SnappyStreamCompressionDecorator Java Examples

The following examples show how to use org.apache.flink.runtime.state.SnappyStreamCompressionDecorator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksDBFullRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #2
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #3
Source File: RocksDBFullRestoreOperation.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
Example #4
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #5
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #6
Source File: RocksDBSavepointWriter.java    From bravo with Apache License 2.0 5 votes vote down vote up 
public RocksDBSavepointWriter(int maxParallelism, int parallelism, Map<String, Integer> stateIdMapping,
		boolean compression, Path opStateDir, byte[] metaBytes) {

	this.maxParallelism = maxParallelism;
	this.parallelism = parallelism;
	this.stateIdMapping = stateIdMapping;
	this.opStateDir = opStateDir;
	this.metaBytes = metaBytes;
	this.keyGroupCompressionDecorator = compression ? SnappyStreamCompressionDecorator.INSTANCE
			: UncompressedStreamCompressionDecorator.INSTANCE;
}
Example #7
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
Example #8
Source File: StateMetadataUtils.java    From bravo with Apache License 2.0 4 votes vote down vote up 
public static StreamCompressionDecorator getCompressionDecorator(KeyedBackendSerializationProxy<?> proxy) {
	return proxy.isUsingKeyGroupCompression()
			? SnappyStreamCompressionDecorator.INSTANCE
			: UncompressedStreamCompressionDecorator.INSTANCE;
}