Java Code Examples for org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot

The following examples show how to use org.apache.flink.runtime.state.metainfo.StateMetaInfoSnapshot. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example 2
private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
	columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
	columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);

	Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();

	LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
		operatorIdentifier, backendUID);

	if (!instanceRocksDBPath.mkdirs()) {
		String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
		LOG.error(errMsg);
		throw new IOException(errMsg);
	}

	restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);

	openDB();

	registerColumnFamilyHandles(stateMetaInfoSnapshots);
}
 
Example 3
@Nonnull
private StateMetaInfoSnapshot computeSnapshot() {
	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateType.toString());
	Map<String, TypeSerializer<?>> serializerMap = new HashMap<>(2);
	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap = new HashMap<>(2);
	String namespaceSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString();
	String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();

	TypeSerializer<N> namespaceSerializer = getNamespaceSerializer();
	serializerMap.put(namespaceSerializerKey, namespaceSerializer.duplicate());
	serializerConfigSnapshotsMap.put(namespaceSerializerKey, namespaceSerializer.snapshotConfiguration());

	TypeSerializer<S> stateSerializer = getStateSerializer();
	serializerMap.put(valueSerializerKey, stateSerializer.duplicate());
	serializerConfigSnapshotsMap.put(valueSerializerKey, stateSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotsMap,
		serializerMap);
}
 
Example 4
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example 5
@Nonnull
private StateMetaInfoSnapshot computeSnapshot() {
	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE.toString(),
		assignmentMode.toString());
	String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();

	TypeSerializer<S> partitionStateSerializer = getPartitionStateSerializer();
	Map<String, TypeSerializer<?>> serializerMap =
		Collections.singletonMap(valueSerializerKey, partitionStateSerializer.duplicate());
	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap =
		Collections.singletonMap(valueSerializerKey, partitionStateSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.OPERATOR,
		optionsMap,
		serializerConfigSnapshotsMap,
		serializerMap);
}
 
Example 6
Source Project: Flink-CEPplus   Source File: HeapSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
 
Example 7
Source Project: flink   Source File: RocksIncrementalSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example 8
Source Project: flink   Source File: RocksIncrementalSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Nonnull
@Override
protected RunnableFuture<SnapshotResult<KeyedStateHandle>> doSnapshot(
	long checkpointId,
	long checkpointTimestamp,
	@Nonnull CheckpointStreamFactory checkpointStreamFactory,
	@Nonnull CheckpointOptions checkpointOptions) throws Exception {

	final SnapshotDirectory snapshotDirectory = prepareLocalSnapshotDirectory(checkpointId);
	LOG.trace("Local RocksDB checkpoint goes to backup path {}.", snapshotDirectory);

	final List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = new ArrayList<>(kvStateInformation.size());
	final Set<StateHandleID> baseSstFiles = snapshotMetaData(checkpointId, stateMetaInfoSnapshots);

	takeDBNativeCheckpoint(snapshotDirectory);

	final RocksDBIncrementalSnapshotOperation snapshotOperation =
		new RocksDBIncrementalSnapshotOperation(
			checkpointId,
			checkpointStreamFactory,
			snapshotDirectory,
			baseSstFiles,
			stateMetaInfoSnapshots);

	return snapshotOperation.toAsyncSnapshotFutureTask(cancelStreamRegistry);
}
 
Example 9
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example 10
Source Project: Flink-CEPplus   Source File: RegisteredStateMetaInfoBase.java    License: Apache License 2.0 6 votes vote down vote up
public static RegisteredStateMetaInfoBase fromMetaInfoSnapshot(@Nonnull StateMetaInfoSnapshot snapshot) {

		final StateMetaInfoSnapshot.BackendStateType backendStateType = snapshot.getBackendStateType();
		switch (backendStateType) {
			case KEY_VALUE:
				return new RegisteredKeyValueStateBackendMetaInfo<>(snapshot);
			case OPERATOR:
				return new RegisteredOperatorStateBackendMetaInfo<>(snapshot);
			case BROADCAST:
				return new RegisteredBroadcastStateBackendMetaInfo<>(snapshot);
			case PRIORITY_QUEUE:
				return new RegisteredPriorityQueueStateBackendMetaInfo<>(snapshot);
			default:
				throw new IllegalArgumentException("Unknown backend state type: " + backendStateType);
		}
	}
 
Example 11
Source Project: bravo   Source File: RocksDBCheckpointIterator.java    License: Apache License 2.0 6 votes vote down vote up
public RocksDBCheckpointIterator(IncrementalKeyedStateHandle handle, FilterFunction<String> stateFilter,
		String localPath) {
	this.localPath = localPath;
	this.cancelStreamRegistry = new CloseableRegistry();
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = StateMetadataUtils
			.getKeyedBackendSerializationProxy(handle.getMetaStateHandle()).getStateMetaInfoSnapshots();

	stateColumnFamilyHandles = new ArrayList<>(stateMetaInfoSnapshots.size() + 1);
	List<ColumnFamilyDescriptor> stateColumnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(
			stateMetaInfoSnapshots);
	try {
		transferAllStateDataToDirectory(handle, new Path(localPath));
		this.db = openDB(localPath, stateColumnFamilyDescriptors, stateColumnFamilyHandles);
		createColumnIterators(stateFilter, stateMetaInfoSnapshots);
	} catch (Exception e) {
		throw new IllegalStateException(e);
	}
}
 
Example 12
private StateMetaInfoSnapshot computeSnapshot() {
	TypeSerializer<T> elementSerializer = getElementSerializer();
	Map<String, TypeSerializer<?>> serializerMap =
		Collections.singletonMap(
			StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
			elementSerializer.duplicate());
	Map<String, TypeSerializerSnapshot<?>> serializerSnapshotMap =
		Collections.singletonMap(
			StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
			elementSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE,
		Collections.emptyMap(),
		serializerSnapshotMap,
		serializerMap);
}
 
Example 13
Source Project: flink   Source File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	boolean registerTtlCompactFilter) {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(stateMetaInfoSnapshots.size());

	for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
		RegisteredStateMetaInfoBase metaInfoBase =
			RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
		ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(
			metaInfoBase, columnFamilyOptionsFactory, registerTtlCompactFilter ? ttlCompactFiltersManager : null);
		columnFamilyDescriptors.add(columnFamilyDescriptor);
	}
	return columnFamilyDescriptors;
}
 
Example 14
@Nonnull
private StateMetaInfoSnapshot computeSnapshot() {
	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.OPERATOR_STATE_DISTRIBUTION_MODE.toString(),
		assignmentMode.toString());
	String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();

	TypeSerializer<S> partitionStateSerializer = getPartitionStateSerializer();
	Map<String, TypeSerializer<?>> serializerMap =
		Collections.singletonMap(valueSerializerKey, partitionStateSerializer.duplicate());
	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap =
		Collections.singletonMap(valueSerializerKey, partitionStateSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.OPERATOR,
		optionsMap,
		serializerConfigSnapshotsMap,
		serializerMap);
}
 
Example 15
@Nonnull
private StateMetaInfoSnapshot computeSnapshot() {
	Map<String, String> optionsMap = Collections.singletonMap(
		StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE.toString(),
		stateType.toString());
	Map<String, TypeSerializer<?>> serializerMap = new HashMap<>(2);
	Map<String, TypeSerializerSnapshot<?>> serializerConfigSnapshotsMap = new HashMap<>(2);
	String namespaceSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER.toString();
	String valueSerializerKey = StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString();

	TypeSerializer<N> namespaceSerializer = getNamespaceSerializer();
	serializerMap.put(namespaceSerializerKey, namespaceSerializer.duplicate());
	serializerConfigSnapshotsMap.put(namespaceSerializerKey, namespaceSerializer.snapshotConfiguration());

	TypeSerializer<S> stateSerializer = getStateSerializer();
	serializerMap.put(valueSerializerKey, stateSerializer.duplicate());
	serializerConfigSnapshotsMap.put(valueSerializerKey, stateSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.KEY_VALUE,
		optionsMap,
		serializerConfigSnapshotsMap,
		serializerMap);
}
 
Example 16
Source Project: Flink-CEPplus   Source File: SerializationProxiesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example 17
Source Project: flink   Source File: RocksDBFullRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Restore the KV-state / ColumnFamily meta data for all key-groups referenced by the current state handle.
 */
private void restoreKVStateMetaData() throws IOException, StateMigrationException {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(currentStateHandleInView);

	this.keygroupStreamCompressionDecorator = serializationProxy.isUsingKeyGroupCompression() ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	List<StateMetaInfoSnapshot> restoredMetaInfos =
		serializationProxy.getStateMetaInfoSnapshots();
	currentStateHandleKVStateColumnFamilies = new ArrayList<>(restoredMetaInfos.size());

	for (StateMetaInfoSnapshot restoredMetaInfo : restoredMetaInfos) {
		RocksDbKvStateInfo registeredStateCFHandle =
			getOrRegisterStateColumnFamilyHandle(null, restoredMetaInfo);
		currentStateHandleKVStateColumnFamilies.add(registeredStateCFHandle.columnFamilyHandle);
	}
}
 
Example 18
Source Project: flink   Source File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
private void restoreFromLocalState(IncrementalLocalKeyedStateHandle localKeyedStateHandle) throws Exception {
	KeyedBackendSerializationProxy<K> serializationProxy = readMetaData(localKeyedStateHandle.getMetaDataState());
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots = serializationProxy.getStateMetaInfoSnapshots();
	columnFamilyDescriptors = createAndRegisterColumnFamilyDescriptors(stateMetaInfoSnapshots, true);
	columnFamilyHandles = new ArrayList<>(columnFamilyDescriptors.size() + 1);

	Path restoreSourcePath = localKeyedStateHandle.getDirectoryStateHandle().getDirectory();

	LOG.debug("Restoring keyed backend uid in operator {} from incremental snapshot to {}.",
		operatorIdentifier, backendUID);

	if (!instanceRocksDBPath.mkdirs()) {
		String errMsg = "Could not create RocksDB data directory: " + instanceBasePath.getAbsolutePath();
		LOG.error(errMsg);
		throw new IOException(errMsg);
	}

	restoreInstanceDirectoryFromPath(restoreSourcePath, dbPath);

	openDB();

	registerColumnFamilyHandles(stateMetaInfoSnapshots);
}
 
Example 19
Source Project: flink   Source File: RocksDBIncrementalRestoreOperation.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method recreates and registers all {@link ColumnFamilyDescriptor} from Flink's state meta data snapshot.
 */
private List<ColumnFamilyDescriptor> createAndRegisterColumnFamilyDescriptors(
	List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
	boolean registerTtlCompactFilter) {

	List<ColumnFamilyDescriptor> columnFamilyDescriptors =
		new ArrayList<>(stateMetaInfoSnapshots.size());

	for (StateMetaInfoSnapshot stateMetaInfoSnapshot : stateMetaInfoSnapshots) {
		RegisteredStateMetaInfoBase metaInfoBase =
			RegisteredStateMetaInfoBase.fromMetaInfoSnapshot(stateMetaInfoSnapshot);
		ColumnFamilyDescriptor columnFamilyDescriptor = RocksDBOperationUtils.createColumnFamilyDescriptor(
			metaInfoBase, columnFamilyOptionsFactory, registerTtlCompactFilter ? ttlCompactFiltersManager : null);
		columnFamilyDescriptors.add(columnFamilyDescriptor);
	}
	return columnFamilyDescriptors;
}
 
Example 20
Source Project: flink   Source File: RocksIncrementalSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private Set<StateHandleID> snapshotMetaData(
	long checkpointId,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	final long lastCompletedCheckpoint;
	final Set<StateHandleID> baseSstFiles;

	// use the last completed checkpoint as the comparison base.
	synchronized (materializedSstFiles) {
		lastCompletedCheckpoint = lastCompletedCheckpointId;
		baseSstFiles = materializedSstFiles.get(lastCompletedCheckpoint);
	}
	LOG.trace("Taking incremental snapshot for checkpoint {}. Snapshot is based on last completed checkpoint {} " +
		"assuming the following (shared) files as base: {}.", checkpointId, lastCompletedCheckpoint, baseSstFiles);

	// snapshot meta data to save
	for (Map.Entry<String, RocksDbKvStateInfo> stateMetaInfoEntry : kvStateInformation.entrySet()) {
		stateMetaInfoSnapshots.add(stateMetaInfoEntry.getValue().metaInfo.snapshot());
	}
	return baseSstFiles;
}
 
Example 21
Source Project: flink   Source File: SerializationProxiesTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testKeyedStateMetaInfoSerialization() throws Exception {

	String name = "test";
	TypeSerializer<?> namespaceSerializer = LongSerializer.INSTANCE;
	TypeSerializer<?> stateSerializer = DoubleSerializer.INSTANCE;

	StateMetaInfoSnapshot metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(
		StateDescriptor.Type.VALUE, name, namespaceSerializer, stateSerializer).snapshot();

	byte[] serialized;
	try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) {
		StateMetaInfoSnapshotReadersWriters.getWriter().
			writeStateMetaInfoSnapshot(metaInfo, new DataOutputViewStreamWrapper(out));
		serialized = out.toByteArray();
	}

	try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) {
		final StateMetaInfoReader reader = StateMetaInfoSnapshotReadersWriters.getReader(
			CURRENT_STATE_META_INFO_SNAPSHOT_VERSION, StateMetaInfoSnapshotReadersWriters.StateTypeHint.KEYED_STATE);
		metaInfo = reader.readStateMetaInfoSnapshot(
			new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader());
	}

	Assert.assertEquals(name, metaInfo.getName());
}
 
Example 22
Source Project: flink   Source File: HeapSnapshotStrategy.java    License: Apache License 2.0 6 votes vote down vote up
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
 
Example 23
private StateMetaInfoSnapshot computeSnapshot() {
	TypeSerializer<T> elementSerializer = getElementSerializer();
	Map<String, TypeSerializer<?>> serializerMap =
		Collections.singletonMap(
			StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
			elementSerializer.duplicate());
	Map<String, TypeSerializerSnapshot<?>> serializerSnapshotMap =
		Collections.singletonMap(
			StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER.toString(),
			elementSerializer.snapshotConfiguration());

	return new StateMetaInfoSnapshot(
		name,
		StateMetaInfoSnapshot.BackendStateType.PRIORITY_QUEUE,
		Collections.emptyMap(),
		serializerSnapshotMap,
		serializerMap);
}
 
Example 24
Source Project: flink   Source File: HeapRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		// always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle related
		kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
	}
}
 
Example 25
Source Project: flink   Source File: OperatorBackendSerializationProxy.java    License: Apache License 2.0 5 votes vote down vote up
private void writeStateMetaInfoSnapshots(
	List<StateMetaInfoSnapshot> snapshots,
	DataOutputView out) throws IOException {
	out.writeShort(snapshots.size());
	for (StateMetaInfoSnapshot state : snapshots) {
		StateMetaInfoSnapshotReadersWriters.getWriter().writeStateMetaInfoSnapshot(state, out);
	}
}
 
Example 26
private RocksDBIncrementalSnapshotOperation(
	long checkpointId,
	@Nonnull CheckpointStreamFactory checkpointStreamFactory,
	@Nonnull SnapshotDirectory localBackupDirectory,
	@Nullable Set<StateHandleID> baseSstFiles,
	@Nonnull List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) {

	this.checkpointStreamFactory = checkpointStreamFactory;
	this.baseSstFiles = baseSstFiles;
	this.checkpointId = checkpointId;
	this.localBackupDirectory = localBackupDirectory;
	this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: HeapRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
private void readStateHandleStateData(
	FSDataInputStream fsDataInputStream,
	DataInputViewStreamWrapper inView,
	KeyGroupRangeOffsets keyGroupOffsets,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int numStates,
	int readVersion,
	boolean isCompressed) throws IOException {

	final StreamCompressionDecorator streamCompressionDecorator = isCompressed ?
		SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE;

	for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) {
		int keyGroupIndex = groupOffset.f0;
		long offset = groupOffset.f1;

		// Check that restored key groups all belong to the backend.
		Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend.");

		fsDataInputStream.seek(offset);

		int writtenKeyGroupIndex = inView.readInt();
		Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex,
			"Unexpected key-group in restore.");

		try (InputStream kgCompressionInStream =
				 streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) {

			readKeyGroupStateData(
				kgCompressionInStream,
				kvStatesById,
				keyGroupIndex,
				numStates,
				readVersion);
		}
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: HeapRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
private void readKeyGroupStateData(
	InputStream inputStream,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int keyGroupIndex,
	int numStates,
	int readVersion) throws IOException {

	DataInputViewStreamWrapper inView =
		new DataInputViewStreamWrapper(inputStream);

	for (int i = 0; i < numStates; i++) {

		final int kvStateId = inView.readShort();
		final StateMetaInfoSnapshot stateMetaInfoSnapshot = kvStatesById.get(kvStateId);
		final StateSnapshotRestore registeredState;

		switch (stateMetaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(stateMetaInfoSnapshot.getName());
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(stateMetaInfoSnapshot.getName());
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					stateMetaInfoSnapshot.getBackendStateType() + ".");
		}

		StateSnapshotKeyGroupReader keyGroupReader = registeredState.keyGroupReader(readVersion);
		keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
	}
}
 
Example 29
@SuppressWarnings("unchecked")
public RegisteredKeyValueStateBackendMetaInfo(@Nonnull StateMetaInfoSnapshot snapshot) {
	this(
		StateDescriptor.Type.valueOf(snapshot.getOption(StateMetaInfoSnapshot.CommonOptionsKeys.KEYED_STATE_TYPE)),
		snapshot.getName(),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<N>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.NAMESPACE_SERIALIZER))),
		StateSerializerProvider.fromPreviousSerializerSnapshot(
			(TypeSerializerSnapshot<S>) Preconditions.checkNotNull(
				snapshot.getTypeSerializerSnapshot(StateMetaInfoSnapshot.CommonSerializerKeys.VALUE_SERIALIZER))),
		StateSnapshotTransformFactory.noTransform());

	Preconditions.checkState(StateMetaInfoSnapshot.BackendStateType.KEY_VALUE == snapshot.getBackendStateType());
}
 
Example 30
Source Project: flink   Source File: KeyedBackendSerializationProxy.java    License: Apache License 2.0 5 votes vote down vote up
public KeyedBackendSerializationProxy(
		TypeSerializer<K> keySerializer,
		List<StateMetaInfoSnapshot> stateMetaInfoSnapshots,
		boolean compression) {

	this.usingKeyGroupCompression = compression;

	this.keySerializer = Preconditions.checkNotNull(keySerializer);
	this.keySerializerSnapshot = Preconditions.checkNotNull(keySerializer.snapshotConfiguration());

	Preconditions.checkNotNull(stateMetaInfoSnapshots);
	Preconditions.checkArgument(stateMetaInfoSnapshots.size() <= Short.MAX_VALUE);
	this.stateMetaInfoSnapshots = stateMetaInfoSnapshots;
}