org.apache.flink.runtime.state.StateSnapshotTransformer Java Examples

The following examples show how to use org.apache.flink.runtime.state.StateSnapshotTransformer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CopyOnWriteStateTableSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new {@link CopyOnWriteStateTableSnapshot}.
 *
 * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot.
 */
CopyOnWriteStateTableSnapshot(
	CopyOnWriteStateTable<K, N, S> owningStateTable,
	TypeSerializer<K> localKeySerializer,
	TypeSerializer<N> localNamespaceSerializer,
	TypeSerializer<S> localStateSerializer,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	super(owningStateTable,
		localKeySerializer,
		localNamespaceSerializer,
		localStateSerializer,
		stateSnapshotTransformer);

	this.keyGroupOffset = owningStateTable.getKeyGroupOffset();
	this.stateMapSnapshots = owningStateTable.getStateMapSnapshotList();
}
 
Example #2
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
void transform(@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	Preconditions.checkNotNull(stateSnapshotTransformer);
	int indexOfFirstChain = moveChainsToBackOfArray();
	int count = 0;
	// reuse the snapshotData to transform and flatten the entries.
	for (int i = indexOfFirstChain; i < snapshotData.length; i++) {
		CopyOnWriteStateMap.StateMapEntry<K, N, S> entry = snapshotData[i];
		while (entry != null) {
			S transformedValue = stateSnapshotTransformer.filterOrTransform(entry.state);
			if (transformedValue != null) {
				CopyOnWriteStateMap.StateMapEntry<K, N, S> filteredEntry = entry;
				if (transformedValue != entry.state) {
					filteredEntry = new CopyOnWriteStateMap.StateMapEntry<>(entry, entry.entryVersion);
					filteredEntry.state = transformedValue;
				}
				snapshotData[count++] = filteredEntry;
			}
			entry = entry.next;
		}
	}
	numberOfEntriesInSnapshotData = count;
}
 
Example #3
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), TtlStateFactory.class);
		throw new FlinkRuntimeException(message);
	}
	IS state = stateFactory.createInternalState(namespaceSerializer, stateDesc);
	stateSnapshotFilters.put(stateDesc.getName(),
		(StateSnapshotTransformer<Object>) getStateSnapshotTransformer(stateDesc, snapshotTransformFactory));
	((MockInternalKvState<K, N, SV>) state).values = () -> stateValues
		.computeIfAbsent(stateDesc.getName(), n -> new HashMap<>())
		.computeIfAbsent(getCurrentKey(), k -> new HashMap<>());
	return state;
}
 
Example #4
Source File: CopyOnWriteStateTableSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates a new {@link CopyOnWriteStateTableSnapshot}.
 *
 * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot.
 */
CopyOnWriteStateTableSnapshot(
	CopyOnWriteStateTable<K, N, S> owningStateTable,
	TypeSerializer<K> localKeySerializer,
	TypeSerializer<N> localNamespaceSerializer,
	TypeSerializer<S> localStateSerializer,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	super(owningStateTable,
		localKeySerializer,
		localNamespaceSerializer,
		localStateSerializer,
		stateSnapshotTransformer);

	this.keyGroupOffset = owningStateTable.getKeyGroupOffset();
	this.stateMapSnapshots = owningStateTable.getStateMapSnapshotList();
}
 
Example #5
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	Map<N, Map<K, S>> mappings = filterMappingsIfNeeded(owningStateMap.getNamespaceMap(), stateSnapshotTransformer);
	int numberOfEntries = countMappingsInKeyGroup(mappings);

	dov.writeInt(numberOfEntries);
	for (Map.Entry<N, Map<K, S>> namespaceEntry : mappings.entrySet()) {
		N namespace = namespaceEntry.getKey();
		for (Map.Entry<K, S> entry : namespaceEntry.getValue().entrySet()) {
			namespaceSerializer.serialize(namespace, dov);
			keySerializer.serialize(entry.getKey(), dov);
			stateSerializer.serialize(entry.getValue(), dov);
		}
	}
}
 
Example #6
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
private Map<N, Map<K, S>> filterMappingsIfNeeded(
	final Map<N, Map<K, S>> keyGroupMap,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	if (stateSnapshotTransformer == null) {
		return keyGroupMap;
	}

	Map<N, Map<K, S>> filtered = new HashMap<>();
	for (Map.Entry<N, Map<K, S>> namespaceEntry : keyGroupMap.entrySet()) {
		N namespace = namespaceEntry.getKey();
		Map<K, S> filteredNamespaceMap = filtered.computeIfAbsent(namespace, n -> new HashMap<>());
		for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
			K key = keyEntry.getKey();
			S transformedvalue = stateSnapshotTransformer.filterOrTransform(keyEntry.getValue());
			if (transformedvalue != null) {
				filteredNamespaceMap.put(key, transformedvalue);
			}
		}
		if (filteredNamespaceMap.isEmpty()) {
			filtered.remove(namespace);
		}
	}

	return filtered;
}
 
Example #7
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), TtlStateFactory.class);
		throw new FlinkRuntimeException(message);
	}
	IS state = stateFactory.createInternalState(namespaceSerializer, stateDesc);
	stateSnapshotFilters.put(stateDesc.getName(),
		(StateSnapshotTransformer<Object>) getStateSnapshotTransformer(stateDesc, snapshotTransformFactory));
	((MockInternalKvState<K, N, SV>) state).values = () -> stateValues
		.computeIfAbsent(stateDesc.getName(), n -> new HashMap<>())
		.computeIfAbsent(getCurrentKey(), k -> new HashMap<>());
	return state;
}
 
Example #8
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #9
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}
 
Example #10
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
MockKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	int numberOfKeyGroups,
	KeyGroupRange keyGroupRange,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters,
	CloseableRegistry cancelStreamRegistry) {
	super(kvStateRegistry, keySerializer, userCodeClassLoader,
		numberOfKeyGroups, keyGroupRange, executionConfig, ttlTimeProvider, cancelStreamRegistry);
	this.stateValues = stateValues;
	this.stateSnapshotFilters = stateSnapshotFilters;
}
 
Example #11
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #12
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	SnapshotIterator<K, N, S> snapshotIterator = stateSnapshotTransformer == null ?
		new NonTransformSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData) :
		new TransformedSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData, stateSnapshotTransformer);

	int size = snapshotIterator.size();
	dov.writeInt(size);
	while (snapshotIterator.hasNext()) {
		StateEntry<K, N, S> stateEntry = snapshotIterator.next();
		namespaceSerializer.serialize(stateEntry.getNamespace(), dov);
		keySerializer.serialize(stateEntry.getKey(), dov);
		stateSerializer.serialize(stateEntry.getState(), dov);
	}
}
 
Example #13
Source File: MockKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public MockKeyedStateBackend<K> build() {
	Map<String, Map<K, Map<Object, Object>>> stateValues = new HashMap<>();
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters = new HashMap<>();
	MockRestoreOperation<K> restoreOperation = new MockRestoreOperation<>(restoreStateHandles, stateValues);
	restoreOperation.restore();
	return new MockKeyedStateBackend<>(
		kvStateRegistry,
		keySerializerProvider.currentSchemaSerializer(),
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		stateValues,
		stateSnapshotFilters,
		cancelStreamRegistry,
		new InternalKeyContextImpl<>(
			keyGroupRange,
			numberOfKeyGroups
		));
}
 
Example #14
Source File: MockKeyedStateBackendBuilder.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public MockKeyedStateBackend<K> build() {
	Map<String, Map<K, Map<Object, Object>>> stateValues = new HashMap<>();
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters = new HashMap<>();
	MockRestoreOperation<K> restoreOperation = new MockRestoreOperation<>(restoreStateHandles, stateValues);
	restoreOperation.restore();
	return new MockKeyedStateBackend<>(
		kvStateRegistry,
		keySerializerProvider.currentSchemaSerializer(),
		userCodeClassLoader,
		executionConfig,
		ttlTimeProvider,
		stateValues,
		stateSnapshotFilters,
		cancelStreamRegistry,
		new InternalKeyContextImpl<>(
			keyGroupRange,
			numberOfKeyGroups
		));
}
 
Example #15
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}
 
Example #16
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private <SV, SEV> StateSnapshotTransformer<SV> getStateSnapshotTransformer(
	StateDescriptor<?, SV> stateDesc,
	StateSnapshotTransformFactory<SEV> snapshotTransformFactory) {
	Optional<StateSnapshotTransformer<SEV>> original = snapshotTransformFactory.createForDeserializedState();
	if (original.isPresent()) {
		if (stateDesc instanceof ListStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.ListStateSnapshotTransformer<>(original.get());
		} else if (stateDesc instanceof MapStateDescriptor) {
			return (StateSnapshotTransformer<SV>) new StateSnapshotTransformers.MapStateSnapshotTransformer<>(original.get());
		} else {
			return (StateSnapshotTransformer<SV>) original.get();
		}
	} else {
		return null;
	}
}
 
Example #17
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), TtlStateFactory.class);
		throw new FlinkRuntimeException(message);
	}
	IS state = stateFactory.createInternalState(namespaceSerializer, stateDesc);
	stateSnapshotFilters.put(stateDesc.getName(),
		(StateSnapshotTransformer<Object>) getStateSnapshotTransformer(stateDesc, snapshotTransformFactory));
	((MockInternalKvState<K, N, SV>) state).values = () -> stateValues
		.computeIfAbsent(stateDesc.getName(), n -> new HashMap<>())
		.computeIfAbsent(getCurrentKey(), k -> new HashMap<>());
	return state;
}
 
Example #18
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	SnapshotIterator<K, N, S> snapshotIterator = stateSnapshotTransformer == null ?
		new NonTransformSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData) :
		new TransformedSnapshotIterator<>(numberOfEntriesInSnapshotData, snapshotData, stateSnapshotTransformer);

	int size = snapshotIterator.size();
	dov.writeInt(size);
	while (snapshotIterator.hasNext()) {
		StateEntry<K, N, S> stateEntry = snapshotIterator.next();
		namespaceSerializer.serialize(stateEntry.getNamespace(), dov);
		keySerializer.serialize(stateEntry.getKey(), dov);
		stateSerializer.serialize(stateEntry.getState(), dov);
	}
}
 
Example #19
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
private Map<N, Map<K, S>> filterMappingsIfNeeded(
	final Map<N, Map<K, S>> keyGroupMap,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	if (stateSnapshotTransformer == null) {
		return keyGroupMap;
	}

	Map<N, Map<K, S>> filtered = new HashMap<>();
	for (Map.Entry<N, Map<K, S>> namespaceEntry : keyGroupMap.entrySet()) {
		N namespace = namespaceEntry.getKey();
		Map<K, S> filteredNamespaceMap = filtered.computeIfAbsent(namespace, n -> new HashMap<>());
		for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
			K key = keyEntry.getKey();
			S transformedvalue = stateSnapshotTransformer.filterOrTransform(keyEntry.getValue());
			if (transformedvalue != null) {
				filteredNamespaceMap.put(key, transformedvalue);
			}
		}
		if (filteredNamespaceMap.isEmpty()) {
			filtered.remove(namespace);
		}
	}

	return filtered;
}
 
Example #20
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
public void writeState(
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer,
	@Nonnull DataOutputView dov,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) throws IOException {
	Map<N, Map<K, S>> mappings = filterMappingsIfNeeded(owningStateMap.getNamespaceMap(), stateSnapshotTransformer);
	int numberOfEntries = countMappingsInKeyGroup(mappings);

	dov.writeInt(numberOfEntries);
	for (Map.Entry<N, Map<K, S>> namespaceEntry : mappings.entrySet()) {
		N namespace = namespaceEntry.getKey();
		for (Map.Entry<K, S> entry : namespaceEntry.getValue().entrySet()) {
			namespaceSerializer.serialize(namespace, dov);
			keySerializer.serialize(entry.getKey(), dov);
			stateSerializer.serialize(entry.getValue(), dov);
		}
	}
}
 
Example #21
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}
 
Example #22
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
void transform(@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	Preconditions.checkNotNull(stateSnapshotTransformer);
	int indexOfFirstChain = moveChainsToBackOfArray();
	int count = 0;
	// reuse the snapshotData to transform and flatten the entries.
	for (int i = indexOfFirstChain; i < snapshotData.length; i++) {
		CopyOnWriteStateMap.StateMapEntry<K, N, S> entry = snapshotData[i];
		while (entry != null) {
			S transformedValue = stateSnapshotTransformer.filterOrTransform(entry.state);
			if (transformedValue != null) {
				CopyOnWriteStateMap.StateMapEntry<K, N, S> filteredEntry = entry;
				if (transformedValue != entry.state) {
					filteredEntry = new CopyOnWriteStateMap.StateMapEntry<>(entry, entry.entryVersion);
					filteredEntry.state = transformedValue;
				}
				snapshotData[count++] = filteredEntry;
			}
			entry = entry.next;
		}
	}
	numberOfEntriesInSnapshotData = count;
}
 
Example #23
Source File: MockKeyedStateBackendBuilder.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
public MockKeyedStateBackend<K> build() {
	Map<String, Map<K, Map<Object, Object>>> stateValues = new HashMap<>();
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters = new HashMap<>();
	MockRestoreOperation<K> restoreOperation = new MockRestoreOperation<>(restoreStateHandles, stateValues);
	restoreOperation.restore();
	return new MockKeyedStateBackend<>(
		kvStateRegistry,
		keySerializerProvider.currentSchemaSerializer(),
		userCodeClassLoader,
		numberOfKeyGroups,
		keyGroupRange,
		executionConfig,
		ttlTimeProvider,
		stateValues,
		stateSnapshotFilters,
		cancelStreamRegistry);
}
 
Example #24
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 
Example #25
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
MockKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters,
	CloseableRegistry cancelStreamRegistry,
	InternalKeyContext<K> keyContext) {
	super(kvStateRegistry, keySerializer, userCodeClassLoader,
		executionConfig, ttlTimeProvider, cancelStreamRegistry, keyContext);
	this.stateValues = stateValues;
	this.stateSnapshotFilters = stateSnapshotFilters;
}
 
Example #26
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
MockKeyedStateBackend(
	TaskKvStateRegistry kvStateRegistry,
	TypeSerializer<K> keySerializer,
	ClassLoader userCodeClassLoader,
	ExecutionConfig executionConfig,
	TtlTimeProvider ttlTimeProvider,
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters,
	CloseableRegistry cancelStreamRegistry,
	InternalKeyContext<K> keyContext) {
	super(kvStateRegistry, keySerializer, userCodeClassLoader,
		executionConfig, ttlTimeProvider, cancelStreamRegistry, keyContext);
	this.stateValues = stateValues;
	this.stateSnapshotFilters = stateSnapshotFilters;
}
 
Example #27
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
SnapshotIterator(
	int numberOfEntriesInSnapshotData,
	CopyOnWriteStateMap.StateMapEntry<K, N, S>[] snapshotData,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	this.numberOfEntriesInSnapshotData = numberOfEntriesInSnapshotData;
	this.snapshotData = snapshotData;

	transform(stateSnapshotTransformer);
	this.chainIterator = getChainIterator();
	this.entryIterator = Collections.emptyIterator();
}
 
Example #28
Source File: NestedMapsStateTable.java    From flink with Apache License 2.0 5 votes vote down vote up
NestedMapsStateTableSnapshot(
	NestedMapsStateTable<K, N, S> owningTable,
	TypeSerializer<K> localKeySerializer,
	TypeSerializer<N> localNamespaceSerializer,
	TypeSerializer<S> localStateSerializer,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	super(owningTable,
		localKeySerializer,
		localNamespaceSerializer,
		localStateSerializer,
		stateSnapshotTransformer);
}
 
Example #29
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
static <K> Map<String, Map<K, Map<Object, Object>>> copy(
	Map<String, Map<K, Map<Object, Object>>> stateValues, Map<String, StateSnapshotTransformer<Object>> stateSnapshotFilters) {
	Map<String, Map<K, Map<Object, Object>>> snapshotStates = new HashMap<>();
	for (String stateName : stateValues.keySet()) {
		StateSnapshotTransformer<Object> stateSnapshotTransformer = stateSnapshotFilters.getOrDefault(stateName, null);
		Map<K, Map<Object, Object>> keyedValues = snapshotStates.computeIfAbsent(stateName, s -> new HashMap<>());
		for (K key : stateValues.get(stateName).keySet()) {
			Map<Object, Object> snapshotedValues = keyedValues.computeIfAbsent(key, s -> new HashMap<>());
			for (Object namespace : stateValues.get(stateName).get(key).keySet()) {
				copyEntry(stateValues, snapshotedValues, stateName, key, namespace, stateSnapshotTransformer);
			}
		}
	}
	return snapshotStates;
}
 
Example #30
Source File: AbstractStateTableSnapshot.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@link AbstractStateTableSnapshot} for and owned by the given table.
 *
 * @param owningStateTable the {@link StateTable} for which this object represents a snapshot.
 */
AbstractStateTableSnapshot(
	StateTable<K, N, S> owningStateTable,
	TypeSerializer<K> localKeySerializer,
	TypeSerializer<N> localNamespaceSerializer,
	TypeSerializer<S> localStateSerializer,
	@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	this.owningStateTable = Preconditions.checkNotNull(owningStateTable);
	this.localKeySerializer = Preconditions.checkNotNull(localKeySerializer);
	this.localNamespaceSerializer = Preconditions.checkNotNull(localNamespaceSerializer);
	this.localStateSerializer = Preconditions.checkNotNull(localStateSerializer);
	this.stateSnapshotTransformer = stateSnapshotTransformer;
}