Java Code Examples for org.apache.flink.runtime.state.StateSnapshotTransformer#filterOrTransform()

The following examples show how to use org.apache.flink.runtime.state.StateSnapshotTransformer#filterOrTransform() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}
Example 2
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
void transform(@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	Preconditions.checkNotNull(stateSnapshotTransformer);
	int indexOfFirstChain = moveChainsToBackOfArray();
	int count = 0;
	// reuse the snapshotData to transform and flatten the entries.
	for (int i = indexOfFirstChain; i < snapshotData.length; i++) {
		CopyOnWriteStateMap.StateMapEntry<K, N, S> entry = snapshotData[i];
		while (entry != null) {
			S transformedValue = stateSnapshotTransformer.filterOrTransform(entry.state);
			if (transformedValue != null) {
				CopyOnWriteStateMap.StateMapEntry<K, N, S> filteredEntry = entry;
				if (transformedValue != entry.state) {
					filteredEntry = new CopyOnWriteStateMap.StateMapEntry<>(entry, entry.entryVersion);
					filteredEntry.state = transformedValue;
				}
				snapshotData[count++] = filteredEntry;
			}
			entry = entry.next;
		}
	}
	numberOfEntriesInSnapshotData = count;
}
Example 3
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Map<N, Map<K, S>> filterMappingsIfNeeded(
	final Map<N, Map<K, S>> keyGroupMap,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	if (stateSnapshotTransformer == null) {
		return keyGroupMap;
	}

	Map<N, Map<K, S>> filtered = new HashMap<>();
	for (Map.Entry<N, Map<K, S>> namespaceEntry : keyGroupMap.entrySet()) {
		N namespace = namespaceEntry.getKey();
		Map<K, S> filteredNamespaceMap = filtered.computeIfAbsent(namespace, n -> new HashMap<>());
		for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
			K key = keyEntry.getKey();
			S transformedvalue = stateSnapshotTransformer.filterOrTransform(keyEntry.getValue());
			if (transformedvalue != null) {
				filteredNamespaceMap.put(key, transformedvalue);
			}
		}
		if (filteredNamespaceMap.isEmpty()) {
			filtered.remove(namespace);
		}
	}

	return filtered;
}
Example 4
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}
Example 5
Source File: CopyOnWriteSkipListStateMapTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up 
static <K, N, S> void verifySnapshotWithTransform(
	@Nonnull Map<N, Map<K, S>> referenceStates,
	@Nonnull CopyOnWriteSkipListStateMapSnapshot<K, N, S> snapshot,
	StateSnapshotTransformer<S> transformer,
	TypeSerializer<K> keySerializer,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<S> stateSerializer) throws IOException {
	ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos();
	DataOutputView outputView = new DataOutputViewStreamWrapper(outputStream);
	snapshot.writeState(keySerializer, namespaceSerializer, stateSerializer, outputView, transformer);

	Map<N, Map<K, S>> transformedStates = new HashMap<>();
	for (Map.Entry<N, Map<K, S>> namespaceEntry : referenceStates.entrySet()) {
		for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
			S state = transformer.filterOrTransform(keyEntry.getValue());
			if (state != null) {
				transformedStates.computeIfAbsent(namespaceEntry.getKey(), (none) -> new HashMap<>())
					.put(keyEntry.getKey(), state);
			}
		}
	}

	Map<N, Map<K, S>> actualStates = readStateFromSnapshot(
		outputStream.toByteArray(), keySerializer, namespaceSerializer, stateSerializer);
	assertEquals(transformedStates, actualStates);
}
Example 6
Source File: CopyOnWriteStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up 
@Override
void transform(@Nullable StateSnapshotTransformer<S> stateSnapshotTransformer) {
	Preconditions.checkNotNull(stateSnapshotTransformer);
	int indexOfFirstChain = moveChainsToBackOfArray();
	int count = 0;
	// reuse the snapshotData to transform and flatten the entries.
	for (int i = indexOfFirstChain; i < snapshotData.length; i++) {
		CopyOnWriteStateMap.StateMapEntry<K, N, S> entry = snapshotData[i];
		while (entry != null) {
			S transformedValue = stateSnapshotTransformer.filterOrTransform(entry.state);
			if (transformedValue != null) {
				CopyOnWriteStateMap.StateMapEntry<K, N, S> filteredEntry = entry;
				if (transformedValue != entry.state) {
					filteredEntry = new CopyOnWriteStateMap.StateMapEntry<>(entry, entry.entryVersion);
					filteredEntry.state = transformedValue;
				}
				snapshotData[count++] = filteredEntry;
			}
			entry = entry.next;
		}
	}
	numberOfEntriesInSnapshotData = count;
}
Example 7
Source File: NestedStateMapSnapshot.java    From flink with Apache License 2.0 6 votes vote down vote up 
private Map<N, Map<K, S>> filterMappingsIfNeeded(
	final Map<N, Map<K, S>> keyGroupMap,
	StateSnapshotTransformer<S> stateSnapshotTransformer) {
	if (stateSnapshotTransformer == null) {
		return keyGroupMap;
	}

	Map<N, Map<K, S>> filtered = new HashMap<>();
	for (Map.Entry<N, Map<K, S>> namespaceEntry : keyGroupMap.entrySet()) {
		N namespace = namespaceEntry.getKey();
		Map<K, S> filteredNamespaceMap = filtered.computeIfAbsent(namespace, n -> new HashMap<>());
		for (Map.Entry<K, S> keyEntry : namespaceEntry.getValue().entrySet()) {
			K key = keyEntry.getKey();
			S transformedvalue = stateSnapshotTransformer.filterOrTransform(keyEntry.getValue());
			if (transformedvalue != null) {
				filteredNamespaceMap.put(key, transformedvalue);
			}
		}
		if (filteredNamespaceMap.isEmpty()) {
			filtered.remove(namespace);
		}
	}

	return filtered;
}
Example 8
Source File: MockKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("unchecked")
private static <K> void copyEntry(
	Map<String, Map<K, Map<Object, Object>>> stateValues,
	Map<Object, Object> snapshotedValues,
	String stateName,
	K key,
	Object namespace,
	StateSnapshotTransformer<Object> stateSnapshotTransformer) {
	Object value = stateValues.get(stateName).get(key).get(namespace);
	value = value instanceof List ? new ArrayList<>((List) value) : value;
	value = value instanceof Map ? new HashMap<>((Map) value) : value;
	Object filteredValue = stateSnapshotTransformer == null ? value : stateSnapshotTransformer.filterOrTransform(value);
	if (filteredValue != null) {
		snapshotedValues.put(namespace, filteredValue);
	}
}