org.apache.flink.runtime.state.StateSnapshotRestore Java Examples

The following examples show how to use org.apache.flink.runtime.state.StateSnapshotRestore. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HeapSnapshotStrategy.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #2
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #3
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void processSnapshotMetaInfoForAllStates(
	List<StateMetaInfoSnapshot> metaInfoSnapshots,
	Map<StateUID, StateSnapshot> cowStateStableSnapshots,
	Map<StateUID, Integer> stateNamesToId,
	Map<String, ? extends StateSnapshotRestore> registeredStates,
	StateMetaInfoSnapshot.BackendStateType stateType) {

	for (Map.Entry<String, ? extends StateSnapshotRestore> kvState : registeredStates.entrySet()) {
		final StateUID stateUid = StateUID.of(kvState.getKey(), stateType);
		stateNamesToId.put(stateUid, stateNamesToId.size());
		StateSnapshotRestore state = kvState.getValue();
		if (null != state) {
			final StateSnapshot stateSnapshot = state.stateSnapshot();
			metaInfoSnapshots.add(stateSnapshot.getMetaInfoSnapshot());
			cowStateStableSnapshots.put(stateUid, stateSnapshot);
		}
	}
}
Example #4
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	if (!registeredKVStates.containsKey(state)) {
		return Stream.empty();
	}

	final StateSnapshotRestore stateSnapshotRestore = registeredKVStates.get(state);
	StateTable<K, N, ?> table = (StateTable<K, N, ?>) stateSnapshotRestore;
	return table.getKeys(namespace);
}
Example #5
Source File: HeapKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the total number of state entries across all keys/namespaces.
 */
@VisibleForTesting
@SuppressWarnings("unchecked")
@Override
public int numKeyValueStateEntries() {
	int sum = 0;
	for (StateSnapshotRestore state : registeredKVStates.values()) {
		sum += ((StateTable<?, ?, ?>) state).size();
	}
	return sum;
}
Example #6
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(backend, registeredKeyedBackendStateMetaInfo));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
Example #7
Source File: HeapRestoreOperation.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private void readKeyGroupStateData(
	InputStream inputStream,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int keyGroupIndex,
	int numStates,
	int readVersion) throws IOException {

	DataInputViewStreamWrapper inView =
		new DataInputViewStreamWrapper(inputStream);

	for (int i = 0; i < numStates; i++) {

		final int kvStateId = inView.readShort();
		final StateMetaInfoSnapshot stateMetaInfoSnapshot = kvStatesById.get(kvStateId);
		final StateSnapshotRestore registeredState;

		switch (stateMetaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(stateMetaInfoSnapshot.getName());
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(stateMetaInfoSnapshot.getName());
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					stateMetaInfoSnapshot.getBackendStateType() + ".");
		}

		StateSnapshotKeyGroupReader keyGroupReader = registeredState.keyGroupReader(readVersion);
		keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
	}
}
Example #8
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	if (!registeredKVStates.containsKey(state)) {
		return Stream.empty();
	}

	final StateSnapshotRestore stateSnapshotRestore = registeredKVStates.get(state);
	StateTable<K, N, ?> table = (StateTable<K, N, ?>) stateSnapshotRestore;
	return table.getKeys(namespace);
}
Example #9
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the total number of state entries across all keys/namespaces.
 */
@VisibleForTesting
@SuppressWarnings("unchecked")
@Override
public int numKeyValueStateEntries() {
	int sum = 0;
	for (StateSnapshotRestore state : registeredKVStates.values()) {
		sum += ((StateTable<?, ?, ?>) state).size();
	}
	return sum;
}
Example #10
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
Example #11
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readKeyGroupStateData(
	InputStream inputStream,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int keyGroupIndex,
	int numStates,
	int readVersion) throws IOException {

	DataInputViewStreamWrapper inView =
		new DataInputViewStreamWrapper(inputStream);

	for (int i = 0; i < numStates; i++) {

		final int kvStateId = inView.readShort();
		final StateMetaInfoSnapshot stateMetaInfoSnapshot = kvStatesById.get(kvStateId);
		final StateSnapshotRestore registeredState;

		switch (stateMetaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(stateMetaInfoSnapshot.getName());
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(stateMetaInfoSnapshot.getName());
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					stateMetaInfoSnapshot.getBackendStateType() + ".");
		}

		StateSnapshotKeyGroupReader keyGroupReader = registeredState.keyGroupReader(readVersion);
		keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
	}
}
Example #12
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	if (!registeredKVStates.containsKey(state)) {
		return Stream.empty();
	}

	final StateSnapshotRestore stateSnapshotRestore = registeredKVStates.get(state);
	StateTable<K, N, ?> table = (StateTable<K, N, ?>) stateSnapshotRestore;
	return table.getKeys(namespace);
}
Example #13
Source File: HeapKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Returns the total number of state entries across all keys/namespaces.
 */
@VisibleForTesting
@SuppressWarnings("unchecked")
@Override
public int numKeyValueStateEntries() {
	int sum = 0;
	for (StateSnapshotRestore state : registeredKVStates.values()) {
		sum += ((StateTable<?, ?, ?>) state).size();
	}
	return sum;
}
Example #14
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		// always put metaInfo into kvStatesById, because kvStatesById is KeyGroupsStateHandle related
		kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
	}
}
Example #15
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up 
private void readKeyGroupStateData(
	InputStream inputStream,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById,
	int keyGroupIndex,
	int numStates,
	int readVersion) throws IOException {

	DataInputViewStreamWrapper inView =
		new DataInputViewStreamWrapper(inputStream);

	for (int i = 0; i < numStates; i++) {

		final int kvStateId = inView.readShort();
		final StateMetaInfoSnapshot stateMetaInfoSnapshot = kvStatesById.get(kvStateId);
		final StateSnapshotRestore registeredState;

		switch (stateMetaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(stateMetaInfoSnapshot.getName());
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(stateMetaInfoSnapshot.getName());
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					stateMetaInfoSnapshot.getBackendStateType() + ".");
		}

		StateSnapshotKeyGroupReader keyGroupReader = registeredState.keyGroupReader(readVersion);
		keyGroupReader.readMappingsInKeyGroup(inView, keyGroupIndex);
	}
}