org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo Java Examples

The following examples show how to use org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateTable.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * @param keyContext    the key context provides the key scope for all put/get/delete operations.
 * @param metaInfo      the meta information, including the type serializer for state copy-on-write.
 * @param keySerializer the serializer of the key.
 */
public StateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo,
	TypeSerializer<K> keySerializer) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.metaInfo = Preconditions.checkNotNull(metaInfo);
	this.keySerializer = Preconditions.checkNotNull(keySerializer);

	this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();

	@SuppressWarnings("unchecked")
	StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
	this.keyGroupedStateMaps = state;
	for (int i = 0; i < this.keyGroupedStateMaps.length; i++) {
		this.keyGroupedStateMaps[i] = createStateMap();
	}
}
 
Example #2
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #3
Source File: StateTable.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * @param keyContext    the key context provides the key scope for all put/get/delete operations.
 * @param metaInfo      the meta information, including the type serializer for state copy-on-write.
 * @param keySerializer the serializer of the key.
 */
public StateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo,
	TypeSerializer<K> keySerializer) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.metaInfo = Preconditions.checkNotNull(metaInfo);
	this.keySerializer = Preconditions.checkNotNull(keySerializer);

	this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();

	@SuppressWarnings("unchecked")
	StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
	this.keyGroupedStateMaps = state;
	for (int i = 0; i < this.keyGroupedStateMaps.length; i++) {
		this.keyGroupedStateMaps[i] = createStateMap();
	}
}
 
Example #4
Source File: CopyOnWriteStateTableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private CopyOnWriteStateTable<Integer, Integer, Float> createStateTableForSnapshotRelease(int numberOfKeyGroups) {
	RegisteredKeyValueStateBackendMetaInfo<Integer, Float> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			IntSerializer.INSTANCE,
			FloatSerializer.INSTANCE);

	MockInternalKeyContext<Integer> mockKeyContext =
		new MockInternalKeyContext<>(0, numberOfKeyGroups - 1, numberOfKeyGroups);
	CopyOnWriteStateTable<Integer, Integer, Float> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, IntSerializer.INSTANCE);

	ThreadLocalRandom random = ThreadLocalRandom.current();
	for (int i = 0; i < 1000; i++) {
		mockKeyContext.setCurrentKeyAndKeyGroup(i);
		table.put(random.nextInt(), random.nextFloat());
	}

	return table;
}
 
Example #5
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #6
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example #7
Source File: RocksDBKeyedStateBackend.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example #8
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example #9
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example #10
Source File: RocksDBMapState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #11
Source File: RocksDBMapState.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #12
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBListState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(),
		(List<E>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #13
Source File: HeapRestoreOperation.java    From flink with Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(
							keyContext,
							registeredKeyedBackendStateMetaInfo,
							keySerializerProvider.currentSchemaSerializer()));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
 
Example #14
Source File: NestedMapsStateTable.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@link NestedMapsStateTable} for the given key context and meta info.
 * @param keyContext the key context.
 * @param metaInfo the meta information for this state table.
 * @param keySerializer the serializer of the key.
 */
public NestedMapsStateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo,
	TypeSerializer<K> keySerializer) {
	super(keyContext, metaInfo, keySerializer);
}
 
Example #15
Source File: RocksDBReducingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBReducingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((ReducingStateDescriptor<SV>) stateDesc).getReduceFunction(),
		backend);
}
 
Example #16
Source File: RocksFullSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
private static List<MetaData> fillMetaData(
	List<RocksDbKvStateInfo> metaDataCopy) {
	List<MetaData> metaData = new ArrayList<>(metaDataCopy.size());
	for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) {
		StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null;
		if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) {
			stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo).
				getStateSnapshotTransformFactory().createForSerializedState().orElse(null);
		}
		metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer));
	}
	return metaData;
}
 
Example #17
Source File: CopyOnWriteStateTableTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * This test triggers incremental rehash and tests for corruptions.
 */
@Test
public void testIncrementalRehash() {
	RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.UNKNOWN,
			"test",
			IntSerializer.INSTANCE,
			new ArrayListSerializer<>(IntSerializer.INSTANCE)); // we use mutable state objects.

	final MockInternalKeyContext<Integer> keyContext = new MockInternalKeyContext<>(IntSerializer.INSTANCE);

	final CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> stateTable =
		new CopyOnWriteStateTable<>(keyContext, metaInfo);

	int insert = 0;
	int remove = 0;
	while (!stateTable.isRehashing()) {
		stateTable.put(insert++, 0, new ArrayList<Integer>());
		if (insert % 8 == 0) {
			stateTable.remove(remove++, 0);
		}
	}
	Assert.assertEquals(insert - remove, stateTable.size());
	while (stateTable.isRehashing()) {
		stateTable.put(insert++, 0, new ArrayList<Integer>());
		if (insert % 8 == 0) {
			stateTable.remove(remove++, 0);
		}
	}
	Assert.assertEquals(insert - remove, stateTable.size());

	for (int i = 0; i < insert; ++i) {
		if (i < remove) {
			Assert.assertFalse(stateTable.containsKey(i, 0));
		} else {
			Assert.assertTrue(stateTable.containsKey(i, 0));
		}
	}
}
 
Example #18
Source File: AsyncSnapshotStrategySynchronicityBehavior.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public <N, V> StateTable<K, N, V> newStateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo,
	TypeSerializer<K> keySerializer) {
	return new CopyOnWriteStateTable<>(keyContext, newMetaInfo, keySerializer);
}
 
Example #19
Source File: RocksDBAggregatingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBAggregatingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((AggregatingStateDescriptor<?, SV, ?>) stateDesc).getAggregateFunction(),
		backend);
}
 
Example #20
Source File: RocksDBValueState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBValueState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		backend);
}
 
Example #21
Source File: OperatorStateWriter.java    From bravo with Apache License 2.0 5 votes vote down vote up
/**
 * Defines/redefines a value state with the given name and type. This can be
 * used to create new states of an operator or change the type of an already
 * existing state.
 * <p>
 * When redefining a pre-existing state make sure you haven't added that as
 * keyed state rows before.
 * 
 * @param stateName
 * @param newState
 * @param valueSerializer
 */
public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState,
		TypeSerializer<V> valueSerializer) {

	metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName,
			VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot());

	updateProxy();
	addKeyedStateRows(newState
			.map(new ValueStateToKeyedStateRow<K, V>(stateName,
					getKeySerializer(),
					valueSerializer,
					baseOpState.getMaxParallelism())));
}
 
Example #22
Source File: CopyOnWriteStateTableTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This tests that serializers used for snapshots are duplicates of the ones used in
 * processing to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {

	final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();

	RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			namespaceSerializer,
			stateSerializer);

	InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
	CopyOnWriteStateTable<Integer, Integer, Integer> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);

	table.put(0, 0, 0, 0);
	table.put(1, 0, 0, 1);
	table.put(2, 0, 1, 2);


	final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();

	final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
	namespaceSerializer.disable();
	keySerializer.disable();
	stateSerializer.disable();

	partitionedSnapshot.writeStateInKeyGroup(
		new DataOutputViewStreamWrapper(
			new ByteArrayOutputStreamWithPos(1024)), 0);
}
 
Example #23
Source File: HeapSnapshotStrategy.java    From flink with Apache License 2.0 5 votes vote down vote up
@Override
public <N, V> StateTable<K, N, V> newStateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo,
	TypeSerializer<K> keySerializer) {
	return snapshotStrategySynchronicityTrait.newStateTable(keyContext, newMetaInfo, keySerializer);
}
 
Example #24
Source File: RocksDbTtlCompactFiltersManager.java    From flink with Apache License 2.0 5 votes vote down vote up
public void setAndRegisterCompactFilterIfStateTtl(
	@Nonnull RegisteredStateMetaInfoBase metaInfoBase,
	@Nonnull ColumnFamilyOptions options) {

	if (enableTtlCompactionFilter && metaInfoBase instanceof RegisteredKeyValueStateBackendMetaInfo) {
		RegisteredKeyValueStateBackendMetaInfo kvMetaInfoBase = (RegisteredKeyValueStateBackendMetaInfo) metaInfoBase;
		if (TtlStateFactory.TtlSerializer.isTtlStateSerializer(kvMetaInfoBase.getStateSerializer())) {
			createAndSetCompactFilterFactory(metaInfoBase.getName(), options);
		}
	}
}
 
Example #25
Source File: RocksDBFoldingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBFoldingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(),
		backend);
}
 
Example #26
Source File: RocksDBKeyedStateBackend.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
	if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
		return Stream.empty();
	}

	RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo =
		(RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;

	final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
	final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
	boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
	final byte[] nameSpaceBytes;
	try {
		RocksDBKeySerializationUtils.writeNameSpace(
			namespace,
			namespaceSerializer,
			namespaceOutputView,
			ambiguousKeyPossible);
		nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
	} catch (IOException ex) {
		throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
	}

	RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle);
	iterator.seekToFirst();

	final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes,
		ambiguousKeyPossible, nameSpaceBytes);

	Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
	return targetStream.onClose(iteratorWrapper::close);
}
 
Example #27
Source File: RocksDBFoldingState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBFoldingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(),
		backend);
}
 
Example #28
Source File: RocksDBValueState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBValueState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		backend);
}
 
Example #29
Source File: RocksDBListState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBListState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(),
		(List<E>) stateDesc.getDefaultValue(),
		backend);
}
 
Example #30
Source File: RocksDBMapState.java    From flink with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}