Java Code Examples for org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo

The following examples show how to use org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 2
Source Project: Flink-CEPplus   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example 3
Source Project: flink   Source File: StateTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param keyContext    the key context provides the key scope for all put/get/delete operations.
 * @param metaInfo      the meta information, including the type serializer for state copy-on-write.
 * @param keySerializer the serializer of the key.
 */
public StateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo,
	TypeSerializer<K> keySerializer) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.metaInfo = Preconditions.checkNotNull(metaInfo);
	this.keySerializer = Preconditions.checkNotNull(keySerializer);

	this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();

	@SuppressWarnings("unchecked")
	StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
	this.keyGroupedStateMaps = state;
	for (int i = 0; i < this.keyGroupedStateMaps.length; i++) {
		this.keyGroupedStateMaps[i] = createStateMap();
	}
}
 
Example 4
Source Project: flink   Source File: CopyOnWriteStateTableTest.java    License: Apache License 2.0 6 votes vote down vote up
private CopyOnWriteStateTable<Integer, Integer, Float> createStateTableForSnapshotRelease(int numberOfKeyGroups) {
	RegisteredKeyValueStateBackendMetaInfo<Integer, Float> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			IntSerializer.INSTANCE,
			FloatSerializer.INSTANCE);

	MockInternalKeyContext<Integer> mockKeyContext =
		new MockInternalKeyContext<>(0, numberOfKeyGroups - 1, numberOfKeyGroups);
	CopyOnWriteStateTable<Integer, Integer, Float> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, IntSerializer.INSTANCE);

	ThreadLocalRandom random = ThreadLocalRandom.current();
	for (int i = 0; i < 1000; i++) {
		mockKeyContext.setCurrentKeyAndKeyGroup(i);
		table.put(random.nextInt(), random.nextFloat());
	}

	return table;
}
 
Example 5
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 6
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example 7
Source Project: flink   Source File: StateTable.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param keyContext    the key context provides the key scope for all put/get/delete operations.
 * @param metaInfo      the meta information, including the type serializer for state copy-on-write.
 * @param keySerializer the serializer of the key.
 */
public StateTable(
	InternalKeyContext<K> keyContext,
	RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo,
	TypeSerializer<K> keySerializer) {
	this.keyContext = Preconditions.checkNotNull(keyContext);
	this.metaInfo = Preconditions.checkNotNull(metaInfo);
	this.keySerializer = Preconditions.checkNotNull(keySerializer);

	this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();

	@SuppressWarnings("unchecked")
	StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
	this.keyGroupedStateMaps = state;
	for (int i = 0; i < this.keyGroupedStateMaps.length; i++) {
		this.keyGroupedStateMaps[i] = createStateMap();
	}
}
 
Example 8
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo(
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo,
	StateDescriptor<S, SV> stateDesc,
	TypeSerializer<N> namespaceSerializer,
	TypeSerializer<SV> stateSerializer) throws Exception {

	@SuppressWarnings("unchecked")
	RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1;

	TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer);
	if (s.isCompatibleAfterMigration() || s.isIncompatible()) {
		throw new StateMigrationException("The new namespace serializer must be compatible.");
	}

	restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc);

	TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility =
		restoredKvStateMetaInfo.updateStateSerializer(stateSerializer);
	if (newStateSerializerCompatibility.isCompatibleAfterMigration()) {
		migrateStateValues(stateDesc, oldStateInfo);
	} else if (newStateSerializerCompatibility.isIncompatible()) {
		throw new StateMigrationException("The new state serializer cannot be incompatible.");
	}

	return restoredKvStateMetaInfo;
}
 
Example 9
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public <N, SV, SEV, S extends State, IS extends S> IS createInternalState(
	@Nonnull TypeSerializer<N> namespaceSerializer,
	@Nonnull StateDescriptor<S, SV> stateDesc,
	@Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception {
	StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass());
	if (stateFactory == null) {
		String message = String.format("State %s is not supported by %s",
			stateDesc.getClass(), this.getClass());
		throw new FlinkRuntimeException(message);
	}
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation(
		stateDesc, namespaceSerializer, snapshotTransformFactory);
	return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 11
Source Project: Flink-CEPplus   Source File: RocksDBListState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBListState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(),
		(List<E>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 12
Source Project: flink   Source File: CopyOnWriteStateTableTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This tests that serializers used for snapshots are duplicates of the ones used in
 * processing to avoid race conditions in stateful serializers.
 */
@Test
public void testSerializerDuplicationInSnapshot() throws IOException {

	final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer();
	final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer();

	RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.VALUE,
			"test",
			namespaceSerializer,
			stateSerializer);

	InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>();
	CopyOnWriteStateTable<Integer, Integer, Integer> table =
		new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer);

	table.put(0, 0, 0, 0);
	table.put(1, 0, 0, 1);
	table.put(2, 0, 1, 2);


	final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot();

	final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter();
	namespaceSerializer.disable();
	keySerializer.disable();
	stateSerializer.disable();

	partitionedSnapshot.writeStateInKeyGroup(
		new DataOutputViewStreamWrapper(
			new ByteArrayOutputStreamWithPos(1024)), 0);
}
 
Example 13
Source Project: Flink-CEPplus   Source File: RocksDBReducingState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBReducingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((ReducingStateDescriptor<SV>) stateDesc).getReduceFunction(),
		backend);
}
 
Example 14
Source Project: Flink-CEPplus   Source File: RocksDBFoldingState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBFoldingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(),
		backend);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
	if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
		return Stream.empty();
	}

	RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo =
		(RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;

	final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
	final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
	boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
	final byte[] nameSpaceBytes;
	try {
		RocksDBKeySerializationUtils.writeNameSpace(
			namespace,
			namespaceSerializer,
			namespaceOutputView,
			ambiguousKeyPossible);
		nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
	} catch (IOException ex) {
		throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
	}

	RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle);
	iterator.seekToFirst();

	final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes,
		ambiguousKeyPossible, nameSpaceBytes);

	Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
	return targetStream.onClose(iteratorWrapper::close);
}
 
Example 16
Source Project: Flink-CEPplus   Source File: RocksDBAggregatingState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBAggregatingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((AggregatingStateDescriptor<?, SV, ?>) stateDesc).getAggregateFunction(),
		backend);
}
 
Example 17
Source Project: Flink-CEPplus   Source File: RocksFullSnapshotStrategy.java    License: Apache License 2.0 5 votes vote down vote up
private static List<MetaData> fillMetaData(
	List<RocksDbKvStateInfo> metaDataCopy) {
	List<MetaData> metaData = new ArrayList<>(metaDataCopy.size());
	for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) {
		StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null;
		if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) {
			stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo).
				getStateSnapshotTransformFactory().createForSerializedState().orElse(null);
		}
		metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer));
	}
	return metaData;
}
 
Example 18
public void setAndRegisterCompactFilterIfStateTtl(
	@Nonnull RegisteredStateMetaInfoBase metaInfoBase,
	@Nonnull ColumnFamilyOptions options) {

	if (enableTtlCompactionFilter && metaInfoBase instanceof RegisteredKeyValueStateBackendMetaInfo) {
		RegisteredKeyValueStateBackendMetaInfo kvMetaInfoBase = (RegisteredKeyValueStateBackendMetaInfo) metaInfoBase;
		if (TtlStateFactory.TtlSerializer.isTtlStateSerializer(kvMetaInfoBase.getStateSerializer())) {
			createAndSetCompactFilterFactory(metaInfoBase.getName(), options);
		}
	}
}
 
Example 19
Source Project: Flink-CEPplus   Source File: NestedMapsStateTable.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a new {@link NestedMapsStateTable} for the given key context and meta info.
 *
 * @param keyContext the key context.
 * @param metaInfo the meta information for this state table.
 */
public NestedMapsStateTable(InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo) {
	super(keyContext, metaInfo);
	this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup();

	@SuppressWarnings("unchecked")
	Map<N, Map<K, S>>[] state = (Map<N, Map<K, S>>[]) new Map[keyContext.getKeyGroupRange().getNumberOfKeyGroups()];
	this.state = state;
}
 
Example 20
Source Project: Flink-CEPplus   Source File: HeapRestoreOperation.java    License: Apache License 2.0 5 votes vote down vote up
private void createOrCheckStateForMetaInfo(
	List<StateMetaInfoSnapshot> restoredMetaInfo,
	Map<Integer, StateMetaInfoSnapshot> kvStatesById) {

	for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) {
		final StateSnapshotRestore registeredState;

		switch (metaInfoSnapshot.getBackendStateType()) {
			case KEY_VALUE:
				registeredState = registeredKVStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo =
						new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot);
					registeredKVStates.put(
						metaInfoSnapshot.getName(),
						snapshotStrategy.newStateTable(backend, registeredKeyedBackendStateMetaInfo));
				}
				break;
			case PRIORITY_QUEUE:
				registeredState = registeredPQStates.get(metaInfoSnapshot.getName());
				if (registeredState == null) {
					createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot));
				}
				break;
			default:
				throw new IllegalStateException("Unexpected state type: " +
					metaInfoSnapshot.getBackendStateType() + ".");
		}

		if (registeredState == null) {
			kvStatesById.put(kvStatesById.size(), metaInfoSnapshot);
		}
	}
}
 
Example 21
Source Project: Flink-CEPplus   Source File: CopyOnWriteStateTable.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Constructs a new {@code StateTable} instance with the specified capacity.
 *
 * @param keyContext the key context.
 * @param metaInfo   the meta information, including the type serializer for state copy-on-write.
 * @param capacity   the initial capacity of this hash map.
 * @throws IllegalArgumentException when the capacity is less than zero.
 */
@SuppressWarnings("unchecked")
private CopyOnWriteStateTable(InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo, int capacity) {
	super(keyContext, metaInfo);

	// initialized tables to EMPTY_TABLE.
	this.primaryTable = (StateTableEntry<K, N, S>[]) EMPTY_TABLE;
	this.incrementalRehashTable = (StateTableEntry<K, N, S>[]) EMPTY_TABLE;

	// initialize sizes to 0.
	this.primaryTableSize = 0;
	this.incrementalRehashTableSize = 0;

	this.rehashIndex = 0;
	this.stateTableVersion = 0;
	this.highestRequiredSnapshotVersion = 0;
	this.snapshotVersions = new TreeSet<>();

	if (capacity < 0) {
		throw new IllegalArgumentException("Capacity: " + capacity);
	}

	if (capacity == 0) {
		threshold = -1;
		return;
	}

	if (capacity < MINIMUM_CAPACITY) {
		capacity = MINIMUM_CAPACITY;
	} else if (capacity > MAXIMUM_CAPACITY) {
		capacity = MAXIMUM_CAPACITY;
	} else {
		capacity = MathUtils.roundUpToPowerOfTwo(capacity);
	}
	primaryTable = makeTable(capacity);
}
 
Example 22
Source Project: Flink-CEPplus   Source File: CopyOnWriteStateTableTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This test triggers incremental rehash and tests for corruptions.
 */
@Test
public void testIncrementalRehash() {
	RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo =
		new RegisteredKeyValueStateBackendMetaInfo<>(
			StateDescriptor.Type.UNKNOWN,
			"test",
			IntSerializer.INSTANCE,
			new ArrayListSerializer<>(IntSerializer.INSTANCE)); // we use mutable state objects.

	final MockInternalKeyContext<Integer> keyContext = new MockInternalKeyContext<>(IntSerializer.INSTANCE);

	final CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> stateTable =
		new CopyOnWriteStateTable<>(keyContext, metaInfo);

	int insert = 0;
	int remove = 0;
	while (!stateTable.isRehashing()) {
		stateTable.put(insert++, 0, new ArrayList<Integer>());
		if (insert % 8 == 0) {
			stateTable.remove(remove++, 0);
		}
	}
	Assert.assertEquals(insert - remove, stateTable.size());
	while (stateTable.isRehashing()) {
		stateTable.put(insert++, 0, new ArrayList<Integer>());
		if (insert % 8 == 0) {
			stateTable.remove(remove++, 0);
		}
	}
	Assert.assertEquals(insert - remove, stateTable.size());

	for (int i = 0; i < insert; ++i) {
		if (i < remove) {
			Assert.assertFalse(stateTable.containsKey(i, 0));
		} else {
			Assert.assertTrue(stateTable.containsKey(i, 0));
		}
	}
}
 
Example 23
Source Project: flink   Source File: RocksDBMapState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <UK, UV, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBMapState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(),
		(Map<UK, UV>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 24
Source Project: flink   Source File: RocksDBListState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <E, K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBListState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		(TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(),
		(List<E>) stateDesc.getDefaultValue(),
		backend);
}
 
Example 25
Source Project: flink   Source File: RocksDBValueState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBValueState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		backend);
}
 
Example 26
Source Project: flink   Source File: RocksDBFoldingState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBFoldingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(),
		backend);
}
 
Example 27
Source Project: flink   Source File: RocksDBKeyedStateBackend.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
@Override
public <N> Stream<K> getKeys(String state, N namespace) {
	RocksDbKvStateInfo columnInfo = kvStateInformation.get(state);
	if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) {
		return Stream.empty();
	}

	RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo =
		(RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo;

	final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer();
	final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8);
	boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer);
	final byte[] nameSpaceBytes;
	try {
		RocksDBKeySerializationUtils.writeNameSpace(
			namespace,
			namespaceSerializer,
			namespaceOutputView,
			ambiguousKeyPossible);
		nameSpaceBytes = namespaceOutputView.getCopyOfBuffer();
	} catch (IOException ex) {
		throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex);
	}

	RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle);
	iterator.seekToFirst();

	final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes,
		ambiguousKeyPossible, nameSpaceBytes);

	Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false);
	return targetStream.onClose(iteratorWrapper::close);
}
 
Example 28
Source Project: flink   Source File: RocksDBAggregatingState.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
static <K, N, SV, S extends State, IS extends S> IS create(
	StateDescriptor<S, SV> stateDesc,
	Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult,
	RocksDBKeyedStateBackend<K> backend) {
	return (IS) new RocksDBAggregatingState<>(
		registerResult.f0,
		registerResult.f1.getNamespaceSerializer(),
		registerResult.f1.getStateSerializer(),
		stateDesc.getDefaultValue(),
		((AggregatingStateDescriptor<?, SV, ?>) stateDesc).getAggregateFunction(),
		backend);
}
 
Example 29
Source Project: flink   Source File: RocksFullSnapshotStrategy.java    License: Apache License 2.0 5 votes vote down vote up
private static List<MetaData> fillMetaData(
	List<RocksDbKvStateInfo> metaDataCopy) {
	List<MetaData> metaData = new ArrayList<>(metaDataCopy.size());
	for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) {
		StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null;
		if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) {
			stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo).
				getStateSnapshotTransformFactory().createForSerializedState().orElse(null);
		}
		metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer));
	}
	return metaData;
}
 
Example 30
Source Project: flink   Source File: RocksDbTtlCompactFiltersManager.java    License: Apache License 2.0 5 votes vote down vote up
public void setAndRegisterCompactFilterIfStateTtl(
	@Nonnull RegisteredStateMetaInfoBase metaInfoBase,
	@Nonnull ColumnFamilyOptions options) {

	if (enableTtlCompactionFilter && metaInfoBase instanceof RegisteredKeyValueStateBackendMetaInfo) {
		RegisteredKeyValueStateBackendMetaInfo kvMetaInfoBase = (RegisteredKeyValueStateBackendMetaInfo) metaInfoBase;
		if (TtlStateFactory.TtlSerializer.isTtlStateSerializer(kvMetaInfoBase.getStateSerializer())) {
			createAndSetCompactFilterFactory(metaInfoBase.getName(), options);
		}
	}
}