org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo Java Examples
The following examples show how to use
org.apache.flink.runtime.state.RegisteredKeyValueStateBackendMetaInfo.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StateTable.java From flink with Apache License 2.0 | 6 votes |
/** * @param keyContext the key context provides the key scope for all put/get/delete operations. * @param metaInfo the meta information, including the type serializer for state copy-on-write. * @param keySerializer the serializer of the key. */ public StateTable( InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo, TypeSerializer<K> keySerializer) { this.keyContext = Preconditions.checkNotNull(keyContext); this.metaInfo = Preconditions.checkNotNull(metaInfo); this.keySerializer = Preconditions.checkNotNull(keySerializer); this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup(); @SuppressWarnings("unchecked") StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()]; this.keyGroupedStateMaps = state; for (int i = 0; i < this.keyGroupedStateMaps.length; i++) { this.keyGroupedStateMaps[i] = createStateMap(); } }
Example #2
Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 6 votes |
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
Example #3
Source File: StateTable.java From flink with Apache License 2.0 | 6 votes |
/** * @param keyContext the key context provides the key scope for all put/get/delete operations. * @param metaInfo the meta information, including the type serializer for state copy-on-write. * @param keySerializer the serializer of the key. */ public StateTable( InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo, TypeSerializer<K> keySerializer) { this.keyContext = Preconditions.checkNotNull(keyContext); this.metaInfo = Preconditions.checkNotNull(metaInfo); this.keySerializer = Preconditions.checkNotNull(keySerializer); this.keyGroupOffset = keyContext.getKeyGroupRange().getStartKeyGroup(); @SuppressWarnings("unchecked") StateMap<K, N, S>[] state = (StateMap<K, N, S>[]) new StateMap[keyContext.getKeyGroupRange().getNumberOfKeyGroups()]; this.keyGroupedStateMaps = state; for (int i = 0; i < this.keyGroupedStateMaps.length; i++) { this.keyGroupedStateMaps[i] = createStateMap(); } }
Example #4
Source File: CopyOnWriteStateTableTest.java From flink with Apache License 2.0 | 6 votes |
private CopyOnWriteStateTable<Integer, Integer, Float> createStateTableForSnapshotRelease(int numberOfKeyGroups) { RegisteredKeyValueStateBackendMetaInfo<Integer, Float> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( StateDescriptor.Type.VALUE, "test", IntSerializer.INSTANCE, FloatSerializer.INSTANCE); MockInternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>(0, numberOfKeyGroups - 1, numberOfKeyGroups); CopyOnWriteStateTable<Integer, Integer, Float> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, IntSerializer.INSTANCE); ThreadLocalRandom random = ThreadLocalRandom.current(); for (int i = 0; i < 1000; i++) { mockKeyContext.setCurrentKeyAndKeyGroup(i); table.put(random.nextInt(), random.nextFloat()); } return table; }
Example #5
Source File: RocksDBKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
Example #6
Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 6 votes |
@Override @Nonnull public <N, SV, SEV, S extends State, IS extends S> IS createInternalState( @Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception { StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass()); if (stateFactory == null) { String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass()); throw new FlinkRuntimeException(message); } Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation( stateDesc, namespaceSerializer, snapshotTransformFactory); return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this); }
Example #7
Source File: RocksDBKeyedStateBackend.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
@Override @Nonnull public <N, SV, SEV, S extends State, IS extends S> IS createInternalState( @Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception { StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass()); if (stateFactory == null) { String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass()); throw new FlinkRuntimeException(message); } Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation( stateDesc, namespaceSerializer, snapshotTransformFactory); return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this); }
Example #8
Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 6 votes |
@Override @Nonnull public <N, SV, SEV, S extends State, IS extends S> IS createInternalState( @Nonnull TypeSerializer<N> namespaceSerializer, @Nonnull StateDescriptor<S, SV> stateDesc, @Nonnull StateSnapshotTransformFactory<SEV> snapshotTransformFactory) throws Exception { StateFactory stateFactory = STATE_FACTORIES.get(stateDesc.getClass()); if (stateFactory == null) { String message = String.format("State %s is not supported by %s", stateDesc.getClass(), this.getClass()); throw new FlinkRuntimeException(message); } Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult = tryRegisterKvStateInformation( stateDesc, namespaceSerializer, snapshotTransformFactory); return stateFactory.createState(stateDesc, registerResult, RocksDBKeyedStateBackend.this); }
Example #9
Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 6 votes |
private <N, S extends State, SV> RegisteredKeyValueStateBackendMetaInfo<N, SV> updateRestoredStateMetaInfo( Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> oldStateInfo, StateDescriptor<S, SV> stateDesc, TypeSerializer<N> namespaceSerializer, TypeSerializer<SV> stateSerializer) throws Exception { @SuppressWarnings("unchecked") RegisteredKeyValueStateBackendMetaInfo<N, SV> restoredKvStateMetaInfo = oldStateInfo.f1; TypeSerializerSchemaCompatibility<N> s = restoredKvStateMetaInfo.updateNamespaceSerializer(namespaceSerializer); if (s.isCompatibleAfterMigration() || s.isIncompatible()) { throw new StateMigrationException("The new namespace serializer must be compatible."); } restoredKvStateMetaInfo.checkStateMetaInfo(stateDesc); TypeSerializerSchemaCompatibility<SV> newStateSerializerCompatibility = restoredKvStateMetaInfo.updateStateSerializer(stateSerializer); if (newStateSerializerCompatibility.isCompatibleAfterMigration()) { migrateStateValues(stateDesc, oldStateInfo); } else if (newStateSerializerCompatibility.isIncompatible()) { throw new StateMigrationException("The new state serializer cannot be incompatible."); } return restoredKvStateMetaInfo; }
Example #10
Source File: RocksDBMapState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <UK, UV, K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBMapState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), (TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(), (Map<UK, UV>) stateDesc.getDefaultValue(), backend); }
Example #11
Source File: RocksDBMapState.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <UK, UV, K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBMapState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), (TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(), (Map<UK, UV>) stateDesc.getDefaultValue(), backend); }
Example #12
Source File: RocksDBListState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <E, K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBListState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), (TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(), (List<E>) stateDesc.getDefaultValue(), backend); }
Example #13
Source File: HeapRestoreOperation.java From flink with Apache License 2.0 | 5 votes |
private void createOrCheckStateForMetaInfo( List<StateMetaInfoSnapshot> restoredMetaInfo, Map<Integer, StateMetaInfoSnapshot> kvStatesById) { for (StateMetaInfoSnapshot metaInfoSnapshot : restoredMetaInfo) { final StateSnapshotRestore registeredState; switch (metaInfoSnapshot.getBackendStateType()) { case KEY_VALUE: registeredState = registeredKVStates.get(metaInfoSnapshot.getName()); if (registeredState == null) { RegisteredKeyValueStateBackendMetaInfo<?, ?> registeredKeyedBackendStateMetaInfo = new RegisteredKeyValueStateBackendMetaInfo<>(metaInfoSnapshot); registeredKVStates.put( metaInfoSnapshot.getName(), snapshotStrategy.newStateTable( keyContext, registeredKeyedBackendStateMetaInfo, keySerializerProvider.currentSchemaSerializer())); } break; case PRIORITY_QUEUE: registeredState = registeredPQStates.get(metaInfoSnapshot.getName()); if (registeredState == null) { createInternal(new RegisteredPriorityQueueStateBackendMetaInfo<>(metaInfoSnapshot)); } break; default: throw new IllegalStateException("Unexpected state type: " + metaInfoSnapshot.getBackendStateType() + "."); } if (registeredState == null) { kvStatesById.put(kvStatesById.size(), metaInfoSnapshot); } } }
Example #14
Source File: NestedMapsStateTable.java From flink with Apache License 2.0 | 5 votes |
/** * Creates a new {@link NestedMapsStateTable} for the given key context and meta info. * @param keyContext the key context. * @param metaInfo the meta information for this state table. * @param keySerializer the serializer of the key. */ public NestedMapsStateTable( InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, S> metaInfo, TypeSerializer<K> keySerializer) { super(keyContext, metaInfo, keySerializer); }
Example #15
Source File: RocksDBReducingState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBReducingState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), ((ReducingStateDescriptor<SV>) stateDesc).getReduceFunction(), backend); }
Example #16
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 5 votes |
private static List<MetaData> fillMetaData( List<RocksDbKvStateInfo> metaDataCopy) { List<MetaData> metaData = new ArrayList<>(metaDataCopy.size()); for (RocksDbKvStateInfo rocksDbKvStateInfo : metaDataCopy) { StateSnapshotTransformer<byte[]> stateSnapshotTransformer = null; if (rocksDbKvStateInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo) { stateSnapshotTransformer = ((RegisteredKeyValueStateBackendMetaInfo<?, ?>) rocksDbKvStateInfo.metaInfo). getStateSnapshotTransformFactory().createForSerializedState().orElse(null); } metaData.add(new MetaData(rocksDbKvStateInfo, stateSnapshotTransformer)); } return metaData; }
Example #17
Source File: CopyOnWriteStateTableTest.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * This test triggers incremental rehash and tests for corruptions. */ @Test public void testIncrementalRehash() { RegisteredKeyValueStateBackendMetaInfo<Integer, ArrayList<Integer>> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( StateDescriptor.Type.UNKNOWN, "test", IntSerializer.INSTANCE, new ArrayListSerializer<>(IntSerializer.INSTANCE)); // we use mutable state objects. final MockInternalKeyContext<Integer> keyContext = new MockInternalKeyContext<>(IntSerializer.INSTANCE); final CopyOnWriteStateTable<Integer, Integer, ArrayList<Integer>> stateTable = new CopyOnWriteStateTable<>(keyContext, metaInfo); int insert = 0; int remove = 0; while (!stateTable.isRehashing()) { stateTable.put(insert++, 0, new ArrayList<Integer>()); if (insert % 8 == 0) { stateTable.remove(remove++, 0); } } Assert.assertEquals(insert - remove, stateTable.size()); while (stateTable.isRehashing()) { stateTable.put(insert++, 0, new ArrayList<Integer>()); if (insert % 8 == 0) { stateTable.remove(remove++, 0); } } Assert.assertEquals(insert - remove, stateTable.size()); for (int i = 0; i < insert; ++i) { if (i < remove) { Assert.assertFalse(stateTable.containsKey(i, 0)); } else { Assert.assertTrue(stateTable.containsKey(i, 0)); } } }
Example #18
Source File: AsyncSnapshotStrategySynchronicityBehavior.java From flink with Apache License 2.0 | 5 votes |
@Override public <N, V> StateTable<K, N, V> newStateTable( InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo, TypeSerializer<K> keySerializer) { return new CopyOnWriteStateTable<>(keyContext, newMetaInfo, keySerializer); }
Example #19
Source File: RocksDBAggregatingState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBAggregatingState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), ((AggregatingStateDescriptor<?, SV, ?>) stateDesc).getAggregateFunction(), backend); }
Example #20
Source File: RocksDBValueState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBValueState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), backend); }
Example #21
Source File: OperatorStateWriter.java From bravo with Apache License 2.0 | 5 votes |
/** * Defines/redefines a value state with the given name and type. This can be * used to create new states of an operator or change the type of an already * existing state. * <p> * When redefining a pre-existing state make sure you haven't added that as * keyed state rows before. * * @param stateName * @param newState * @param valueSerializer */ public <K, V> void createNewValueState(String stateName, DataSet<Tuple2<K, V>> newState, TypeSerializer<V> valueSerializer) { metaSnapshots.put(stateName, new RegisteredKeyValueStateBackendMetaInfo<>(StateDescriptor.Type.VALUE, stateName, VoidNamespaceSerializer.INSTANCE, valueSerializer).snapshot()); updateProxy(); addKeyedStateRows(newState .map(new ValueStateToKeyedStateRow<K, V>(stateName, getKeySerializer(), valueSerializer, baseOpState.getMaxParallelism()))); }
Example #22
Source File: CopyOnWriteStateTableTest.java From flink with Apache License 2.0 | 5 votes |
/** * This tests that serializers used for snapshots are duplicates of the ones used in * processing to avoid race conditions in stateful serializers. */ @Test public void testSerializerDuplicationInSnapshot() throws IOException { final TestDuplicateSerializer namespaceSerializer = new TestDuplicateSerializer(); final TestDuplicateSerializer stateSerializer = new TestDuplicateSerializer(); final TestDuplicateSerializer keySerializer = new TestDuplicateSerializer(); RegisteredKeyValueStateBackendMetaInfo<Integer, Integer> metaInfo = new RegisteredKeyValueStateBackendMetaInfo<>( StateDescriptor.Type.VALUE, "test", namespaceSerializer, stateSerializer); InternalKeyContext<Integer> mockKeyContext = new MockInternalKeyContext<>(); CopyOnWriteStateTable<Integer, Integer, Integer> table = new CopyOnWriteStateTable<>(mockKeyContext, metaInfo, keySerializer); table.put(0, 0, 0, 0); table.put(1, 0, 0, 1); table.put(2, 0, 1, 2); final CopyOnWriteStateTableSnapshot<Integer, Integer, Integer> snapshot = table.stateSnapshot(); final StateSnapshot.StateKeyGroupWriter partitionedSnapshot = snapshot.getKeyGroupWriter(); namespaceSerializer.disable(); keySerializer.disable(); stateSerializer.disable(); partitionedSnapshot.writeStateInKeyGroup( new DataOutputViewStreamWrapper( new ByteArrayOutputStreamWithPos(1024)), 0); }
Example #23
Source File: HeapSnapshotStrategy.java From flink with Apache License 2.0 | 5 votes |
@Override public <N, V> StateTable<K, N, V> newStateTable( InternalKeyContext<K> keyContext, RegisteredKeyValueStateBackendMetaInfo<N, V> newMetaInfo, TypeSerializer<K> keySerializer) { return snapshotStrategySynchronicityTrait.newStateTable(keyContext, newMetaInfo, keySerializer); }
Example #24
Source File: RocksDbTtlCompactFiltersManager.java From flink with Apache License 2.0 | 5 votes |
public void setAndRegisterCompactFilterIfStateTtl( @Nonnull RegisteredStateMetaInfoBase metaInfoBase, @Nonnull ColumnFamilyOptions options) { if (enableTtlCompactionFilter && metaInfoBase instanceof RegisteredKeyValueStateBackendMetaInfo) { RegisteredKeyValueStateBackendMetaInfo kvMetaInfoBase = (RegisteredKeyValueStateBackendMetaInfo) metaInfoBase; if (TtlStateFactory.TtlSerializer.isTtlStateSerializer(kvMetaInfoBase.getStateSerializer())) { createAndSetCompactFilterFactory(metaInfoBase.getName(), options); } } }
Example #25
Source File: RocksDBFoldingState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBFoldingState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), ((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(), backend); }
Example #26
Source File: RocksDBKeyedStateBackend.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") @Override public <N> Stream<K> getKeys(String state, N namespace) { RocksDbKvStateInfo columnInfo = kvStateInformation.get(state); if (columnInfo == null || !(columnInfo.metaInfo instanceof RegisteredKeyValueStateBackendMetaInfo)) { return Stream.empty(); } RegisteredKeyValueStateBackendMetaInfo<N, ?> registeredKeyValueStateBackendMetaInfo = (RegisteredKeyValueStateBackendMetaInfo<N, ?>) columnInfo.metaInfo; final TypeSerializer<N> namespaceSerializer = registeredKeyValueStateBackendMetaInfo.getNamespaceSerializer(); final DataOutputSerializer namespaceOutputView = new DataOutputSerializer(8); boolean ambiguousKeyPossible = RocksDBKeySerializationUtils.isAmbiguousKeyPossible(getKeySerializer(), namespaceSerializer); final byte[] nameSpaceBytes; try { RocksDBKeySerializationUtils.writeNameSpace( namespace, namespaceSerializer, namespaceOutputView, ambiguousKeyPossible); nameSpaceBytes = namespaceOutputView.getCopyOfBuffer(); } catch (IOException ex) { throw new FlinkRuntimeException("Failed to get keys from RocksDB state backend.", ex); } RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(db, columnInfo.columnFamilyHandle); iterator.seekToFirst(); final RocksStateKeysIterator<K> iteratorWrapper = new RocksStateKeysIterator<>(iterator, state, getKeySerializer(), keyGroupPrefixBytes, ambiguousKeyPossible, nameSpaceBytes); Stream<K> targetStream = StreamSupport.stream(Spliterators.spliteratorUnknownSize(iteratorWrapper, Spliterator.ORDERED), false); return targetStream.onClose(iteratorWrapper::close); }
Example #27
Source File: RocksDBFoldingState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBFoldingState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), ((FoldingStateDescriptor<?, SV>) stateDesc).getFoldFunction(), backend); }
Example #28
Source File: RocksDBValueState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBValueState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), registerResult.f1.getStateSerializer(), stateDesc.getDefaultValue(), backend); }
Example #29
Source File: RocksDBListState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <E, K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBListState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), (TypeSerializer<List<E>>) registerResult.f1.getStateSerializer(), (List<E>) stateDesc.getDefaultValue(), backend); }
Example #30
Source File: RocksDBMapState.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") static <UK, UV, K, N, SV, S extends State, IS extends S> IS create( StateDescriptor<S, SV> stateDesc, Tuple2<ColumnFamilyHandle, RegisteredKeyValueStateBackendMetaInfo<N, SV>> registerResult, RocksDBKeyedStateBackend<K> backend) { return (IS) new RocksDBMapState<>( registerResult.f0, registerResult.f1.getNamespaceSerializer(), (TypeSerializer<Map<UK, UV>>) registerResult.f1.getStateSerializer(), (Map<UK, UV>) stateDesc.getDefaultValue(), backend); }