org.apache.flink.contrib.streaming.state.RocksIteratorWrapper Java Examples
The following examples show how to use
org.apache.flink.contrib.streaming.state.RocksIteratorWrapper.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RocksStateKeysIterator.java From flink with Apache License 2.0 | 6 votes |
public RocksStateKeysIterator( @Nonnull RocksIteratorWrapper iterator, @Nonnull String state, @Nonnull TypeSerializer<K> keySerializer, int keyGroupPrefixBytes, boolean ambiguousKeyPossible, @Nonnull byte[] namespaceBytes) { this.iterator = iterator; this.state = state; this.keySerializer = keySerializer; this.keyGroupPrefixBytes = keyGroupPrefixBytes; this.namespaceBytes = namespaceBytes; this.nextKey = null; this.previousKey = null; this.ambiguousKeyPossible = ambiguousKeyPossible; this.byteArrayDataInputView = new DataInputDeserializer(); }
Example #2
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 6 votes |
private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) { Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1); PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue = new PriorityQueue<>(kvStateIterators.size(), iteratorComparator); for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) { final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0; rocksIterator.seekToFirst(); if (rocksIterator.isValid()) { iteratorPriorityQueue.offer( new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1)); } else { IOUtils.closeQuietly(rocksIterator); } } return iteratorPriorityQueue; }
Example #3
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void writeSnapshotToOutputStream( @Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider, @Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException { final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators = new ArrayList<>(metaData.size()); final DataOutputView outputView = new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream()); final ReadOptions readOptions = new ReadOptions(); try { readOptions.setSnapshot(snapshot); writeKVStateMetaData(kvStateIterators, readOptions, outputView); writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets); } finally { for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) { IOUtils.closeQuietly(kvStateIterator.f0); } IOUtils.closeQuietly(readOptions); } }
Example #4
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 6 votes |
public RocksStatesPerKeyGroupMergeIterator( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final int keyGroupPrefixByteCount) { Preconditions.checkNotNull(kvStateIterators); Preconditions.checkArgument(keyGroupPrefixByteCount >= 1); this.keyGroupPrefixByteCount = keyGroupPrefixByteCount; if (kvStateIterators.size() > 0) { this.heap = buildIteratorHeap(kvStateIterators); this.valid = !heap.isEmpty(); this.currentSubIterator = heap.poll(); kvStateIterators.clear(); } else { // creating a PriorityQueue of size 0 results in an exception. this.heap = null; this.valid = false; } this.newKeyGroup = true; this.newKVState = true; }
Example #5
Source File: RocksStateKeysIterator.java From flink with Apache License 2.0 | 6 votes |
public RocksStateKeysIterator( @Nonnull RocksIteratorWrapper iterator, @Nonnull String state, @Nonnull TypeSerializer<K> keySerializer, int keyGroupPrefixBytes, boolean ambiguousKeyPossible, @Nonnull byte[] namespaceBytes) { this.iterator = iterator; this.state = state; this.keySerializer = keySerializer; this.keyGroupPrefixBytes = keyGroupPrefixBytes; this.namespaceBytes = namespaceBytes; this.nextKey = null; this.previousKey = null; this.ambiguousKeyPossible = ambiguousKeyPossible; this.byteArrayDataInputView = new DataInputDeserializer(); }
Example #6
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void writeKVStateMetaData( final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final ReadOptions readOptions, final DataOutputView outputView) throws IOException { int kvStateId = 0; for (MetaData metaDataEntry : metaData) { RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator( db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions); kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId)); ++kvStateId; } KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>( // TODO: this code assumes that writing a serializer is threadsafe, we should support to // get a serialized form already at state registration time in the future keySerializer, stateMetaInfoSnapshots, !Objects.equals( UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); serializationProxy.write(outputView); }
Example #7
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void writeKVStateMetaData( final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final ReadOptions readOptions, final DataOutputView outputView) throws IOException { int kvStateId = 0; for (MetaData metaDataEntry : metaData) { RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator( db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions); kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId)); ++kvStateId; } KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>( // TODO: this code assumes that writing a serializer is threadsafe, we should support to // get a serialized form already at state registration time in the future keySerializer, stateMetaInfoSnapshots, !Objects.equals( UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); serializationProxy.write(outputView); }
Example #8
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 6 votes |
private void writeSnapshotToOutputStream( @Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider, @Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException { final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators = new ArrayList<>(metaData.size()); final DataOutputView outputView = new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream()); final ReadOptions readOptions = new ReadOptions(); try { readOptions.setSnapshot(snapshot); writeKVStateMetaData(kvStateIterators, readOptions, outputView); writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets); } finally { for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) { IOUtils.closeQuietly(kvStateIterator.f0); } IOUtils.closeQuietly(readOptions); } }
Example #9
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 6 votes |
public RocksStatesPerKeyGroupMergeIterator( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final int keyGroupPrefixByteCount) { Preconditions.checkNotNull(kvStateIterators); Preconditions.checkArgument(keyGroupPrefixByteCount >= 1); this.keyGroupPrefixByteCount = keyGroupPrefixByteCount; if (kvStateIterators.size() > 0) { this.heap = buildIteratorHeap(kvStateIterators); this.valid = !heap.isEmpty(); this.currentSubIterator = heap.poll(); kvStateIterators.clear(); } else { // creating a PriorityQueue of size 0 results in an exception. this.heap = null; this.valid = false; } this.newKeyGroup = true; this.newKVState = true; }
Example #10
Source File: RocksStatesPerKeyGroupMergeIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) { Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1); PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue = new PriorityQueue<>(kvStateIterators.size(), iteratorComparator); for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) { final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0; rocksIterator.seekToFirst(); if (rocksIterator.isValid()) { iteratorPriorityQueue.offer( new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1)); } else { IOUtils.closeQuietly(rocksIterator); } } return iteratorPriorityQueue; }
Example #11
Source File: RocksStatesPerKeyGroupMergeIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public RocksStatesPerKeyGroupMergeIterator( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final int keyGroupPrefixByteCount) { Preconditions.checkNotNull(kvStateIterators); Preconditions.checkArgument(keyGroupPrefixByteCount >= 1); this.keyGroupPrefixByteCount = keyGroupPrefixByteCount; if (kvStateIterators.size() > 0) { this.heap = buildIteratorHeap(kvStateIterators); this.valid = !heap.isEmpty(); this.currentSubIterator = heap.poll(); kvStateIterators.clear(); } else { // creating a PriorityQueue of size 0 results in an exception. this.heap = null; this.valid = false; } this.newKeyGroup = true; this.newKVState = true; }
Example #12
Source File: RocksStateKeysIterator.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
public RocksStateKeysIterator( @Nonnull RocksIteratorWrapper iterator, @Nonnull String state, @Nonnull TypeSerializer<K> keySerializer, int keyGroupPrefixBytes, boolean ambiguousKeyPossible, @Nonnull byte[] namespaceBytes) { this.iterator = iterator; this.state = state; this.keySerializer = keySerializer; this.keyGroupPrefixBytes = keyGroupPrefixBytes; this.namespaceBytes = namespaceBytes; this.nextKey = null; this.previousKey = null; this.ambiguousKeyPossible = ambiguousKeyPossible; this.byteArrayDataInputView = new DataInputDeserializer(); }
Example #13
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 6 votes |
private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap( List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) { Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1); PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue = new PriorityQueue<>(kvStateIterators.size(), iteratorComparator); for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) { final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0; rocksIterator.seekToFirst(); if (rocksIterator.isValid()) { iteratorPriorityQueue.offer( new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1)); } else { IOUtils.closeQuietly(rocksIterator); } } return iteratorPriorityQueue; }
Example #14
Source File: RocksFullSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void writeKVStateMetaData( final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators, final ReadOptions readOptions, final DataOutputView outputView) throws IOException { int kvStateId = 0; for (MetaData metaDataEntry : metaData) { RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator( db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions); kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId)); ++kvStateId; } KeyedBackendSerializationProxy<K> serializationProxy = new KeyedBackendSerializationProxy<>( // TODO: this code assumes that writing a serializer is threadsafe, we should support to // get a serialized form already at state registration time in the future keySerializer, stateMetaInfoSnapshots, !Objects.equals( UncompressedStreamCompressionDecorator.INSTANCE, keyGroupCompressionDecorator)); serializationProxy.write(outputView); }
Example #15
Source File: RocksFullSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 6 votes |
private void writeSnapshotToOutputStream( @Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider, @Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException { final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators = new ArrayList<>(metaData.size()); final DataOutputView outputView = new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream()); final ReadOptions readOptions = new ReadOptions(); try { readOptions.setSnapshot(snapshot); writeKVStateMetaData(kvStateIterators, readOptions, outputView); writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets); } finally { for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) { IOUtils.closeQuietly(kvStateIterator.f0); } IOUtils.closeQuietly(readOptions); } }
Example #16
Source File: RocksDBCheckpointIterator.java From bravo with Apache License 2.0 | 5 votes |
private void createColumnIterators(FilterFunction<String> stateFilter, List<StateMetaInfoSnapshot> stateMetaInfoSnapshots) throws Exception { Map<String, RocksIteratorWrapper> iterators = new HashMap<>(); for (int i = 0; i < stateMetaInfoSnapshots.size(); i++) { String name = stateMetaInfoSnapshots.get(i).getName(); if (stateFilter.filter(name)) { RocksIteratorWrapper iterator = new RocksIteratorWrapper( this.db.newIterator(stateColumnFamilyHandles.get(i + 1))); iterators.put(name, iterator); iterator.seekToFirst(); } } iteratorQueue = new LinkedList<>(iterators.entrySet()); updateCurrentIterator(); }
Example #17
Source File: RocksDBCheckpointIterator.java From bravo with Apache License 2.0 | 5 votes |
private void updateCurrentIterator() { IOUtils.closeQuietly(currentIterator); if (iteratorQueue.isEmpty()) { currentIterator = null; currentName = null; return; } else { Entry<String, RocksIteratorWrapper> e = iteratorQueue.pop(); currentName = e.getKey(); currentIterator = e.getValue(); } }
Example #18
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 5 votes |
/** * Advances the iterator. Should only be called if {@link #isValid()} returned true. * Valid flag can only change after calling {@link #next()}. */ public void next() { newKeyGroup = false; newKVState = false; final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator(); rocksIterator.next(); byte[] oldKey = currentSubIterator.getCurrentKey(); if (rocksIterator.isValid()) { currentSubIterator.setCurrentKey(rocksIterator.key()); if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) { heap.offer(currentSubIterator); currentSubIterator = heap.remove(); newKVState = currentSubIterator.getIterator() != rocksIterator; detectNewKeyGroup(oldKey); } } else { IOUtils.closeQuietly(rocksIterator); if (heap.isEmpty()) { currentSubIterator = null; valid = false; } else { currentSubIterator = heap.remove(); newKVState = true; detectNewKeyGroup(oldKey); } } }
Example #19
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static RocksIteratorWrapper getRocksIterator( RocksDB db, ColumnFamilyHandle columnFamilyHandle, StateSnapshotTransformer<byte[]> stateSnapshotTransformer, ReadOptions readOptions) { RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions); return stateSnapshotTransformer == null ? new RocksIteratorWrapper(rocksIterator) : new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer); }
Example #20
Source File: RocksStatesPerKeyGroupMergeIterator.java From flink with Apache License 2.0 | 5 votes |
/** * Advances the iterator. Should only be called if {@link #isValid()} returned true. * Valid flag can only change after calling {@link #next()}. */ public void next() { newKeyGroup = false; newKVState = false; final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator(); rocksIterator.next(); byte[] oldKey = currentSubIterator.getCurrentKey(); if (rocksIterator.isValid()) { currentSubIterator.setCurrentKey(rocksIterator.key()); if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) { heap.offer(currentSubIterator); currentSubIterator = heap.remove(); newKVState = currentSubIterator.getIterator() != rocksIterator; detectNewKeyGroup(oldKey); } } else { IOUtils.closeQuietly(rocksIterator); if (heap.isEmpty()) { currentSubIterator = null; valid = false; } else { currentSubIterator = heap.remove(); newKVState = true; detectNewKeyGroup(oldKey); } } }
Example #21
Source File: RocksFullSnapshotStrategy.java From flink with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static RocksIteratorWrapper getRocksIterator( RocksDB db, ColumnFamilyHandle columnFamilyHandle, StateSnapshotTransformer<byte[]> stateSnapshotTransformer, ReadOptions readOptions) { RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions); return stateSnapshotTransformer == null ? new RocksIteratorWrapper(rocksIterator) : new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer); }
Example #22
Source File: RocksStatesPerKeyGroupMergeIterator.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
/** * Advances the iterator. Should only be called if {@link #isValid()} returned true. * Valid flag can only change after calling {@link #next()}. */ public void next() { newKeyGroup = false; newKVState = false; final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator(); rocksIterator.next(); byte[] oldKey = currentSubIterator.getCurrentKey(); if (rocksIterator.isValid()) { currentSubIterator.setCurrentKey(rocksIterator.key()); if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) { heap.offer(currentSubIterator); currentSubIterator = heap.remove(); newKVState = currentSubIterator.getIterator() != rocksIterator; detectNewKeyGroup(oldKey); } } else { IOUtils.closeQuietly(rocksIterator); if (heap.isEmpty()) { currentSubIterator = null; valid = false; } else { currentSubIterator = heap.remove(); newKVState = true; detectNewKeyGroup(oldKey); } } }
Example #23
Source File: RocksFullSnapshotStrategy.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
@SuppressWarnings("unchecked") private static RocksIteratorWrapper getRocksIterator( RocksDB db, ColumnFamilyHandle columnFamilyHandle, StateSnapshotTransformer<byte[]> stateSnapshotTransformer, ReadOptions readOptions) { RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions); return stateSnapshotTransformer == null ? new RocksIteratorWrapper(rocksIterator) : new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer); }
Example #24
Source File: RocksSingleStateIterator.java From flink with Apache License 2.0 | 4 votes |
@Nonnull public RocksIteratorWrapper getIterator() { return iterator; }
Example #25
Source File: RocksSingleStateIterator.java From flink with Apache License 2.0 | 4 votes |
/** * @param iterator underlying {@link RocksIteratorWrapper} * @param kvStateId Id of the K/V state to which this iterator belongs. */ RocksSingleStateIterator(@Nonnull RocksIteratorWrapper iterator, int kvStateId) { this.iterator = iterator; this.currentKey = iterator.key(); this.kvStateId = kvStateId; }
Example #26
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = instanceBasePath.getAbsoluteFile().toPath().resolve(UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db, writeBatchSize)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle, tmpRestoreDBInfo.readOptions)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example #27
Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example #28
Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0 | 4 votes |
/** * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the * real restore instance and then the temporary instance is discarded. */ private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception { // Prepare for restore with rescaling KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial( restoreStateHandles, keyGroupRange); // Init base DB instance if (initialHandle != null) { restoreStateHandles.remove(initialHandle); initDBWithRescaling(initialHandle); } else { openDB(); } // Transfer remaining key-groups from temporary instance into base DB byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes); byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes]; RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes); for (KeyedStateHandle rawStateHandle : restoreStateHandles) { if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) { throw new IllegalStateException("Unexpected state handle type, " + "expected " + IncrementalRemoteKeyedStateHandle.class + ", but found " + rawStateHandle.getClass()); } Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString()); try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle( (IncrementalRemoteKeyedStateHandle) rawStateHandle, temporaryRestoreInstancePath); RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) { List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors; List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles; // iterating only the requested descriptors automatically skips the default column family handle for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) { ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i); ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle( null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i)) .columnFamilyHandle; try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) { iterator.seek(startKeyGroupPrefixBytes); while (iterator.isValid()) { if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) { writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value()); } else { // Since the iterator will visit the record according to the sorted order, // we can just break here. break; } iterator.next(); } } // releases native iterator resources } } finally { cleanUpPathQuietly(temporaryRestoreInstancePath); } } }
Example #29
Source File: RocksSingleStateIterator.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
@Nonnull public RocksIteratorWrapper getIterator() { return iterator; }
Example #30
Source File: RocksSingleStateIterator.java From Flink-CEPplus with Apache License 2.0 | 4 votes |
/** * @param iterator underlying {@link RocksIteratorWrapper} * @param kvStateId Id of the K/V state to which this iterator belongs. */ RocksSingleStateIterator(@Nonnull RocksIteratorWrapper iterator, int kvStateId) { this.iterator = iterator; this.currentKey = iterator.key(); this.kvStateId = kvStateId; }