org.apache.flink.contrib.streaming.state.RocksIteratorWrapper Java Examples

public RocksStateKeysIterator(
	@Nonnull RocksIteratorWrapper iterator,
	@Nonnull String state,
	@Nonnull TypeSerializer<K> keySerializer,
	int keyGroupPrefixBytes,
	boolean ambiguousKeyPossible,
	@Nonnull byte[] namespaceBytes) {
	this.iterator = iterator;
	this.state = state;
	this.keySerializer = keySerializer;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.namespaceBytes = namespaceBytes;
	this.nextKey = null;
	this.previousKey = null;
	this.ambiguousKeyPossible = ambiguousKeyPossible;
	this.byteArrayDataInputView = new DataInputDeserializer();
}
 

private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) {

	Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1);

	PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue =
		new PriorityQueue<>(kvStateIterators.size(), iteratorComparator);

	for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) {
		final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0;
		rocksIterator.seekToFirst();
		if (rocksIterator.isValid()) {
			iteratorPriorityQueue.offer(
				new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1));
		} else {
			IOUtils.closeQuietly(rocksIterator);
		}
	}
	return iteratorPriorityQueue;
}
 

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

public RocksStatesPerKeyGroupMergeIterator(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final int keyGroupPrefixByteCount) {
	Preconditions.checkNotNull(kvStateIterators);
	Preconditions.checkArgument(keyGroupPrefixByteCount >= 1);

	this.keyGroupPrefixByteCount = keyGroupPrefixByteCount;

	if (kvStateIterators.size() > 0) {
		this.heap = buildIteratorHeap(kvStateIterators);
		this.valid = !heap.isEmpty();
		this.currentSubIterator = heap.poll();
		kvStateIterators.clear();
	} else {
		// creating a PriorityQueue of size 0 results in an exception.
		this.heap = null;
		this.valid = false;
	}

	this.newKeyGroup = true;
	this.newKVState = true;
}
 

public RocksStateKeysIterator(
	@Nonnull RocksIteratorWrapper iterator,
	@Nonnull String state,
	@Nonnull TypeSerializer<K> keySerializer,
	int keyGroupPrefixBytes,
	boolean ambiguousKeyPossible,
	@Nonnull byte[] namespaceBytes) {
	this.iterator = iterator;
	this.state = state;
	this.keySerializer = keySerializer;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.namespaceBytes = namespaceBytes;
	this.nextKey = null;
	this.previousKey = null;
	this.ambiguousKeyPossible = ambiguousKeyPossible;
	this.byteArrayDataInputView = new DataInputDeserializer();
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

public RocksStatesPerKeyGroupMergeIterator(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final int keyGroupPrefixByteCount) {
	Preconditions.checkNotNull(kvStateIterators);
	Preconditions.checkArgument(keyGroupPrefixByteCount >= 1);

	this.keyGroupPrefixByteCount = keyGroupPrefixByteCount;

	if (kvStateIterators.size() > 0) {
		this.heap = buildIteratorHeap(kvStateIterators);
		this.valid = !heap.isEmpty();
		this.currentSubIterator = heap.poll();
		kvStateIterators.clear();
	} else {
		// creating a PriorityQueue of size 0 results in an exception.
		this.heap = null;
		this.valid = false;
	}

	this.newKeyGroup = true;
	this.newKVState = true;
}
 

private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) {

	Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1);

	PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue =
		new PriorityQueue<>(kvStateIterators.size(), iteratorComparator);

	for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) {
		final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0;
		rocksIterator.seekToFirst();
		if (rocksIterator.isValid()) {
			iteratorPriorityQueue.offer(
				new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1));
		} else {
			IOUtils.closeQuietly(rocksIterator);
		}
	}
	return iteratorPriorityQueue;
}
 

public RocksStatesPerKeyGroupMergeIterator(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final int keyGroupPrefixByteCount) {
	Preconditions.checkNotNull(kvStateIterators);
	Preconditions.checkArgument(keyGroupPrefixByteCount >= 1);

	this.keyGroupPrefixByteCount = keyGroupPrefixByteCount;

	if (kvStateIterators.size() > 0) {
		this.heap = buildIteratorHeap(kvStateIterators);
		this.valid = !heap.isEmpty();
		this.currentSubIterator = heap.poll();
		kvStateIterators.clear();
	} else {
		// creating a PriorityQueue of size 0 results in an exception.
		this.heap = null;
		this.valid = false;
	}

	this.newKeyGroup = true;
	this.newKVState = true;
}
 

public RocksStateKeysIterator(
	@Nonnull RocksIteratorWrapper iterator,
	@Nonnull String state,
	@Nonnull TypeSerializer<K> keySerializer,
	int keyGroupPrefixBytes,
	boolean ambiguousKeyPossible,
	@Nonnull byte[] namespaceBytes) {
	this.iterator = iterator;
	this.state = state;
	this.keySerializer = keySerializer;
	this.keyGroupPrefixBytes = keyGroupPrefixBytes;
	this.namespaceBytes = namespaceBytes;
	this.nextKey = null;
	this.previousKey = null;
	this.ambiguousKeyPossible = ambiguousKeyPossible;
	this.byteArrayDataInputView = new DataInputDeserializer();
}
 

private PriorityQueue<RocksSingleStateIterator> buildIteratorHeap(
	List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators) {

	Comparator<RocksSingleStateIterator> iteratorComparator = COMPARATORS.get(keyGroupPrefixByteCount - 1);

	PriorityQueue<RocksSingleStateIterator> iteratorPriorityQueue =
		new PriorityQueue<>(kvStateIterators.size(), iteratorComparator);

	for (Tuple2<RocksIteratorWrapper, Integer> rocksIteratorWithKVStateId : kvStateIterators) {
		final RocksIteratorWrapper rocksIterator = rocksIteratorWithKVStateId.f0;
		rocksIterator.seekToFirst();
		if (rocksIterator.isValid()) {
			iteratorPriorityQueue.offer(
				new RocksSingleStateIterator(rocksIterator, rocksIteratorWithKVStateId.f1));
		} else {
			IOUtils.closeQuietly(rocksIterator);
		}
	}
	return iteratorPriorityQueue;
}
 

private void writeKVStateMetaData(
	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators,
	final ReadOptions readOptions,
	final DataOutputView outputView) throws IOException {

	int kvStateId = 0;

	for (MetaData metaDataEntry : metaData) {
		RocksIteratorWrapper rocksIteratorWrapper = getRocksIterator(
			db, metaDataEntry.rocksDbKvStateInfo.columnFamilyHandle, metaDataEntry.stateSnapshotTransformer, readOptions);
		kvStateIterators.add(Tuple2.of(rocksIteratorWrapper, kvStateId));
		++kvStateId;
	}

	KeyedBackendSerializationProxy<K> serializationProxy =
		new KeyedBackendSerializationProxy<>(
			// TODO: this code assumes that writing a serializer is threadsafe, we should support to
			// get a serialized form already at state registration time in the future
			keySerializer,
			stateMetaInfoSnapshots,
			!Objects.equals(
				UncompressedStreamCompressionDecorator.INSTANCE,
				keyGroupCompressionDecorator));

	serializationProxy.write(outputView);
}
 

private void writeSnapshotToOutputStream(
	@Nonnull CheckpointStreamWithResultProvider checkpointStreamWithResultProvider,
	@Nonnull KeyGroupRangeOffsets keyGroupRangeOffsets) throws IOException, InterruptedException {

	final List<Tuple2<RocksIteratorWrapper, Integer>> kvStateIterators =
		new ArrayList<>(metaData.size());
	final DataOutputView outputView =
		new DataOutputViewStreamWrapper(checkpointStreamWithResultProvider.getCheckpointOutputStream());
	final ReadOptions readOptions = new ReadOptions();
	try {
		readOptions.setSnapshot(snapshot);
		writeKVStateMetaData(kvStateIterators, readOptions, outputView);
		writeKVStateData(kvStateIterators, checkpointStreamWithResultProvider, keyGroupRangeOffsets);
	} finally {

		for (Tuple2<RocksIteratorWrapper, Integer> kvStateIterator : kvStateIterators) {
			IOUtils.closeQuietly(kvStateIterator.f0);
		}

		IOUtils.closeQuietly(readOptions);
	}
}
 

private void createColumnIterators(FilterFunction<String> stateFilter,
		List<StateMetaInfoSnapshot> stateMetaInfoSnapshots)
		throws Exception {
	Map<String, RocksIteratorWrapper> iterators = new HashMap<>();
	for (int i = 0; i < stateMetaInfoSnapshots.size(); i++) {
		String name = stateMetaInfoSnapshots.get(i).getName();
		if (stateFilter.filter(name)) {
			RocksIteratorWrapper iterator = new RocksIteratorWrapper(
					this.db.newIterator(stateColumnFamilyHandles.get(i + 1)));
			iterators.put(name, iterator);
			iterator.seekToFirst();
		}
	}

	iteratorQueue = new LinkedList<>(iterators.entrySet());
	updateCurrentIterator();
}
 

private void updateCurrentIterator() {
	IOUtils.closeQuietly(currentIterator);
	if (iteratorQueue.isEmpty()) {
		currentIterator = null;
		currentName = null;
		return;
	} else {
		Entry<String, RocksIteratorWrapper> e = iteratorQueue.pop();
		currentName = e.getKey();
		currentIterator = e.getValue();
	}
}
 

/**
 * Advances the iterator. Should only be called if {@link #isValid()} returned true.
 * Valid flag can only change after calling {@link #next()}.
 */
public void next() {
	newKeyGroup = false;
	newKVState = false;

	final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator();
	rocksIterator.next();

	byte[] oldKey = currentSubIterator.getCurrentKey();
	if (rocksIterator.isValid()) {

		currentSubIterator.setCurrentKey(rocksIterator.key());

		if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) {
			heap.offer(currentSubIterator);
			currentSubIterator = heap.remove();
			newKVState = currentSubIterator.getIterator() != rocksIterator;
			detectNewKeyGroup(oldKey);
		}
	} else {
		IOUtils.closeQuietly(rocksIterator);

		if (heap.isEmpty()) {
			currentSubIterator = null;
			valid = false;
		} else {
			currentSubIterator = heap.remove();
			newKVState = true;
			detectNewKeyGroup(oldKey);
		}
	}
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

/**
 * Advances the iterator. Should only be called if {@link #isValid()} returned true.
 * Valid flag can only change after calling {@link #next()}.
 */
public void next() {
	newKeyGroup = false;
	newKVState = false;

	final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator();
	rocksIterator.next();

	byte[] oldKey = currentSubIterator.getCurrentKey();
	if (rocksIterator.isValid()) {

		currentSubIterator.setCurrentKey(rocksIterator.key());

		if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) {
			heap.offer(currentSubIterator);
			currentSubIterator = heap.remove();
			newKVState = currentSubIterator.getIterator() != rocksIterator;
			detectNewKeyGroup(oldKey);
		}
	} else {
		IOUtils.closeQuietly(rocksIterator);

		if (heap.isEmpty()) {
			currentSubIterator = null;
			valid = false;
		} else {
			currentSubIterator = heap.remove();
			newKVState = true;
			detectNewKeyGroup(oldKey);
		}
	}
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

/**
 * Advances the iterator. Should only be called if {@link #isValid()} returned true.
 * Valid flag can only change after calling {@link #next()}.
 */
public void next() {
	newKeyGroup = false;
	newKVState = false;

	final RocksIteratorWrapper rocksIterator = currentSubIterator.getIterator();
	rocksIterator.next();

	byte[] oldKey = currentSubIterator.getCurrentKey();
	if (rocksIterator.isValid()) {

		currentSubIterator.setCurrentKey(rocksIterator.key());

		if (isDifferentKeyGroup(oldKey, currentSubIterator.getCurrentKey())) {
			heap.offer(currentSubIterator);
			currentSubIterator = heap.remove();
			newKVState = currentSubIterator.getIterator() != rocksIterator;
			detectNewKeyGroup(oldKey);
		}
	} else {
		IOUtils.closeQuietly(rocksIterator);

		if (heap.isEmpty()) {
			currentSubIterator = null;
			valid = false;
		} else {
			currentSubIterator = heap.remove();
			newKVState = true;
			detectNewKeyGroup(oldKey);
		}
	}
}
 

@SuppressWarnings("unchecked")
private static RocksIteratorWrapper getRocksIterator(
	RocksDB db,
	ColumnFamilyHandle columnFamilyHandle,
	StateSnapshotTransformer<byte[]> stateSnapshotTransformer,
	ReadOptions readOptions) {
	RocksIterator rocksIterator = db.newIterator(columnFamilyHandle, readOptions);
	return stateSnapshotTransformer == null ?
		new RocksIteratorWrapper(rocksIterator) :
		new RocksTransformingIteratorWrapper(rocksIterator, stateSnapshotTransformer);
}
 

@Nonnull
public RocksIteratorWrapper getIterator() {
	return iterator;
}
 

/**
 * @param iterator underlying {@link RocksIteratorWrapper}
 * @param kvStateId Id of the K/V state to which this iterator belongs.
 */
RocksSingleStateIterator(@Nonnull RocksIteratorWrapper iterator, int kvStateId) {
	this.iterator = iterator;
	this.currentKey = iterator.key();
	this.kvStateId = kvStateId;
}
 

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = instanceBasePath.getAbsoluteFile().toPath().resolve(UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db, writeBatchSize)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle, tmpRestoreDBInfo.readOptions)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}
 

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}
 

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}
 

@Nonnull
public RocksIteratorWrapper getIterator() {
	return iterator;
}
 

/**
 * @param iterator underlying {@link RocksIteratorWrapper}
 * @param kvStateId Id of the K/V state to which this iterator belongs.
 */
RocksSingleStateIterator(@Nonnull RocksIteratorWrapper iterator, int kvStateId) {
	this.iterator = iterator;
	this.currentKey = iterator.key();
	this.kvStateId = kvStateId;
}