Java Code Examples for org.apache.flink.runtime.state.KeyedStateHandle#getClass()

The following examples show how to use org.apache.flink.runtime.state.KeyedStateHandle#getClass() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Recovery from a single remote incremental state without rescaling.
 */
private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception {
	if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle);
		restoreFromRemoteState(incrementalRemoteKeyedStateHandle);
	} else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) {
		IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle =
			(IncrementalLocalKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle);
		restoreFromLocalState(incrementalLocalKeyedStateHandle);
	} else {
		throw new BackendBuildingException("Unexpected state handle type, " +
			"expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class +
			", but found " + keyedStateHandle.getClass());
	}
}

Example 2

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example 3

Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Recovery from a single remote incremental state without rescaling.
 */
private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception {
	if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle);
		restoreFromRemoteState(incrementalRemoteKeyedStateHandle);
	} else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) {
		IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle =
			(IncrementalLocalKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle);
		restoreFromLocalState(incrementalLocalKeyedStateHandle);
	} else {
		throw new BackendBuildingException("Unexpected state handle type, " +
			"expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class +
			", but found " + keyedStateHandle.getClass());
	}
}

Example 4

Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example 5

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example 6

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 7

Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Recovery from a single remote incremental state without rescaling.
 */
private void restoreWithoutRescaling(KeyedStateHandle keyedStateHandle) throws Exception {
	if (keyedStateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalRemoteKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalRemoteKeyedStateHandle);
		restoreFromRemoteState(incrementalRemoteKeyedStateHandle);
	} else if (keyedStateHandle instanceof IncrementalLocalKeyedStateHandle) {
		IncrementalLocalKeyedStateHandle incrementalLocalKeyedStateHandle =
			(IncrementalLocalKeyedStateHandle) keyedStateHandle;
		restorePreviousIncrementalFilesStatus(incrementalLocalKeyedStateHandle);
		restoreFromLocalState(incrementalLocalKeyedStateHandle);
	} else {
		throw new BackendBuildingException("Unexpected state handle type, " +
			"expected " + IncrementalRemoteKeyedStateHandle.class + " or " + IncrementalLocalKeyedStateHandle.class +
			", but found " + keyedStateHandle.getClass());
	}
}

Example 8

Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example 9

Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example 10

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 11

Source File: RocksDBFullRestoreOperation.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example 12

Source File: SavepointV2Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 13

Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 14

Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0

5 votes

void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 15

Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0

4 votes

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}

Example 16

Source File: HeapRestoreOperation.java From Flink-CEPplus with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}

Example 17

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();
	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}

Example 18

Source File: RocksDBIncrementalRestoreOperation.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = new Path(instanceBasePath.getAbsolutePath() + UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}

Example 19

Source File: RocksDBIncrementalRestoreOperation.java From flink with Apache License 2.0

4 votes

/**
 * Recovery from multi incremental states with rescaling. For rescaling, this method creates a temporary
 * RocksDB instance for a key-groups shard. All contents from the temporary instance are copied into the
 * real restore instance and then the temporary instance is discarded.
 */
private void restoreWithRescaling(Collection<KeyedStateHandle> restoreStateHandles) throws Exception {

	// Prepare for restore with rescaling
	KeyedStateHandle initialHandle = RocksDBIncrementalCheckpointUtils.chooseTheBestStateHandleForInitial(
		restoreStateHandles, keyGroupRange);

	// Init base DB instance
	if (initialHandle != null) {
		restoreStateHandles.remove(initialHandle);
		initDBWithRescaling(initialHandle);
	} else {
		openDB();
	}

	// Transfer remaining key-groups from temporary instance into base DB
	byte[] startKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getStartKeyGroup(), startKeyGroupPrefixBytes);

	byte[] stopKeyGroupPrefixBytes = new byte[keyGroupPrefixBytes];
	RocksDBKeySerializationUtils.serializeKeyGroup(keyGroupRange.getEndKeyGroup() + 1, stopKeyGroupPrefixBytes);

	for (KeyedStateHandle rawStateHandle : restoreStateHandles) {

		if (!(rawStateHandle instanceof IncrementalRemoteKeyedStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected " + IncrementalRemoteKeyedStateHandle.class +
				", but found " + rawStateHandle.getClass());
		}

		Path temporaryRestoreInstancePath = instanceBasePath.getAbsoluteFile().toPath().resolve(UUID.randomUUID().toString());
		try (RestoredDBInstance tmpRestoreDBInfo = restoreDBInstanceFromStateHandle(
			(IncrementalRemoteKeyedStateHandle) rawStateHandle,
			temporaryRestoreInstancePath);
			RocksDBWriteBatchWrapper writeBatchWrapper = new RocksDBWriteBatchWrapper(this.db, writeBatchSize)) {

			List<ColumnFamilyDescriptor> tmpColumnFamilyDescriptors = tmpRestoreDBInfo.columnFamilyDescriptors;
			List<ColumnFamilyHandle> tmpColumnFamilyHandles = tmpRestoreDBInfo.columnFamilyHandles;

			// iterating only the requested descriptors automatically skips the default column family handle
			for (int i = 0; i < tmpColumnFamilyDescriptors.size(); ++i) {
				ColumnFamilyHandle tmpColumnFamilyHandle = tmpColumnFamilyHandles.get(i);

				ColumnFamilyHandle targetColumnFamilyHandle = getOrRegisterStateColumnFamilyHandle(
					null, tmpRestoreDBInfo.stateMetaInfoSnapshots.get(i))
					.columnFamilyHandle;

				try (RocksIteratorWrapper iterator = RocksDBOperationUtils.getRocksIterator(tmpRestoreDBInfo.db, tmpColumnFamilyHandle, tmpRestoreDBInfo.readOptions)) {

					iterator.seek(startKeyGroupPrefixBytes);

					while (iterator.isValid()) {

						if (RocksDBIncrementalCheckpointUtils.beforeThePrefixBytes(iterator.key(), stopKeyGroupPrefixBytes)) {
							writeBatchWrapper.put(targetColumnFamilyHandle, iterator.key(), iterator.value());
						} else {
							// Since the iterator will visit the record according to the sorted order,
							// we can just break here.
							break;
						}

						iterator.next();
					}
				} // releases native iterator resources
			}
		} finally {
			cleanUpPathQuietly(temporaryRestoreInstancePath);
		}
	}
}

Example 20

Source File: HeapRestoreOperation.java From flink with Apache License 2.0

4 votes

@Override
public Void restore() throws Exception {

	registeredKVStates.clear();
	registeredPQStates.clear();

	boolean keySerializerRestored = false;

	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {

		if (keyedStateHandle == null) {
			continue;
		}

		if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
			throw new IllegalStateException("Unexpected state handle type, " +
				"expected: " + KeyGroupsStateHandle.class +
				", but found: " + keyedStateHandle.getClass());
		}

		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
		FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream();
		cancelStreamRegistry.registerCloseable(fsDataInputStream);

		try {
			DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream);

			KeyedBackendSerializationProxy<K> serializationProxy =
				new KeyedBackendSerializationProxy<>(userCodeClassLoader);

			serializationProxy.read(inView);

			if (!keySerializerRestored) {
				// check for key serializer compatibility; this also reconfigures the
				// key serializer to be compatible, if it is required and is possible
				TypeSerializerSchemaCompatibility<K> keySerializerSchemaCompat =
					keySerializerProvider.setPreviousSerializerSnapshotForRestoredState(serializationProxy.getKeySerializerSnapshot());
				if (keySerializerSchemaCompat.isCompatibleAfterMigration() || keySerializerSchemaCompat.isIncompatible()) {
					throw new StateMigrationException("The new key serializer must be compatible.");
				}

				keySerializerRestored = true;
			}

			List<StateMetaInfoSnapshot> restoredMetaInfos =
				serializationProxy.getStateMetaInfoSnapshots();

			final Map<Integer, StateMetaInfoSnapshot> kvStatesById = new HashMap<>();

			createOrCheckStateForMetaInfo(restoredMetaInfos, kvStatesById);

			readStateHandleStateData(
				fsDataInputStream,
				inView,
				keyGroupsStateHandle.getGroupRangeOffsets(),
				kvStatesById, restoredMetaInfos.size(),
				serializationProxy.getReadVersion(),
				serializationProxy.isUsingKeyGroupCompression());
		} finally {
			if (cancelStreamRegistry.unregisterCloseable(fsDataInputStream)) {
				IOUtils.closeQuietly(fsDataInputStream);
			}
		}
	}
	return null;
}