org.apache.flink.runtime.state.KeyGroupsStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.KeyGroupsStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

6 votes

public static void verifyStateRestore(
	JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
	List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
			generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
			expectedOpStateBackend.get(0).openInputStream(),
			operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
			jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #2

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #3

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

static TaskStateSnapshot mockSubtaskState(
	JobVertexID jobVertexID,
	int index,
	KeyGroupRange keyGroupRange) throws IOException {

	OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
	KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);

	TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot());
	OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState(
		partitionableState, null, partitionedKeyGroupState, null)
	);

	subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState);

	return subtaskStates;
}

Example #4

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

public static KeyGroupsStateHandle generateKeyGroupState(
		KeyGroupRange keyGroupRange,
		List<? extends Serializable> states) throws IOException {

	Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());

	Tuple2<byte[], List<long[]>> serializedDataWithOffsets =
			serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));

	KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));

	ByteStreamStateHandle allSerializedStatesHandle = new ByteStreamStateHandle(
			String.valueOf(UUID.randomUUID()),
			serializedDataWithOffsets.f0);

	return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}

Example #5

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

public static KeyGroupsStateHandle generateKeyGroupState(
		JobVertexID jobVertexID,
		KeyGroupRange keyGroupPartition, boolean rawState) throws IOException {

	List<Integer> testStatesLists = new ArrayList<>(keyGroupPartition.getNumberOfKeyGroups());

	// generate state for one keygroup
	for (int keyGroupIndex : keyGroupPartition) {
		int vertexHash = jobVertexID.hashCode();
		int seed = rawState ? (vertexHash * (31 + keyGroupIndex)) : (vertexHash + keyGroupIndex);
		Random random = new Random(seed);
		int simulatedStateValue = random.nextInt();
		testStatesLists.add(simulatedStateValue);
	}

	return generateKeyGroupState(keyGroupPartition, testStatesLists);
}

Example #6

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example #7

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

6 votes

@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}

Example #8

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example #9

Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example #10

Source File: RocksDBFullRestoreOperation.java From flink with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example #11

Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example #12

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

6 votes

static TaskStateSnapshot mockSubtaskState(
	JobVertexID jobVertexID,
	int index,
	KeyGroupRange keyGroupRange) throws IOException {

	OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
	KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);

	TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot());
	OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState(
		partitionableState, null, partitionedKeyGroupState, null, null, null)
	);

	subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState);

	return subtaskStates;
}

Example #13

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isSteamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #14

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

static TaskStateSnapshot mockSubtaskState(
	JobVertexID jobVertexID,
	int index,
	KeyGroupRange keyGroupRange) throws IOException {

	OperatorStateHandle partitionableState = generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
	KeyGroupsStateHandle partitionedKeyGroupState = generateKeyGroupState(jobVertexID, keyGroupRange, false);

	TaskStateSnapshot subtaskStates = spy(new TaskStateSnapshot());
	OperatorSubtaskState subtaskState = spy(new OperatorSubtaskState(
		partitionableState, null, partitionedKeyGroupState, null)
	);

	subtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), subtaskState);

	return subtaskStates;
}

Example #15

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

public static KeyGroupsStateHandle generateKeyGroupState(
		KeyGroupRange keyGroupRange,
		List<? extends Serializable> states) throws IOException {

	Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());

	Tuple2<byte[], List<long[]>> serializedDataWithOffsets =
			serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));

	KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));

	ByteStreamStateHandle allSerializedStatesHandle = new ByteStreamStateHandle(
			String.valueOf(UUID.randomUUID()),
			serializedDataWithOffsets.f0);

	return new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
}

Example #16

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

public static KeyGroupsStateHandle generateKeyGroupState(
		JobVertexID jobVertexID,
		KeyGroupRange keyGroupPartition, boolean rawState) throws IOException {

	List<Integer> testStatesLists = new ArrayList<>(keyGroupPartition.getNumberOfKeyGroups());

	// generate state for one keygroup
	for (int keyGroupIndex : keyGroupPartition) {
		int vertexHash = jobVertexID.hashCode();
		int seed = rawState ? (vertexHash * (31 + keyGroupIndex)) : (vertexHash + keyGroupIndex);
		Random random = new Random(seed);
		int simulatedStateValue = random.nextInt();
		testStatesLists.add(simulatedStateValue);
	}

	return generateKeyGroupState(keyGroupPartition, testStatesLists);
}

Example #17

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

6 votes

public static KeyGroupsStateHandle generateKeyGroupState(
	JobVertexID jobVertexID,
	KeyGroupRange keyGroupPartition, boolean rawState) throws IOException {

	List<Integer> testStatesLists = new ArrayList<>(keyGroupPartition.getNumberOfKeyGroups());

	// generate state for one keygroup
	for (int keyGroupIndex : keyGroupPartition) {
		int vertexHash = jobVertexID.hashCode();
		int seed = rawState ? (vertexHash * (31 + keyGroupIndex)) : (vertexHash + keyGroupIndex);
		Random random = new Random(seed);
		int simulatedStateValue = random.nextInt();
		testStatesLists.add(simulatedStateValue);
	}

	return generateKeyGroupState(keyGroupPartition, testStatesLists);
}

Example #18

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

6 votes

private static Collection<KeyGroupsStateHandle> transform(Collection<KeyedStateHandle> keyedStateHandles) {

		if (keyedStateHandles == null) {
			return null;
		}

		List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(keyedStateHandles.size());

		for (KeyedStateHandle keyedStateHandle : keyedStateHandles) {

			if (keyedStateHandle instanceof KeyGroupsStateHandle) {
				keyGroupsStateHandles.add((KeyGroupsStateHandle) keyedStateHandle);
			} else if (keyedStateHandle != null) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass() + ".");
			}
		}

		return keyGroupsStateHandles;
	}

Example #19

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

6 votes

@VisibleForTesting
public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException {
	final int type = dis.readByte();
	if (NULL_HANDLE == type) {
		return null;
	} else if (KEY_GROUPS_HANDLE == type) {
		int startKeyGroup = dis.readInt();
		int numKeyGroups = dis.readInt();
		KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1);
		long[] offsets = new long[numKeyGroups];
		for (int i = 0; i < numKeyGroups; ++i) {
			offsets[i] = dis.readLong();
		}
		KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(
			keyGroupRange, offsets);
		StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
		return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle);
	} else {
		throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type);
	}
}

Example #20

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example #21

Source File: RocksDBFullRestoreOperation.java From Flink-CEPplus with Apache License 2.0

6 votes

/**
 * Restores all key-groups data that is referenced by the passed state handles.
 *
 */
@Override
public RocksDBRestoreResult restore()
	throws IOException, StateMigrationException, RocksDBException {
	openDB();
	for (KeyedStateHandle keyedStateHandle : restoreStateHandles) {
		if (keyedStateHandle != null) {

			if (!(keyedStateHandle instanceof KeyGroupsStateHandle)) {
				throw new IllegalStateException("Unexpected state handle type, " +
					"expected: " + KeyGroupsStateHandle.class +
					", but found: " + keyedStateHandle.getClass());
			}
			this.currentKeyGroupsStateHandle = (KeyGroupsStateHandle) keyedStateHandle;
			restoreKeyGroupsInStateHandle();
		}
	}
	return new RocksDBRestoreResult(this.db, defaultColumnFamilyHandle, nativeMetricMonitor,
		-1, null, null);
}

Example #22

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

5 votes

static void compareKeyedState(
	Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
	Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for (KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
				InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
							deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example #23

Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example #24

Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0

5 votes

void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example #25

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

5 votes

protected CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs(
	Iterator<StateObjectCollection<KeyedStateHandle>> restoreStateAlternatives) {

	if (restoreStateAlternatives.hasNext()) {
		Collection<KeyedStateHandle> rawKeyedState = restoreStateAlternatives.next();

		// TODO currently this does not support local state recovery, so we expect there is only one handle.
		Preconditions.checkState(
			!restoreStateAlternatives.hasNext(),
			"Local recovery is currently not implemented for raw keyed state, but found state alternative.");

		if (rawKeyedState != null) {
			Collection<KeyGroupsStateHandle> keyGroupsStateHandles = transform(rawKeyedState);
			final CloseableRegistry closeableRegistry = new CloseableRegistry();

			return new CloseableIterable<KeyGroupStatePartitionStreamProvider>() {
				@Override
				public void close() throws IOException {
					closeableRegistry.close();
				}

				@Override
				public Iterator<KeyGroupStatePartitionStreamProvider> iterator() {
					return new KeyGroupStreamIterator(keyGroupsStateHandles.iterator(), closeableRegistry);
				}
			};
		}
	}

	return CloseableIterable.empty();
}

Example #26

Source File: StreamTaskStateInitializerImpl.java From flink with Apache License 2.0

5 votes

protected CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs(
	Iterator<StateObjectCollection<KeyedStateHandle>> restoreStateAlternatives) {

	if (restoreStateAlternatives.hasNext()) {
		Collection<KeyedStateHandle> rawKeyedState = restoreStateAlternatives.next();

		// TODO currently this does not support local state recovery, so we expect there is only one handle.
		Preconditions.checkState(
			!restoreStateAlternatives.hasNext(),
			"Local recovery is currently not implemented for raw keyed state, but found state alternative.");

		if (rawKeyedState != null) {
			Collection<KeyGroupsStateHandle> keyGroupsStateHandles = transform(rawKeyedState);
			final CloseableRegistry closeableRegistry = new CloseableRegistry();

			return new CloseableIterable<KeyGroupStatePartitionStreamProvider>() {
				@Override
				public void close() throws IOException {
					closeableRegistry.close();
				}

				@Override
				public Iterator<KeyGroupStatePartitionStreamProvider> iterator() {
					return new KeyGroupStreamIterator(keyGroupsStateHandles.iterator(), closeableRegistry);
				}
			};
		}
	}

	return CloseableIterable.empty();
}

Example #27

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example #28

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example #29

Source File: SavepointV2Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example #30

Source File: StreamTaskStateInitializerImpl.java From Flink-CEPplus with Apache License 2.0

5 votes

protected CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs(
	Iterator<StateObjectCollection<KeyedStateHandle>> restoreStateAlternatives) {

	if (restoreStateAlternatives.hasNext()) {
		Collection<KeyedStateHandle> rawKeyedState = restoreStateAlternatives.next();

		// TODO currently this does not support local state recovery, so we expect there is only one handle.
		Preconditions.checkState(
			!restoreStateAlternatives.hasNext(),
			"Local recovery is currently not implemented for raw keyed state, but found state alternative.");

		if (rawKeyedState != null) {
			Collection<KeyGroupsStateHandle> keyGroupsStateHandles = transform(rawKeyedState);
			final CloseableRegistry closeableRegistry = new CloseableRegistry();

			return new CloseableIterable<KeyGroupStatePartitionStreamProvider>() {
				@Override
				public void close() throws IOException {
					closeableRegistry.close();
				}

				@Override
				public Iterator<KeyGroupStatePartitionStreamProvider> iterator() {
					return new KeyGroupStreamIterator(keyGroupsStateHandles.iterator(), closeableRegistry);
				}
			};
		}
	}

	return CloseableIterable.empty();
}