Java Code Examples for org.apache.flink.runtime.state.KeyGroupsStateHandle#getKeyGroupRange()

The following examples show how to use org.apache.flink.runtime.state.KeyGroupsStateHandle#getKeyGroupRange() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SavepointV1Serializer.java From Flink-CEPplus with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 2

Source File: SavepointV1Serializer.java From flink with Apache License 2.0

6 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 3

Source File: SavepointV2Serializer.java From Flink-CEPplus with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 4

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example 5

Source File: SavepointV2Serializer.java From flink with Apache License 2.0

5 votes

@VisibleForTesting
public static void serializeKeyedStateHandle(
		KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {

	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 6

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

5 votes

public static void compareKeyedState(
		Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
		Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for(KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
					InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
								deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}

Example 7

Source File: MetadataV2V3SerializerBase.java From flink with Apache License 2.0

5 votes

void serializeKeyedStateHandle(KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException {
	if (stateHandle == null) {
		dos.writeByte(NULL_HANDLE);
	} else if (stateHandle instanceof KeyGroupsStateHandle) {
		KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle;

		dos.writeByte(KEY_GROUPS_HANDLE);
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups());
		for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) {
			dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup));
		}
		serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos);
	} else if (stateHandle instanceof IncrementalRemoteKeyedStateHandle) {
		IncrementalRemoteKeyedStateHandle incrementalKeyedStateHandle =
			(IncrementalRemoteKeyedStateHandle) stateHandle;

		dos.writeByte(INCREMENTAL_KEY_GROUPS_HANDLE);

		dos.writeLong(incrementalKeyedStateHandle.getCheckpointId());
		dos.writeUTF(String.valueOf(incrementalKeyedStateHandle.getBackendIdentifier()));
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getStartKeyGroup());
		dos.writeInt(incrementalKeyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups());

		serializeStreamStateHandle(incrementalKeyedStateHandle.getMetaStateHandle(), dos);

		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getSharedState(), dos);
		serializeStreamStateHandleMap(incrementalKeyedStateHandle.getPrivateState(), dos);
	} else {
		throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass());
	}
}

Example 8

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

5 votes

static void compareKeyedState(
	Collection<KeyGroupsStateHandle> expectPartitionedKeyGroupState,
	Collection<? extends KeyedStateHandle> actualPartitionedKeyGroupState) throws Exception {

	KeyGroupsStateHandle expectedHeadOpKeyGroupStateHandle = expectPartitionedKeyGroupState.iterator().next();
	int expectedTotalKeyGroups = expectedHeadOpKeyGroupStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	int actualTotalKeyGroups = 0;
	for (KeyedStateHandle keyedStateHandle: actualPartitionedKeyGroupState) {
		assertTrue(keyedStateHandle instanceof KeyGroupsStateHandle);

		actualTotalKeyGroups += keyedStateHandle.getKeyGroupRange().getNumberOfKeyGroups();
	}

	assertEquals(expectedTotalKeyGroups, actualTotalKeyGroups);

	try (FSDataInputStream inputStream = expectedHeadOpKeyGroupStateHandle.openInputStream()) {
		for (int groupId : expectedHeadOpKeyGroupStateHandle.getKeyGroupRange()) {
			long offset = expectedHeadOpKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
			inputStream.seek(offset);
			int expectedKeyGroupState =
				InstantiationUtil.deserializeObject(inputStream, Thread.currentThread().getContextClassLoader());
			for (KeyedStateHandle oneActualKeyedStateHandle : actualPartitionedKeyGroupState) {

				assertTrue(oneActualKeyedStateHandle instanceof KeyGroupsStateHandle);

				KeyGroupsStateHandle oneActualKeyGroupStateHandle = (KeyGroupsStateHandle) oneActualKeyedStateHandle;
				if (oneActualKeyGroupStateHandle.getKeyGroupRange().contains(groupId)) {
					long actualOffset = oneActualKeyGroupStateHandle.getOffsetForKeyGroup(groupId);
					try (FSDataInputStream actualInputStream = oneActualKeyGroupStateHandle.openInputStream()) {
						actualInputStream.seek(actualOffset);
						int actualGroupState = InstantiationUtil.
							deserializeObject(actualInputStream, Thread.currentThread().getContextClassLoader());
						assertEquals(expectedKeyGroupState, actualGroupState);
					}
				}
			}
		}
	}
}