org.apache.flink.runtime.state.ChainedStateHandle Java Examples

The following examples show how to use org.apache.flink.runtime.state.ChainedStateHandle. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

6 votes

public static void verifyStateRestore(
	JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
	List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
			generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
			expectedOpStateBackend.get(0).openInputStream(),
			operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
			jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #2

Source File: CheckpointCoordinatorTestingUtils.java From flink with Apache License 2.0

6 votes

static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
	JobVertexID jobVertexID,
	int index,
	int namedStates,
	int partitionsPerState,
	boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}

Example #3

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}

Example #4

Source File: CheckpointCoordinatorTest.java From Flink-CEPplus with Apache License 2.0

6 votes

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isSteamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #5

Source File: SubtaskState.java From flink with Apache License 2.0

6 votes

public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}

Example #6

Source File: SubtaskState.java From Flink-CEPplus with Apache License 2.0

6 votes

public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}

Example #7

Source File: SubtaskState.java From flink with Apache License 2.0

6 votes

public SubtaskState(
		ChainedStateHandle<OperatorStateHandle> managedOperatorState,
		ChainedStateHandle<OperatorStateHandle> rawOperatorState,
		KeyedStateHandle managedKeyedState,
		KeyedStateHandle rawKeyedState) {

	this.managedOperatorState = managedOperatorState;
	this.rawOperatorState = rawOperatorState;
	this.managedKeyedState = managedKeyedState;
	this.rawKeyedState = rawKeyedState;

	try {
		long calculateStateSize = getSizeNullSafe(managedOperatorState);
		calculateStateSize += getSizeNullSafe(rawOperatorState);
		calculateStateSize += getSizeNullSafe(managedKeyedState);
		calculateStateSize += getSizeNullSafe(rawKeyedState);
		stateSize = calculateStateSize;
	} catch (Exception e) {
		throw new RuntimeException("Failed to get state size.", e);
	}
}

Example #8

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

public static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(
		JobVertexID jobVertexID,
		int index,
		int namedStates,
		int partitionsPerState,
		boolean rawState) throws IOException {

	Map<String, List<? extends Serializable>> statesListsMap = new HashMap<>(namedStates);

	for (int i = 0; i < namedStates; ++i) {
		List<Integer> testStatesLists = new ArrayList<>(partitionsPerState);
		// generate state
		int seed = jobVertexID.hashCode() * index + i * namedStates;
		if (rawState) {
			seed = (seed + 1) * 31;
		}
		Random random = new Random(seed);
		for (int j = 0; j < partitionsPerState; ++j) {
			int simulatedStateValue = random.nextInt();
			testStatesLists.add(simulatedStateValue);
		}
		statesListsMap.put("state-" + i, testStatesLists);
	}

	return ChainedStateHandle.wrapSingleHandle(generatePartitionableStateHandle(statesListsMap));
}

Example #9

Source File: CheckpointCoordinatorTest.java From flink with Apache License 2.0

6 votes

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}

Example #10

Source File: SubtaskState.java From flink with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}

Example #11

Source File: SubtaskState.java From flink with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}

Example #12

Source File: SubtaskState.java From flink with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}

Example #13

Source File: CheckpointTestUtils.java From flink with Apache License 2.0

4 votes

/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}

Example #14

Source File: SubtaskState.java From flink with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}

Example #15

Source File: CheckpointTestUtils.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Creates a random collection of TaskState objects containing various types of state handles.
 */
public static Collection<TaskState> createTaskStates(
		Random random,
		int numTaskStates,
		int numSubtasksPerTask) {

	List<TaskState> taskStates = new ArrayList<>(numTaskStates);

	for (int stateIdx = 0; stateIdx < numTaskStates; ++stateIdx) {

		int chainLength = 1 + random.nextInt(8);

		TaskState taskState = new TaskState(new JobVertexID(), numSubtasksPerTask, 128, chainLength);

		int noNonPartitionableStateAtIndex = random.nextInt(chainLength);
		int noOperatorStateBackendAtIndex = random.nextInt(chainLength);
		int noOperatorStateStreamAtIndex = random.nextInt(chainLength);

		boolean hasKeyedBackend = random.nextInt(4) != 0;
		boolean hasKeyedStream = random.nextInt(4) != 0;

		for (int subtaskIdx = 0; subtaskIdx < numSubtasksPerTask; subtaskIdx++) {

			List<OperatorStateHandle> operatorStatesBackend = new ArrayList<>(chainLength);
			List<OperatorStateHandle> operatorStatesStream = new ArrayList<>(chainLength);

			for (int chainIdx = 0; chainIdx < chainLength; ++chainIdx) {

				StreamStateHandle operatorStateBackend =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				StreamStateHandle operatorStateStream =
						new ByteStreamStateHandle("b-" + chainIdx, ("Beautiful-" + chainIdx).getBytes(ConfigConstants.DEFAULT_CHARSET));
				Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
				offsetsMap.put("A", new OperatorStateHandle.StateMetaInfo(new long[]{0, 10, 20}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("B", new OperatorStateHandle.StateMetaInfo(new long[]{30, 40, 50}, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
				offsetsMap.put("C", new OperatorStateHandle.StateMetaInfo(new long[]{60, 70, 80}, OperatorStateHandle.Mode.UNION));

				if (chainIdx != noOperatorStateBackendAtIndex) {
					OperatorStateHandle operatorStateHandleBackend =
							new OperatorStreamStateHandle(offsetsMap, operatorStateBackend);
					operatorStatesBackend.add(operatorStateHandleBackend);
				}

				if (chainIdx != noOperatorStateStreamAtIndex) {
					OperatorStateHandle operatorStateHandleStream =
							new OperatorStreamStateHandle(offsetsMap, operatorStateStream);
					operatorStatesStream.add(operatorStateHandleStream);
				}
			}

			KeyGroupsStateHandle keyedStateBackend = null;
			KeyGroupsStateHandle keyedStateStream = null;

			if (hasKeyedBackend) {
				keyedStateBackend = createDummyKeyGroupStateHandle(random);
			}

			if (hasKeyedStream) {
				keyedStateStream = createDummyKeyGroupStateHandle(random);
			}

			taskState.putState(subtaskIdx, new SubtaskState(
					new ChainedStateHandle<>(operatorStatesBackend),
					new ChainedStateHandle<>(operatorStatesStream),
					keyedStateStream,
					keyedStateBackend));
		}

		taskStates.add(taskState);
	}

	return taskStates;
}

Example #16

Source File: SubtaskState.java From Flink-CEPplus with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getRawOperatorState() {
	return rawOperatorState;
}

Example #17

Source File: SubtaskState.java From Flink-CEPplus with Apache License 2.0

4 votes

public ChainedStateHandle<OperatorStateHandle> getManagedOperatorState() {
	return managedOperatorState;
}