Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionJobVertex#getParallelism()

public static void verifyStateRestore(
	JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
	List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
			generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
			expectedOpStateBackend.get(0).openInputStream(),
			operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
			jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
 

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isSteamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
 

public static void verifyStateRestore(
		JobVertexID jobVertexID, ExecutionJobVertex executionJobVertex,
		List<KeyGroupRange> keyGroupPartitions) throws Exception {

	for (int i = 0; i < executionJobVertex.getParallelism(); i++) {

		JobManagerTaskRestore taskRestore = executionJobVertex.getTaskVertices()[i].getCurrentExecutionAttempt().getTaskRestore();
		Assert.assertEquals(1L, taskRestore.getRestoreCheckpointId());
		TaskStateSnapshot stateSnapshot = taskRestore.getTaskStateSnapshot();

		OperatorSubtaskState operatorState = stateSnapshot.getSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID));

		ChainedStateHandle<OperatorStateHandle> expectedOpStateBackend =
				generateChainedPartitionableStateHandle(jobVertexID, i, 2, 8, false);

		assertTrue(CommonTestUtils.isStreamContentEqual(
				expectedOpStateBackend.get(0).openInputStream(),
				operatorState.getManagedOperatorState().iterator().next().openInputStream()));

		KeyGroupsStateHandle expectPartitionedKeyGroupState = generateKeyGroupState(
				jobVertexID, keyGroupPartitions.get(i), false);
		compareKeyedState(Collections.singletonList(expectPartitionedKeyGroupState), operatorState.getManagedKeyedState());
	}
}
 

/**
 * Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
 *
 * @param operatorState      state to restore
 * @param executionJobVertex task for which the state should be restored
 */
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
	//----------------------------------------max parallelism preconditions-------------------------------------

	if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
		throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
			" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
			") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
			"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
		);
	}

	// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
	if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {

		if (!executionJobVertex.isMaxParallelismConfigured()) {
			// if the max parallelism was not explicitly specified by the user, we derive it from the state

			LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
				executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());

			executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
		} else {
			// if the max parallelism was explicitly specified, we complain on mismatch
			throw new IllegalStateException("The maximum parallelism (" +
				operatorState.getMaxParallelism() + ") with which the latest " +
				"checkpoint of the execution job vertex " + executionJobVertex +
				" has been taken and the current maximum parallelism (" +
				executionJobVertex.getMaxParallelism() + ") changed. This " +
				"is currently not supported.");
		}
	}
}
 

/**
 * Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
 *
 * @param operatorState      state to restore
 * @param executionJobVertex task for which the state should be restored
 */
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
	//----------------------------------------max parallelism preconditions-------------------------------------

	if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
		throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
			" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
			") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
			"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
		);
	}

	// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
	if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {

		if (!executionJobVertex.isMaxParallelismConfigured()) {
			// if the max parallelism was not explicitly specified by the user, we derive it from the state

			LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
				executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());

			executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
		} else {
			// if the max parallelism was explicitly specified, we complain on mismatch
			throw new IllegalStateException("The maximum parallelism (" +
				operatorState.getMaxParallelism() + ") with which the latest " +
				"checkpoint of the execution job vertex " + executionJobVertex +
				" has been taken and the current maximum parallelism (" +
				executionJobVertex.getMaxParallelism() + ") changed. This " +
				"is currently not supported.");
		}
	}
}
 

/**
 * Creates an empty map with a {@link TaskStateStats} instance per task
 * that is involved in the checkpoint.
 *
 * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
 */
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
	ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
	for (ExecutionJobVertex vertex : jobVertices) {
		TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
		taskStatsMap.put(vertex.getJobVertexId(), taskStats);
	}
	return taskStatsMap;
}
 

/**
 * Creates a new checkpoint stats tracker.
 *
 * @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
 * @param jobVertices Job vertices involved in the checkpoints.
 * @param jobCheckpointingConfiguration Checkpointing configuration.
 * @param metricGroup Metric group for exposed metrics
 */
public CheckpointStatsTracker(
	int numRememberedCheckpoints,
	List<ExecutionJobVertex> jobVertices,
	CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
	MetricGroup metricGroup) {

	checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
	this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
	this.jobVertices = checkNotNull(jobVertices, "JobVertices");
	this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);

	// Compute the total subtask count. We do this here in order to only
	// do it once.
	int count = 0;
	for (ExecutionJobVertex vertex : jobVertices) {
		count += vertex.getParallelism();
	}
	this.totalSubtaskCount = count;

	// Latest snapshot is empty
	latestSnapshot = new CheckpointStatsSnapshot(
		counts.createSnapshot(),
		summary.createSnapshot(),
		history.createSnapshot(),
		null);

	// Register the metrics
	registerMetrics(metricGroup);
}
 

/**
 * Creates a new checkpoint stats tracker.
 *
 * @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
 * @param jobVertices Job vertices involved in the checkpoints.
 * @param jobCheckpointingConfiguration Checkpointing configuration.
 * @param metricGroup Metric group for exposed metrics
 */
public CheckpointStatsTracker(
	int numRememberedCheckpoints,
	List<ExecutionJobVertex> jobVertices,
	CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
	MetricGroup metricGroup) {

	checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
	this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
	this.jobVertices = checkNotNull(jobVertices, "JobVertices");
	this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);

	// Compute the total subtask count. We do this here in order to only
	// do it once.
	int count = 0;
	for (ExecutionJobVertex vertex : jobVertices) {
		count += vertex.getParallelism();
	}
	this.totalSubtaskCount = count;

	// Latest snapshot is empty
	latestSnapshot = new CheckpointStatsSnapshot(
		counts.createSnapshot(),
		summary.createSnapshot(),
		history.createSnapshot(),
		null);

	// Register the metrics
	registerMetrics(metricGroup);
}
 

/**
 * Creates an empty map with a {@link TaskStateStats} instance per task
 * that is involved in the checkpoint.
 *
 * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
 */
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
	ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
	for (ExecutionJobVertex vertex : jobVertices) {
		TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
		taskStatsMap.put(vertex.getJobVertexId(), taskStats);
	}
	return taskStatsMap;
}
 

/**
 * Creates a new checkpoint stats tracker.
 *
 * @param numRememberedCheckpoints Maximum number of checkpoints to remember, including in progress ones.
 * @param jobVertices Job vertices involved in the checkpoints.
 * @param jobCheckpointingConfiguration Checkpointing configuration.
 * @param metricGroup Metric group for exposed metrics
 */
public CheckpointStatsTracker(
	int numRememberedCheckpoints,
	List<ExecutionJobVertex> jobVertices,
	CheckpointCoordinatorConfiguration jobCheckpointingConfiguration,
	MetricGroup metricGroup) {

	checkArgument(numRememberedCheckpoints >= 0, "Negative number of remembered checkpoints");
	this.history = new CheckpointStatsHistory(numRememberedCheckpoints);
	this.jobVertices = checkNotNull(jobVertices, "JobVertices");
	this.jobCheckpointingConfiguration = checkNotNull(jobCheckpointingConfiguration);

	// Compute the total subtask count. We do this here in order to only
	// do it once.
	int count = 0;
	for (ExecutionJobVertex vertex : jobVertices) {
		count += vertex.getParallelism();
	}
	this.totalSubtaskCount = count;

	// Latest snapshot is empty
	latestSnapshot = new CheckpointStatsSnapshot(
		counts.createSnapshot(),
		summary.createSnapshot(),
		history.createSnapshot(),
		null);

	// Register the metrics
	registerMetrics(metricGroup);
}
 

/**
 * Verifies conditions in regards to parallelism and maxParallelism that must be met when restoring state.
 *
 * @param operatorState      state to restore
 * @param executionJobVertex task for which the state should be restored
 */
private static void checkParallelismPreconditions(OperatorState operatorState, ExecutionJobVertex executionJobVertex) {
	//----------------------------------------max parallelism preconditions-------------------------------------

	if (operatorState.getMaxParallelism() < executionJobVertex.getParallelism()) {
		throw new IllegalStateException("The state for task " + executionJobVertex.getJobVertexId() +
			" can not be restored. The maximum parallelism (" + operatorState.getMaxParallelism() +
			") of the restored state is lower than the configured parallelism (" + executionJobVertex.getParallelism() +
			"). Please reduce the parallelism of the task to be lower or equal to the maximum parallelism."
		);
	}

	// check that the number of key groups have not changed or if we need to override it to satisfy the restored state
	if (operatorState.getMaxParallelism() != executionJobVertex.getMaxParallelism()) {

		if (!executionJobVertex.isMaxParallelismConfigured()) {
			// if the max parallelism was not explicitly specified by the user, we derive it from the state

			LOG.debug("Overriding maximum parallelism for JobVertex {} from {} to {}",
				executionJobVertex.getJobVertexId(), executionJobVertex.getMaxParallelism(), operatorState.getMaxParallelism());

			executionJobVertex.setMaxParallelism(operatorState.getMaxParallelism());
		} else {
			// if the max parallelism was explicitly specified, we complain on mismatch
			throw new IllegalStateException("The maximum parallelism (" +
				operatorState.getMaxParallelism() + ") with which the latest " +
				"checkpoint of the execution job vertex " + executionJobVertex +
				" has been taken and the current maximum parallelism (" +
				executionJobVertex.getMaxParallelism() + ") changed. This " +
				"is currently not supported.");
		}
	}
}
 

/**
 * Creates an empty map with a {@link TaskStateStats} instance per task
 * that is involved in the checkpoint.
 *
 * @return An empty map with an {@link TaskStateStats} entry for each task that is involved in the checkpoint.
 */
private ConcurrentHashMap<JobVertexID, TaskStateStats> createEmptyTaskStateStatsMap() {
	ConcurrentHashMap<JobVertexID, TaskStateStats> taskStatsMap = new ConcurrentHashMap<>(jobVertices.size());
	for (ExecutionJobVertex vertex : jobVertices) {
		TaskStateStats taskStats = new TaskStateStats(vertex.getJobVertexId(), vertex.getParallelism());
		taskStatsMap.put(vertex.getJobVertexId(), taskStats);
	}
	return taskStatsMap;
}
 

private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {

		List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();

		//1. first compute the new parallelism
		checkParallelismPreconditions(operatorStates, executionJobVertex);

		int newParallelism = executionJobVertex.getParallelism();

		List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
			executionJobVertex.getMaxParallelism(),
			newParallelism);

		final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();

		/*
		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
		 *
		 * The old ManagedOperatorStates with old parallelism 3:
		 *
		 * 		parallelism0 parallelism1 parallelism2
		 * op0   states0,0    state0,1	   state0,2
		 * op1
		 * op2   states2,0    state2,1	   state1,2
		 * op3   states3,0    state3,1     state3,2
		 *
		 * The new ManagedOperatorStates with new parallelism 4:
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   state0,0	  state0,1 	   state0,2		state0,3
		 * op1
		 * op2   state2,0	  state2,1 	   state2,2		state2,3
		 * op3   state3,0	  state3,1 	   state3,2		state3,3
		 */
		Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
			new HashMap<>(expectedNumberOfSubTasks);
		Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
			new HashMap<>(expectedNumberOfSubTasks);

		reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			newManagedOperatorStates,
			newRawOperatorStates);

		Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
			new HashMap<>(expectedNumberOfSubTasks);
		Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
			new HashMap<>(expectedNumberOfSubTasks);

		reDistributeKeyedStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			keyGroupPartitions,
			newManagedKeyedState,
			newRawKeyedState);

		/*
		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
		 *
		 */
		assignTaskStateToExecutionJobVertices(
			executionJobVertex,
			newManagedOperatorStates,
			newRawOperatorStates,
			newManagedKeyedState,
			newRawKeyedState,
			newParallelism);
	}
 

private void performIncrementalCheckpoint(
	JobID jid,
	CheckpointCoordinator coord,
	ExecutionJobVertex jobVertex1,
	List<KeyGroupRange> keyGroupPartitions1,
	long timestamp,
	int cpSequenceNumber) throws Exception {

	// trigger the checkpoint
	coord.triggerCheckpoint(timestamp, false);

	assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
	long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());

	for (int index = 0; index < jobVertex1.getParallelism(); index++) {

		KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);

		Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
		privateState.put(
			new StateHandleID("private-1"),
			spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));

		Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();

		// let all but the first CP overlap by one shared state.
		if (cpSequenceNumber > 0) {
			sharedState.put(
				new StateHandleID("shared-" + (cpSequenceNumber - 1)),
				spy(new PlaceholderStreamStateHandle()));
		}

		sharedState.put(
			new StateHandleID("shared-" + cpSequenceNumber),
			spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));

		IncrementalRemoteKeyedStateHandle managedState =
			spy(new IncrementalRemoteKeyedStateHandle(
				new UUID(42L, 42L),
				keyGroupRange,
				checkpointId,
				sharedState,
				privateState,
				spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));

		OperatorSubtaskState operatorSubtaskState =
			spy(new OperatorSubtaskState(
				StateObjectCollection.empty(),
				StateObjectCollection.empty(),
				StateObjectCollection.singleton(managedState),
				StateObjectCollection.empty()));

		Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();

		opStates.put(jobVertex1.getOperatorIDs().get(0), operatorSubtaskState);

		TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);

		AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
			jid,
			jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
			checkpointId,
			new CheckpointMetrics(),
			taskStateSnapshot);

		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint, TASK_MANAGER_LOCATION_INFO);
	}
}
 

private void performIncrementalCheckpoint(
	JobID jid,
	CheckpointCoordinator coord,
	ExecutionJobVertex jobVertex1,
	List<KeyGroupRange> keyGroupPartitions1,
	long timestamp,
	int cpSequenceNumber) throws Exception {

	// trigger the checkpoint
	coord.triggerCheckpoint(timestamp, false);

	assertTrue(coord.getPendingCheckpoints().keySet().size() == 1);
	long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());

	for (int index = 0; index < jobVertex1.getParallelism(); index++) {

		KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);

		Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
		privateState.put(
			new StateHandleID("private-1"),
			spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));

		Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();

		// let all but the first CP overlap by one shared state.
		if (cpSequenceNumber > 0) {
			sharedState.put(
				new StateHandleID("shared-" + (cpSequenceNumber - 1)),
				spy(new PlaceholderStreamStateHandle()));
		}

		sharedState.put(
			new StateHandleID("shared-" + cpSequenceNumber),
			spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));

		IncrementalRemoteKeyedStateHandle managedState =
			spy(new IncrementalRemoteKeyedStateHandle(
				new UUID(42L, 42L),
				keyGroupRange,
				checkpointId,
				sharedState,
				privateState,
				spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));

		OperatorSubtaskState operatorSubtaskState =
			spy(new OperatorSubtaskState(
				StateObjectCollection.empty(),
				StateObjectCollection.empty(),
				StateObjectCollection.singleton(managedState),
				StateObjectCollection.empty()));

		Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();

		opStates.put(jobVertex1.getOperatorIDs().get(0), operatorSubtaskState);

		TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);

		AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
			jid,
			jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
			checkpointId,
			new CheckpointMetrics(),
			taskStateSnapshot);

		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint);
	}
}
 

private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {

		List<OperatorIDPair> operatorIDs = executionJobVertex.getOperatorIDs();

		//1. first compute the new parallelism
		checkParallelismPreconditions(operatorStates, executionJobVertex);

		int newParallelism = executionJobVertex.getParallelism();

		List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
			executionJobVertex.getMaxParallelism(),
			newParallelism);

		final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();

		/*
		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
		 *
		 * The old ManagedOperatorStates with old parallelism 3:
		 *
		 * 		parallelism0 parallelism1 parallelism2
		 * op0   states0,0    state0,1	   state0,2
		 * op1
		 * op2   states2,0    state2,1	   state1,2
		 * op3   states3,0    state3,1     state3,2
		 *
		 * The new ManagedOperatorStates with new parallelism 4:
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   state0,0	  state0,1 	   state0,2		state0,3
		 * op1
		 * op2   state2,0	  state2,1 	   state2,2		state2,3
		 * op3   state3,0	  state3,1 	   state3,2		state3,3
		 */
		Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates = reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			OperatorSubtaskState::getManagedOperatorState,
			RoundRobinOperatorStateRepartitioner.INSTANCE);
		Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates = reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			OperatorSubtaskState::getRawOperatorState,
			RoundRobinOperatorStateRepartitioner.INSTANCE);
		final Map<OperatorInstanceID, List<InputChannelStateHandle>> newInputChannelState = reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			OperatorSubtaskState::getInputChannelState,
			channelStateNonRescalingRepartitioner("input channel"));
		final Map<OperatorInstanceID, List<ResultSubpartitionStateHandle>> newResultSubpartitionState = reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			OperatorSubtaskState::getResultSubpartitionState,
			channelStateNonRescalingRepartitioner("result subpartition"));

		Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState = new HashMap<>(expectedNumberOfSubTasks);
		Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState = new HashMap<>(expectedNumberOfSubTasks);

		reDistributeKeyedStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			keyGroupPartitions,
			newManagedKeyedState,
			newRawKeyedState);

		/*
		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
		 *
		 */
		assignTaskStateToExecutionJobVertices(
			executionJobVertex,
			newManagedOperatorStates,
			newRawOperatorStates,
			newInputChannelState,
			newResultSubpartitionState,
			newManagedKeyedState,
			newRawKeyedState,
			newParallelism);
	}
 

private void assignAttemptState(ExecutionJobVertex executionJobVertex, List<OperatorState> operatorStates) {

		List<OperatorID> operatorIDs = executionJobVertex.getOperatorIDs();

		//1. first compute the new parallelism
		checkParallelismPreconditions(operatorStates, executionJobVertex);

		int newParallelism = executionJobVertex.getParallelism();

		List<KeyGroupRange> keyGroupPartitions = createKeyGroupPartitions(
			executionJobVertex.getMaxParallelism(),
			newParallelism);

		final int expectedNumberOfSubTasks = newParallelism * operatorIDs.size();

		/*
		 * Redistribute ManagedOperatorStates and RawOperatorStates from old parallelism to new parallelism.
		 *
		 * The old ManagedOperatorStates with old parallelism 3:
		 *
		 * 		parallelism0 parallelism1 parallelism2
		 * op0   states0,0    state0,1	   state0,2
		 * op1
		 * op2   states2,0    state2,1	   state1,2
		 * op3   states3,0    state3,1     state3,2
		 *
		 * The new ManagedOperatorStates with new parallelism 4:
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   state0,0	  state0,1 	   state0,2		state0,3
		 * op1
		 * op2   state2,0	  state2,1 	   state2,2		state2,3
		 * op3   state3,0	  state3,1 	   state3,2		state3,3
		 */
		Map<OperatorInstanceID, List<OperatorStateHandle>> newManagedOperatorStates =
			new HashMap<>(expectedNumberOfSubTasks);
		Map<OperatorInstanceID, List<OperatorStateHandle>> newRawOperatorStates =
			new HashMap<>(expectedNumberOfSubTasks);

		reDistributePartitionableStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			newManagedOperatorStates,
			newRawOperatorStates);

		Map<OperatorInstanceID, List<KeyedStateHandle>> newManagedKeyedState =
			new HashMap<>(expectedNumberOfSubTasks);
		Map<OperatorInstanceID, List<KeyedStateHandle>> newRawKeyedState =
			new HashMap<>(expectedNumberOfSubTasks);

		reDistributeKeyedStates(
			operatorStates,
			newParallelism,
			operatorIDs,
			keyGroupPartitions,
			newManagedKeyedState,
			newRawKeyedState);

		/*
		 *  An executionJobVertex's all state handles needed to restore are something like a matrix
		 *
		 * 		parallelism0 parallelism1 parallelism2 parallelism3
		 * op0   sh(0,0)     sh(0,1)       sh(0,2)	    sh(0,3)
		 * op1   sh(1,0)	 sh(1,1)	   sh(1,2)	    sh(1,3)
		 * op2   sh(2,0)	 sh(2,1)	   sh(2,2)		sh(2,3)
		 * op3   sh(3,0)	 sh(3,1)	   sh(3,2)		sh(3,3)
		 *
		 */
		assignTaskStateToExecutionJobVertices(
			executionJobVertex,
			newManagedOperatorStates,
			newRawOperatorStates,
			newManagedKeyedState,
			newRawKeyedState,
			newParallelism);
	}
 

private void performIncrementalCheckpoint(
	JobID jid,
	CheckpointCoordinator coord,
	ExecutionJobVertex jobVertex1,
	List<KeyGroupRange> keyGroupPartitions1,
	int cpSequenceNumber) throws Exception {

	// trigger the checkpoint
	coord.triggerCheckpoint(false);
	manuallyTriggeredScheduledExecutor.triggerAll();

	assertEquals(1, coord.getPendingCheckpoints().size());
	long checkpointId = Iterables.getOnlyElement(coord.getPendingCheckpoints().keySet());

	for (int index = 0; index < jobVertex1.getParallelism(); index++) {

		KeyGroupRange keyGroupRange = keyGroupPartitions1.get(index);

		Map<StateHandleID, StreamStateHandle> privateState = new HashMap<>();
		privateState.put(
			new StateHandleID("private-1"),
			spy(new ByteStreamStateHandle("private-1", new byte[]{'p'})));

		Map<StateHandleID, StreamStateHandle> sharedState = new HashMap<>();

		// let all but the first CP overlap by one shared state.
		if (cpSequenceNumber > 0) {
			sharedState.put(
				new StateHandleID("shared-" + (cpSequenceNumber - 1)),
				spy(new PlaceholderStreamStateHandle()));
		}

		sharedState.put(
			new StateHandleID("shared-" + cpSequenceNumber),
			spy(new ByteStreamStateHandle("shared-" + cpSequenceNumber + "-" + keyGroupRange, new byte[]{'s'})));

		IncrementalRemoteKeyedStateHandle managedState =
			spy(new IncrementalRemoteKeyedStateHandle(
				new UUID(42L, 42L),
				keyGroupRange,
				checkpointId,
				sharedState,
				privateState,
				spy(new ByteStreamStateHandle("meta", new byte[]{'m'}))));

		OperatorSubtaskState operatorSubtaskState =
			spy(new OperatorSubtaskState(
				StateObjectCollection.empty(),
				StateObjectCollection.empty(),
				StateObjectCollection.singleton(managedState),
				StateObjectCollection.empty()));

		Map<OperatorID, OperatorSubtaskState> opStates = new HashMap<>();

		opStates.put(jobVertex1.getOperatorIDs().get(0).getGeneratedOperatorID(), operatorSubtaskState);

		TaskStateSnapshot taskStateSnapshot = new TaskStateSnapshot(opStates);

		AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
			jid,
			jobVertex1.getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
			checkpointId,
			new CheckpointMetrics(),
			taskStateSnapshot);

		coord.receiveAcknowledgeMessage(acknowledgeCheckpoint, TASK_MANAGER_LOCATION_INFO);
	}
}