Java Code Examples for org.apache.flink.runtime.checkpoint.TaskStateSnapshot#putSubtaskStateByOperatorID()

The following examples show how to use org.apache.flink.runtime.checkpoint.TaskStateSnapshot#putSubtaskStateByOperatorID() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JobMasterTriggerSavepointIT.java From Flink-CEPplus with Apache License 2.0

6 votes

@Override
public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return true;
}

Example 2

Source File: JobMasterTriggerSavepointITCase.java From flink with Apache License 2.0

6 votes

@Override
public boolean triggerCheckpoint(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions, final boolean advanceToEndOfEventTime) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return true;
}

Example 3

Source File: FailoverRegionTest.java From flink with Apache License 2.0

6 votes

/**
 * Let the checkpoint coordinator to receive all acknowledges from given executionVertexes so that to complete the expected checkpoint.
 */
private void acknowledgeAllCheckpoints(CheckpointCoordinator checkpointCoordinator, Iterator<ExecutionVertex> executionVertexes) throws IOException, CheckpointException {
	while (executionVertexes.hasNext()) {
		ExecutionVertex executionVertex = executionVertexes.next();
		for (int index = 0; index < executionVertex.getJobVertex().getParallelism(); index++) {
			JobVertexID jobVertexID = executionVertex.getJobvertexId();
			OperatorStateHandle opStateBackend = CheckpointCoordinatorTest.generatePartitionableStateHandle(jobVertexID, index, 2, 8, false);
			OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState(opStateBackend, null, null, null);
			TaskStateSnapshot taskOperatorSubtaskStates = new TaskStateSnapshot();
			taskOperatorSubtaskStates.putSubtaskStateByOperatorID(OperatorID.fromJobVertexID(jobVertexID), operatorSubtaskState);

			AcknowledgeCheckpoint acknowledgeCheckpoint = new AcknowledgeCheckpoint(
				executionVertex.getJobId(),
				executionVertex.getJobVertex().getTaskVertices()[index].getCurrentExecutionAttempt().getAttemptId(),
				checkpointId,
				new CheckpointMetrics(),
				taskOperatorSubtaskStates);

			checkpointCoordinator.receiveAcknowledgeMessage(acknowledgeCheckpoint, "Unknown location");
		}
	}
}

Example 4

Source File: JobMasterTriggerSavepointITCase.java From flink with Apache License 2.0

6 votes

@Override
public Future<Boolean> triggerCheckpointAsync(final CheckpointMetaData checkpointMetaData, final CheckpointOptions checkpointOptions, final boolean advanceToEndOfEventTime) {
	final TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
	checkpointStateHandles.putSubtaskStateByOperatorID(
		OperatorID.fromJobVertexID(getEnvironment().getJobVertexId()),
		new OperatorSubtaskState());

	getEnvironment().acknowledgeCheckpoint(
		checkpointMetaData.getCheckpointId(),
		new CheckpointMetrics(),
		checkpointStateHandles);

	triggerCheckpointLatch.countDown();

	return CompletableFuture.completedFuture(true);
}

Example 5

Source File: CheckpointMessagesTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testConfirmTaskCheckpointed() {
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42,42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTest.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 6

Source File: TaskLocalStateStoreImplTest.java From Flink-CEPplus with Apache License 2.0

5 votes

private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}

Example 7

Source File: RestoreStreamTaskTest.java From Flink-CEPplus with Apache License 2.0

5 votes

@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}

Example 8

Source File: AbstractStreamOperatorTestHarness.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState();
	initializeCalled = true;
}

Example 9

Source File: CheckpointMessagesTest.java From flink with Apache License 2.0

5 votes

@Test
public void testConfirmTaskCheckpointed() {
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42,42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTest.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTest.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 10

Source File: TaskLocalStateStoreImplTest.java From flink with Apache License 2.0

5 votes

private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}

Example 11

Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}

Example 12

Source File: AbstractStreamOperatorTestHarness.java From flink with Apache License 2.0

5 votes

/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.SetupableStreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState();
	initializeCalled = true;
}

Example 13

Source File: CheckpointMessagesTest.java From flink with Apache License 2.0

5 votes

@Test
public void testConfirmTaskCheckpointed() {
	final Random rnd = new Random();
	try {
		AcknowledgeCheckpoint noState = new AcknowledgeCheckpoint(
				new JobID(), new ExecutionAttemptID(), 569345L);

		KeyGroupRange keyGroupRange = KeyGroupRange.of(42, 42);

		TaskStateSnapshot checkpointStateHandles = new TaskStateSnapshot();
		checkpointStateHandles.putSubtaskStateByOperatorID(
			new OperatorID(),
			new OperatorSubtaskState(
				CheckpointCoordinatorTestingUtils.generatePartitionableStateHandle(new JobVertexID(), 0, 2, 8, false),
				null,
				CheckpointCoordinatorTestingUtils.generateKeyGroupState(keyGroupRange, Collections.singletonList(new MyHandle())),
				null,
				singleton(createNewInputChannelStateHandle(10, rnd)),
				singleton(createNewResultSubpartitionStateHandle(10, rnd))
			)
		);

		AcknowledgeCheckpoint withState = new AcknowledgeCheckpoint(
				new JobID(),
				new ExecutionAttemptID(),
				87658976143L,
				new CheckpointMetrics(),
				checkpointStateHandles);

		testSerializabilityEqualsHashCode(noState);
		testSerializabilityEqualsHashCode(withState);
	} catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}

Example 14

Source File: TaskLocalStateStoreImplTest.java From flink with Apache License 2.0

5 votes

private List<TaskStateSnapshot> storeStates(int count) {
	List<TaskStateSnapshot> taskStateSnapshots = new ArrayList<>(count);
	for (int i = 0; i < count; ++i) {
		OperatorID operatorID = new OperatorID();
		TaskStateSnapshot taskStateSnapshot = spy(new TaskStateSnapshot());
		OperatorSubtaskState operatorSubtaskState = new OperatorSubtaskState();
		taskStateSnapshot.putSubtaskStateByOperatorID(operatorID, operatorSubtaskState);
		taskLocalStateStore.storeLocalState(i, taskStateSnapshot);
		taskStateSnapshots.add(taskStateSnapshot);
	}
	return taskStateSnapshots;
}

Example 15

Source File: RestoreStreamTaskTest.java From flink with Apache License 2.0

5 votes

@Test
public void testRestoreAfterScaleUp() throws Exception {
	OperatorID headOperatorID = new OperatorID(42L, 42L);
	OperatorID tailOperatorID = new OperatorID(44L, 44L);

	JobManagerTaskRestore restore = createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.empty());

	TaskStateSnapshot stateHandles = restore.getTaskStateSnapshot();

	assertEquals(2, stateHandles.getSubtaskStateMappings().size());

	// test empty state in case of scale up
	OperatorSubtaskState emptyHeadOperatorState = StateAssignmentOperation.operatorSubtaskStateFrom(
		new OperatorInstanceID(0, headOperatorID),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap(),
		Collections.emptyMap());

	stateHandles.putSubtaskStateByOperatorID(headOperatorID, emptyHeadOperatorState);

	createRunAndCheckpointOperatorChain(
		headOperatorID,
		new CounterOperator(),
		tailOperatorID,
		new CounterOperator(),
		Optional.of(restore));

	assertEquals(new HashSet<>(Arrays.asList(headOperatorID, tailOperatorID)), RESTORED_OPERATORS);
}

Example 16

Source File: AbstractStreamOperatorTestHarness.java From flink with Apache License 2.0

5 votes

/**
 * Calls {@link org.apache.flink.streaming.api.operators.StreamOperator#initializeState()}.
 * Calls {@link org.apache.flink.streaming.api.operators.SetupableStreamOperator#setup(StreamTask, StreamConfig, Output)}
 * if it was not called before.
 *
 * @param jmOperatorStateHandles the primary state (owned by JM)
 * @param tmOperatorStateHandles the (optional) local state (owned by TM) or null.
 * @throws Exception
 */
public void initializeState(
	OperatorSubtaskState jmOperatorStateHandles,
	OperatorSubtaskState tmOperatorStateHandles) throws Exception {

	checkState(!initializeCalled, "TestHarness has already been initialized. Have you " +
		"opened this harness before initializing it?");
	if (!setupCalled) {
		setup();
	}

	if (jmOperatorStateHandles != null) {

		TaskStateSnapshot jmTaskStateSnapshot = new TaskStateSnapshot();
		jmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), jmOperatorStateHandles);

		taskStateManager.setReportedCheckpointId(0);
		taskStateManager.setJobManagerTaskStateSnapshotsByCheckpointId(
			Collections.singletonMap(0L, jmTaskStateSnapshot));

		if (tmOperatorStateHandles != null) {
			TaskStateSnapshot tmTaskStateSnapshot = new TaskStateSnapshot();
			tmTaskStateSnapshot.putSubtaskStateByOperatorID(operator.getOperatorID(), tmOperatorStateHandles);
			taskStateManager.setTaskManagerTaskStateSnapshotsByCheckpointId(
				Collections.singletonMap(0L, tmTaskStateSnapshot));
		}
	}

	operator.initializeState(mockTask.createStreamTaskStateInitializer());
	initializeCalled = true;
}