Java Code Examples for org.apache.flink.runtime.executiongraph.ExecutionGraph#getCheckpointCoordinator()

The following examples show how to use org.apache.flink.runtime.executiongraph.ExecutionGraph#getCheckpointCoordinator() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createAndRestoreExecutionGraph(JobManagerJobMetricGroup currentJobManagerJobMetricGroup) throws Exception {

		ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup);

		final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

		if (checkpointCoordinator != null) {
			// check whether we find a valid checkpoint
			if (!checkpointCoordinator.restoreLatestCheckpointedState(
				newExecutionGraph.getAllVertices(),
				false,
				false)) {

				// check whether we can restore from a savepoint
				tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
			}
		}

		return newExecutionGraph;
	}
Example 2
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is failed.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFailure() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.failGlobal(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FAILED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FAILED));
}
Example 3
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is suspended.
 */
@Test
public void testShutdownCheckpointCoordinatorOnSuspend() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.suspend(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.SUSPENDED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.SUSPENDED));
}
Example 4
Source File: LegacyScheduler.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createAndRestoreExecutionGraph(
		JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
		ShuffleMaster<?> shuffleMaster,
		PartitionTracker partitionTracker) throws Exception {

	ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker);

	final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

	if (checkpointCoordinator != null) {
		// check whether we find a valid checkpoint
		if (!checkpointCoordinator.restoreLatestCheckpointedState(
			newExecutionGraph.getAllVertices(),
			false,
			false)) {

			// check whether we can restore from a savepoint
			tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
		}
	}

	return newExecutionGraph;
}
Example 5
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is failed.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFailure() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.failGlobal(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FAILED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FAILED));
}
Example 6
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is suspended.
 */
@Test
public void testShutdownCheckpointCoordinatorOnSuspend() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.suspend(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.SUSPENDED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.SUSPENDED));
}
Example 7
Source File: SchedulerBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
private ExecutionGraph createAndRestoreExecutionGraph(
	JobManagerJobMetricGroup currentJobManagerJobMetricGroup,
	ShuffleMaster<?> shuffleMaster,
	JobMasterPartitionTracker partitionTracker) throws Exception {

	ExecutionGraph newExecutionGraph = createExecutionGraph(currentJobManagerJobMetricGroup, shuffleMaster, partitionTracker);

	final CheckpointCoordinator checkpointCoordinator = newExecutionGraph.getCheckpointCoordinator();

	if (checkpointCoordinator != null) {
		// check whether we find a valid checkpoint
		if (!checkpointCoordinator.restoreLatestCheckpointedStateToAll(
			new HashSet<>(newExecutionGraph.getAllVertices().values()),
			false)) {

			// check whether we can restore from a savepoint
			tryRestoreExecutionGraphFromSavepoint(newExecutionGraph, jobGraph.getSavepointRestoreSettings());
		}
	}

	return newExecutionGraph;
}
Example 8
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is failed.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFailure() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.failGlobal(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FAILED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FAILED));
}
Example 9
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 6 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is suspended.
 */
@Test
public void testShutdownCheckpointCoordinatorOnSuspend() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.suspend(new Exception("Test Exception"));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.SUSPENDED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.SUSPENDED));
}
Example 10
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}.
 *
 * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored
 * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from
 * @throws Exception if the {@link ExecutionGraph} could not be restored
 */
private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception {
	if (savepointRestoreSettings.restoreSavepoint()) {
		final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator();
		if (checkpointCoordinator != null) {
			checkpointCoordinator.restoreSavepoint(
				savepointRestoreSettings.getRestorePath(),
				savepointRestoreSettings.allowNonRestoredState(),
				executionGraphToRestore.getAllVertices(),
				userCodeLoader);
		}
	}
}
Example 11
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
Example 12
Source File: LegacyScheduler.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}.
 *
 * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored
 * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from
 * @throws Exception if the {@link ExecutionGraph} could not be restored
 */
private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception {
	if (savepointRestoreSettings.restoreSavepoint()) {
		final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator();
		if (checkpointCoordinator != null) {
			checkpointCoordinator.restoreSavepoint(
				savepointRestoreSettings.getRestorePath(),
				savepointRestoreSettings.allowNonRestoredState(),
				executionGraphToRestore.getAllVertices(),
				userCodeLoader);
		}
	}
}
Example 13
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}
Example 14
Source File: SchedulerBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tries to restore the given {@link ExecutionGraph} from the provided {@link SavepointRestoreSettings}.
 *
 * @param executionGraphToRestore {@link ExecutionGraph} which is supposed to be restored
 * @param savepointRestoreSettings {@link SavepointRestoreSettings} containing information about the savepoint to restore from
 * @throws Exception if the {@link ExecutionGraph} could not be restored
 */
private void tryRestoreExecutionGraphFromSavepoint(ExecutionGraph executionGraphToRestore, SavepointRestoreSettings savepointRestoreSettings) throws Exception {
	if (savepointRestoreSettings.restoreSavepoint()) {
		final CheckpointCoordinator checkpointCoordinator = executionGraphToRestore.getCheckpointCoordinator();
		if (checkpointCoordinator != null) {
			checkpointCoordinator.restoreSavepoint(
				savepointRestoreSettings.getRestorePath(),
				savepointRestoreSettings.allowNonRestoredState(),
				executionGraphToRestore.getAllVertices(),
				userCodeLoader);
		}
	}
}
Example 15
Source File: ExecutionGraphCheckpointCoordinatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that the checkpoint coordinator is shut down if the execution graph
 * is finished.
 */
@Test
public void testShutdownCheckpointCoordinatorOnFinished() throws Exception {
	final CompletableFuture<JobStatus> counterShutdownFuture = new CompletableFuture<>();
	CheckpointIDCounter counter = new TestingCheckpointIDCounter(counterShutdownFuture);

	final CompletableFuture<JobStatus> storeShutdownFuture = new CompletableFuture<>();
	CompletedCheckpointStore store = new TestingCompletedCheckpointStore(storeShutdownFuture);

	ExecutionGraph graph = createExecutionGraphAndEnableCheckpointing(counter, store);
	final CheckpointCoordinator checkpointCoordinator = graph.getCheckpointCoordinator();

	assertThat(checkpointCoordinator, Matchers.notNullValue());
	assertThat(checkpointCoordinator.isShutdown(), is(false));

	graph.scheduleForExecution();

	for (ExecutionVertex executionVertex : graph.getAllExecutionVertices()) {
		final Execution currentExecutionAttempt = executionVertex.getCurrentExecutionAttempt();
		graph.updateState(new TaskExecutionState(graph.getJobID(), currentExecutionAttempt.getAttemptId(), ExecutionState.FINISHED));
	}

	assertThat(graph.getTerminationFuture().get(), is(JobStatus.FINISHED));

	assertThat(checkpointCoordinator.isShutdown(), is(true));
	assertThat(counterShutdownFuture.get(), is(JobStatus.FINISHED));
	assertThat(storeShutdownFuture.get(), is(JobStatus.FINISHED));
}