Java Code Examples for org.apache.flink.runtime.jobgraph.SavepointRestoreSettings#forPath()

The following examples show how to use org.apache.flink.runtime.jobgraph.SavepointRestoreSettings#forPath() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: JarRunHandler.java From Flink-CEPplus with Apache License 2.0

6 votes

private SavepointRestoreSettings getSavepointRestoreSettings(
		final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
			throws RestHandlerException {

	final JarRunRequestBody requestBody = request.getRequestBody();

	final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
		requestBody.getAllowNonRestoredState(),
		() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
		false,
		log);
	final String savepointPath = fromRequestBodyOrQueryParameter(
		emptyToNull(requestBody.getSavepointPath()),
		() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
		null,
		log);
	final SavepointRestoreSettings savepointRestoreSettings;
	if (savepointPath != null) {
		savepointRestoreSettings = SavepointRestoreSettings.forPath(
			savepointPath,
			allowNonRestoredState);
	} else {
		savepointRestoreSettings = SavepointRestoreSettings.none();
	}
	return savepointRestoreSettings;
}

Example 2

Source File: JarRunHandler.java From flink with Apache License 2.0

6 votes

private SavepointRestoreSettings getSavepointRestoreSettings(
		final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
			throws RestHandlerException {

	final JarRunRequestBody requestBody = request.getRequestBody();

	final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
		requestBody.getAllowNonRestoredState(),
		() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
		false,
		log);
	final String savepointPath = fromRequestBodyOrQueryParameter(
		emptyToNull(requestBody.getSavepointPath()),
		() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
		null,
		log);
	final SavepointRestoreSettings savepointRestoreSettings;
	if (savepointPath != null) {
		savepointRestoreSettings = SavepointRestoreSettings.forPath(
			savepointPath,
			allowNonRestoredState);
	} else {
		savepointRestoreSettings = SavepointRestoreSettings.none();
	}
	return savepointRestoreSettings;
}

Example 3

Source File: ClassPathPackagedProgramRetrieverTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSavepointRestoreSettings() throws FlinkException, IOException, ProgramInvocationException {
	final Configuration configuration = new Configuration();
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
	final JobID jobId = new JobID();

	configuration.setString(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID, jobId.toHexString());
	SavepointRestoreSettings.toConfiguration(savepointRestoreSettings, configuration);

	final ClassPathPackagedProgramRetriever retrieverUnderTest =
		ClassPathPackagedProgramRetriever.newBuilder(PROGRAM_ARGUMENTS)
		.setJobClassName(TestJob.class.getCanonicalName())
		.build();

	final JobGraph jobGraph = retrieveJobGraph(retrieverUnderTest, configuration);

	assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
	assertEquals(jobGraph.getJobID(), jobId);
}

Example 4

Source File: JarRunHandler.java From flink with Apache License 2.0

6 votes

private SavepointRestoreSettings getSavepointRestoreSettings(
		final @Nonnull HandlerRequest<JarRunRequestBody, JarRunMessageParameters> request)
			throws RestHandlerException {

	final JarRunRequestBody requestBody = request.getRequestBody();

	final boolean allowNonRestoredState = fromRequestBodyOrQueryParameter(
		requestBody.getAllowNonRestoredState(),
		() -> getQueryParameter(request, AllowNonRestoredStateQueryParameter.class),
		false,
		log);
	final String savepointPath = fromRequestBodyOrQueryParameter(
		emptyToNull(requestBody.getSavepointPath()),
		() -> emptyToNull(getQueryParameter(request, SavepointPathQueryParameter.class)),
		null,
		log);
	final SavepointRestoreSettings savepointRestoreSettings;
	if (savepointPath != null) {
		savepointRestoreSettings = SavepointRestoreSettings.forPath(
			savepointPath,
			allowNonRestoredState);
	} else {
		savepointRestoreSettings = SavepointRestoreSettings.none();
	}
	return savepointRestoreSettings;
}

Example 5

Source File: ClassPathJobGraphRetrieverTest.java From flink with Apache License 2.0

6 votes

@Test
public void testSavepointRestoreSettings() throws FlinkException {
	final Configuration configuration = new Configuration();
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
	final JobID jobId = new JobID();

	final ClassPathJobGraphRetriever classPathJobGraphRetriever = new ClassPathJobGraphRetriever(
		jobId,
		savepointRestoreSettings,
		PROGRAM_ARGUMENTS,
		TestJob.class.getCanonicalName());

	final JobGraph jobGraph = classPathJobGraphRetriever.retrieveJobGraph(configuration);

	assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
	assertEquals(jobGraph.getJobID(), jobId);
}

Example 6

Source File: ClassPathJobGraphRetrieverTest.java From Flink-CEPplus with Apache License 2.0

6 votes

@Test
public void testSavepointRestoreSettings() throws FlinkException {
	final Configuration configuration = new Configuration();
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("foobar", true);
	final JobID jobId = new JobID();

	final ClassPathJobGraphRetriever classPathJobGraphRetriever = new ClassPathJobGraphRetriever(
		jobId,
		savepointRestoreSettings,
		PROGRAM_ARGUMENTS,
		TestJob.class.getCanonicalName());

	final JobGraph jobGraph = classPathJobGraphRetriever.retrieveJobGraph(configuration);

	assertThat(jobGraph.getSavepointRestoreSettings(), is(equalTo(savepointRestoreSettings)));
	assertEquals(jobGraph.getJobID(), jobId);
}

Example 7

Source File: CliFrontendParser.java From Flink-CEPplus with Apache License 2.0

5 votes

public static SavepointRestoreSettings createSavepointRestoreSettings(CommandLine commandLine) {
	if (commandLine.hasOption(SAVEPOINT_PATH_OPTION.getOpt())) {
		String savepointPath = commandLine.getOptionValue(SAVEPOINT_PATH_OPTION.getOpt());
		boolean allowNonRestoredState = commandLine.hasOption(SAVEPOINT_ALLOW_NON_RESTORED_OPTION.getOpt());
		return SavepointRestoreSettings.forPath(savepointPath, allowNonRestoredState);
	} else {
		return SavepointRestoreSettings.none();
	}
}

Example 8

Source File: StreamingJobGraphGeneratorTest.java From flink with Apache License 2.0

5 votes

@Test
public void generatorForwardsSavepointRestoreSettings() {
	StreamGraph streamGraph = new StreamGraph(
			new ExecutionConfig(),
			new CheckpointConfig(),
			SavepointRestoreSettings.forPath("hello"));

	JobGraph jobGraph = StreamingJobGraphGenerator.createJobGraph(streamGraph);

	SavepointRestoreSettings savepointRestoreSettings = jobGraph.getSavepointRestoreSettings();
	assertThat(savepointRestoreSettings.getRestorePath(), is("hello"));
}

Example 9

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests that in a streaming use case where checkpointing is enabled, a
 * fixed delay with Integer.MAX_VALUE retries is instantiated if no other restart
 * strategy has been specified.
 */
@Test
public void testAutomaticRestartingWhenCheckpointing() throws Exception {
	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(
		completedCheckpointStore,
		new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
	final JobMaster jobMaster = createJobMaster(
		new Configuration(),
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder()
			.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
			.build());

	RestartStrategy restartStrategy = jobMaster.getRestartStrategy();

	assertNotNull(restartStrategy);
	assertTrue(restartStrategy instanceof FixedDelayRestartStrategy);
}

Example 10

Source File: JobMasterTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests that a JobMaster will restore the given JobGraph from its savepoint upon
 * initial submission.
 */
@Test
public void testRestoringFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 11

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests that a JobMaster will restore the given JobGraph from its savepoint upon
 * initial submission.
 */
@Test
public void testRestoringFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 12

Source File: FlinkRequiresStableInputTest.java From beam with Apache License 2.0

5 votes

private JobID restoreFromSavepoint(Pipeline pipeline, String savepointDir)
    throws ExecutionException, InterruptedException {
  JobGraph jobGraph = getJobGraph(pipeline);
  SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.forPath(savepointDir);
  jobGraph.setSavepointRestoreSettings(savepointSettings);
  return flinkCluster.submitJob(jobGraph).get().getJobID();
}

Example 13

Source File: FlinkSavepointTest.java From beam with Apache License 2.0

5 votes

private void restoreFromSavepointLegacy(Pipeline pipeline, String savepointDir)
    throws ExecutionException, InterruptedException {
  JobGraph jobGraph = getJobGraph(pipeline);
  SavepointRestoreSettings savepointSettings = SavepointRestoreSettings.forPath(savepointDir);
  jobGraph.setSavepointRestoreSettings(savepointSettings);
  flinkCluster.submitJob(jobGraph).get();
}

Example 14

Source File: JobMasterTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests that a JobMaster will restore the given JobGraph from its savepoint upon
 * initial submission.
 */
@Test
public void testRestoringFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);
	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 15

Source File: StandaloneApplicationClusterConfigurationParserFactoryTest.java From flink with Apache License 2.0

5 votes

@Test
public void testEntrypointClusterConfigurationToConfigurationParsing() throws FlinkParseException {
	final JobID jobID = JobID.generate();
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("/test/savepoint/path", true);
	final String key = DeploymentOptions.TARGET.key();
	final String value = "testDynamicExecutorConfig";
	final int restPort = 1234;
	final String arg1 = "arg1";
	final String arg2 = "arg2";
	final String[] args = {
			"--configDir", confDirPath,
			"--job-id", jobID.toHexString(),
			"--fromSavepoint", savepointRestoreSettings.getRestorePath(),
			"--allowNonRestoredState",
			"--webui-port", String.valueOf(restPort),
			"--job-classname", JOB_CLASS_NAME,
			String.format("-D%s=%s", key, value),
			arg1, arg2};

	final StandaloneApplicationClusterConfiguration clusterConfiguration = commandLineParser.parse(args);
	assertThat(clusterConfiguration.getJobClassName(), is(equalTo(JOB_CLASS_NAME)));
	assertThat(clusterConfiguration.getArgs(), arrayContaining(arg1, arg2));

	final Configuration configuration = StandaloneApplicationClusterEntryPoint
			.loadConfigurationFromClusterConfig(clusterConfiguration);

	final String strJobId = configuration.get(PipelineOptionsInternal.PIPELINE_FIXED_JOB_ID);
	assertThat(JobID.fromHexString(strJobId), is(equalTo(jobID)));
	assertThat(SavepointRestoreSettings.fromConfiguration(configuration), is(equalTo(savepointRestoreSettings)));

	assertThat(configuration.get(RestOptions.PORT), is(equalTo(restPort)));
	assertThat(configuration.get(DeploymentOptions.TARGET), is(equalTo(value)));
}

Example 16

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests that an existing checkpoint will have precedence over an savepoint.
 */
@Test
public void testCheckpointPrecedesSavepointRecovery() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("" +
			savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final long checkpointId = 1L;

	final CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(
		jobGraph.getJobID(),
		checkpointId,
		1L,
		1L,
		Collections.emptyMap(),
		null,
		CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
		new DummyCheckpointStorageLocation());

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	completedCheckpointStore.addCheckpoint(completedCheckpoint);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);

	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(checkpointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 17

Source File: JobMasterTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests that a JobMaster will only restore a modified JobGraph if non
 * restored state is allowed.
 */
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final OperatorID operatorID = new OperatorID();
	final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);

	// set savepoint settings which don't allow non restored state
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		false);

	// create a new operator
	final JobVertex jobVertex = new JobVertex("New operator");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);

	try {
		createJobMaster(
			configuration,
			jobGraphWithNewOperator,
			haServices,
			new TestingJobManagerSharedServicesBuilder().build());
		fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
	} catch (IllegalStateException expected) {
		// that was expected :-)
	}

	// allow for non restored state
	jobGraphWithNewOperator.setSavepointRestoreSettings(
		SavepointRestoreSettings.forPath(
			savepointFile.getAbsolutePath(),
			true));

	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraphWithNewOperator,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint();

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 18

Source File: JobMasterTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests that a JobMaster will only restore a modified JobGraph if non
 * restored state is allowed.
 */
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final OperatorID operatorID = new OperatorID();
	final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);

	// set savepoint settings which don't allow non restored state
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		false);

	// create a new operator
	final JobVertex jobVertex = new JobVertex("New operator");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);

	try {
		createJobMaster(
			configuration,
			jobGraphWithNewOperator,
			haServices,
			new TestingJobManagerSharedServicesBuilder().build());
		fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
	} catch (IllegalStateException expected) {
		// that was expected :-)
	}

	// allow for non restored state
	jobGraphWithNewOperator.setSavepointRestoreSettings(
		SavepointRestoreSettings.forPath(
			savepointFile.getAbsolutePath(),
			true));

	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraphWithNewOperator,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 19

Source File: JobMasterTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests that an existing checkpoint will have precedence over an savepoint.
 */
@Test
public void testCheckpointPrecedesSavepointRecovery() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final File savepointFile = createSavepoint(savepointId);

	// set savepoint settings
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath("" +
			savepointFile.getAbsolutePath(),
		true);
	final JobGraph jobGraph = createJobGraphWithCheckpointing(savepointRestoreSettings);

	final long checkpointId = 1L;

	final CompletedCheckpoint completedCheckpoint = new CompletedCheckpoint(
		jobGraph.getJobID(),
		checkpointId,
		1L,
		1L,
		Collections.emptyMap(),
		null,
		CheckpointProperties.forCheckpoint(CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION),
		new DummyCheckpointStorageLocation());

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	completedCheckpointStore.addCheckpoint(completedCheckpoint);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);

	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraph,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(checkpointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}

Example 20

Source File: JobMasterTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests that a JobMaster will only restore a modified JobGraph if non
 * restored state is allowed.
 */
@Test
public void testRestoringModifiedJobFromSavepoint() throws Exception {

	// create savepoint data
	final long savepointId = 42L;
	final OperatorID operatorID = new OperatorID();
	final File savepointFile = createSavepointWithOperatorState(savepointId, operatorID);

	// set savepoint settings which don't allow non restored state
	final SavepointRestoreSettings savepointRestoreSettings = SavepointRestoreSettings.forPath(
		savepointFile.getAbsolutePath(),
		false);

	// create a new operator
	final JobVertex jobVertex = new JobVertex("New operator");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	final JobGraph jobGraphWithNewOperator = createJobGraphFromJobVerticesWithCheckpointing(savepointRestoreSettings, jobVertex);

	final StandaloneCompletedCheckpointStore completedCheckpointStore = new StandaloneCompletedCheckpointStore(1);
	final TestingCheckpointRecoveryFactory testingCheckpointRecoveryFactory = new TestingCheckpointRecoveryFactory(completedCheckpointStore, new StandaloneCheckpointIDCounter());
	haServices.setCheckpointRecoveryFactory(testingCheckpointRecoveryFactory);

	try {
		createJobMaster(
			configuration,
			jobGraphWithNewOperator,
			haServices,
			new TestingJobManagerSharedServicesBuilder().build());
		fail("Should fail because we cannot resume the changed JobGraph from the savepoint.");
	} catch (IllegalStateException expected) {
		// that was expected :-)
	}

	// allow for non restored state
	jobGraphWithNewOperator.setSavepointRestoreSettings(
		SavepointRestoreSettings.forPath(
			savepointFile.getAbsolutePath(),
			true));

	final JobMaster jobMaster = createJobMaster(
		configuration,
		jobGraphWithNewOperator,
		haServices,
		new TestingJobManagerSharedServicesBuilder().build());

	try {
		// starting the JobMaster should have read the savepoint
		final CompletedCheckpoint savepointCheckpoint = completedCheckpointStore.getLatestCheckpoint(false);

		assertThat(savepointCheckpoint, Matchers.notNullValue());

		assertThat(savepointCheckpoint.getCheckpointID(), is(savepointId));
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}