Java Code Examples for org.apache.flink.api.common.restartstrategy.RestartStrategies#RestartStrategyConfiguration

The following examples show how to use org.apache.flink.api.common.restartstrategy.RestartStrategies#RestartStrategyConfiguration . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 
Example 2
Source File: ExecutionConfig.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the restart strategy which has been set for the current job.
 *
 * @return The specified restart configuration
 */
@PublicEvolving
@SuppressWarnings("deprecation")
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	if (restartStrategyConfiguration instanceof RestartStrategies.FallbackRestartStrategyConfiguration) {
		// support the old API calls by creating a restart strategy from them
		if (getNumberOfExecutionRetries() > 0 && getExecutionRetryDelay() >= 0) {
			return RestartStrategies.fixedDelayRestart(getNumberOfExecutionRetries(), getExecutionRetryDelay());
		} else if (getNumberOfExecutionRetries() == 0) {
			return RestartStrategies.noRestart();
		} else {
			return restartStrategyConfiguration;
		}
	} else {
		return restartStrategyConfiguration;
	}
}
 
Example 3
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
 
Example 4
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 0, restarting is deactivated.
 */
@Test
public void testNoRestartingWhenCheckpointingAndExplicitExecutionRetriesZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(0);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.NoRestartStrategyConfiguration);
}
 
Example 5
Source File: RestartStrategyTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that in a streaming use case where checkpointing is enabled, there is no default strategy set on the
 * client side.
 */
@Test
public void testFallbackStrategyOnClientSideWhenCheckpointingEnabled() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FallbackRestartStrategyConfiguration);
}
 
Example 6
Source File: RestartStrategyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Checks that in a streaming use case where checkpointing is enabled and the number
 * of execution retries is set to 42 and the delay to 1337, fixed delay restarting is used.
 */
@Test
public void testFixedRestartingWhenCheckpointingAndExplicitExecutionRetriesNonZero() throws Exception {
	StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
	env.enableCheckpointing(500);
	env.setNumberOfExecutionRetries(42);
	env.getConfig().setExecutionRetryDelay(1337);

	env.fromElements(1).print();

	StreamGraph graph = env.getStreamGraph();
	JobGraph jobGraph = graph.getJobGraph();

	RestartStrategies.RestartStrategyConfiguration restartStrategy =
		jobGraph.getSerializedExecutionConfig().deserializeValue(getClass().getClassLoader()).getRestartStrategy();

	Assert.assertNotNull(restartStrategy);
	Assert.assertTrue(restartStrategy instanceof RestartStrategies.FixedDelayRestartStrategyConfiguration);
	Assert.assertEquals(42, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getRestartAttempts());
	Assert.assertEquals(1337, ((RestartStrategies.FixedDelayRestartStrategyConfiguration) restartStrategy).getDelayBetweenAttemptsInterval().toMilliseconds());
}
 
Example 7
Source File: ExecutionConfig.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Returns the restart strategy which has been set for the current job.
 *
 * @return The specified restart configuration
 */
@PublicEvolving
@SuppressWarnings("deprecation")
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	if (restartStrategyConfiguration instanceof RestartStrategies.FallbackRestartStrategyConfiguration) {
		// support the old API calls by creating a restart strategy from them
		if (getNumberOfExecutionRetries() > 0 && getExecutionRetryDelay() >= 0) {
			return RestartStrategies.fixedDelayRestart(getNumberOfExecutionRetries(), getExecutionRetryDelay());
		} else if (getNumberOfExecutionRetries() == 0) {
			return RestartStrategies.noRestart();
		} else {
			return restartStrategyConfiguration;
		}
	} else {
		return restartStrategyConfiguration;
	}
}
 
Example 8
Source File: DataStreamAllroundTestJobFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private static void setupRestartStrategy(final StreamExecutionEnvironment env, final ParameterTool pt) {
	String restartStrategyConfig = pt.get(ENVIRONMENT_RESTART_STRATEGY.key());
	if (restartStrategyConfig != null) {
		RestartStrategies.RestartStrategyConfiguration restartStrategy;
		switch (restartStrategyConfig) {
			case "fixed_delay":
				restartStrategy = RestartStrategies.fixedDelayRestart(
					pt.getInt(
						ENVIRONMENT_RESTART_STRATEGY_FIXED_ATTEMPTS.key(),
						ENVIRONMENT_RESTART_STRATEGY_FIXED_ATTEMPTS.defaultValue()),
					pt.getLong(
						ENVIRONMENT_RESTART_STRATEGY_FIXED_DELAY.key(),
						ENVIRONMENT_RESTART_STRATEGY_FIXED_DELAY.defaultValue()));
				break;
			case "no_restart":
				restartStrategy = RestartStrategies.noRestart();
				break;
			default:
				throw new IllegalArgumentException("Unknown restart strategy: " + restartStrategyConfig);
		}
		env.setRestartStrategy(restartStrategy);
	}
}
 
Example 9
Source File: RestartBackoffTimeStrategyFactoryLoader.java    From flink with Apache License 2.0 5 votes vote down vote up
private static Optional<RestartBackoffTimeStrategy.Factory> getJobRestartStrategyFactory(
		final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration) {

	if (restartStrategyConfiguration instanceof NoRestartStrategyConfiguration) {
		return Optional.of(NoRestartBackoffTimeStrategy.NoRestartBackoffTimeStrategyFactory.INSTANCE);
	} else if (restartStrategyConfiguration instanceof FixedDelayRestartStrategyConfiguration) {
		final FixedDelayRestartStrategyConfiguration fixedDelayConfig =
			(FixedDelayRestartStrategyConfiguration) restartStrategyConfiguration;

		return Optional.of(new FixedDelayRestartBackoffTimeStrategy.FixedDelayRestartBackoffTimeStrategyFactory(
			fixedDelayConfig.getRestartAttempts(),
			fixedDelayConfig.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategyConfiguration instanceof FailureRateRestartStrategyConfiguration) {
		final FailureRateRestartStrategyConfiguration failureRateConfig =
			(FailureRateRestartStrategyConfiguration) restartStrategyConfiguration;

		return Optional.of(new FailureRateRestartBackoffTimeStrategy.FailureRateRestartBackoffTimeStrategyFactory(
			failureRateConfig.getMaxFailureRate(),
			failureRateConfig.getFailureInterval().toMilliseconds(),
			failureRateConfig.getDelayBetweenAttemptsInterval().toMilliseconds()));
	} else if (restartStrategyConfiguration instanceof FallbackRestartStrategyConfiguration) {
		return Optional.empty();
	} else {
		throw new IllegalArgumentException("Unknown restart strategy configuration " +
			restartStrategyConfiguration + ".");
	}
}
 
Example 10
Source File: ExecutionEntry.java    From flink with Apache License 2.0 5 votes vote down vote up
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	return properties.getOptionalString(EXECUTION_RESTART_STRATEGY_TYPE)
		.flatMap((v) -> {
			switch (v) {
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_NONE:
					return Optional.of(RestartStrategies.noRestart());
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FIXED_DELAY:
					final int attempts = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_ATTEMPTS)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_ATTEMPTS, Integer.MAX_VALUE));
					final long fixedDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
					return Optional.of(RestartStrategies.fixedDelayRestart(attempts, fixedDelay));
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FAILURE_RATE:
					final int failureRate = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL, 1));
					final long failureInterval = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL, 60_000L));
					final long attemptDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
					return Optional.of(RestartStrategies.failureRateRestart(
						failureRate,
						Time.milliseconds(failureInterval),
						Time.milliseconds(attemptDelay)));
				default:
					return Optional.empty();
				}
		})
		.orElseGet(() ->
			useDefaultValue(
				EXECUTION_RESTART_STRATEGY_TYPE,
				RestartStrategies.fallBackRestart(),
				EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FALLBACK));
}
 
Example 11
Source File: ExecutionEntry.java    From flink with Apache License 2.0 5 votes vote down vote up
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	return properties.getOptionalString(EXECUTION_RESTART_STRATEGY_TYPE)
		.flatMap((v) -> {
			switch (v) {
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_NONE:
					return Optional.of(RestartStrategies.noRestart());
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FIXED_DELAY:
					final int attempts = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_ATTEMPTS)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_ATTEMPTS, Integer.MAX_VALUE));
					final long fixedDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
					return Optional.of(RestartStrategies.fixedDelayRestart(attempts, fixedDelay));
				case EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FAILURE_RATE:
					final int failureRate = properties.getOptionalInt(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_MAX_FAILURES_PER_INTERVAL, 1));
					final long failureInterval = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_FAILURE_RATE_INTERVAL, 60_000L));
					final long attemptDelay = properties.getOptionalLong(EXECUTION_RESTART_STRATEGY_DELAY)
						.orElseGet(() -> useDefaultValue(EXECUTION_RESTART_STRATEGY_DELAY, 10_000L));
					return Optional.of(RestartStrategies.failureRateRestart(
						failureRate,
						Time.milliseconds(failureInterval),
						Time.milliseconds(attemptDelay)));
				default:
					return Optional.empty();
				}
		})
		.orElseGet(() ->
			useDefaultValue(
				EXECUTION_RESTART_STRATEGY_TYPE,
				RestartStrategies.fallBackRestart(),
				EXECUTION_RESTART_STRATEGY_TYPE_VALUE_FALLBACK));
}
 
Example 12
Source File: DummyStreamExecutionEnvironment.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	return realExecEnv.getRestartStrategy();
}
 
Example 13
Source File: JobMasterStopWithSavepointIT.java    From flink with Apache License 2.0 4 votes vote down vote up
private void setUpJobGraph(
		final Class<? extends AbstractInvokable> invokable,
		final RestartStrategies.RestartStrategyConfiguration restartStrategy) throws Exception {

	finishingLatch = new OneShotLatch();

	invokeLatch = new CountDownLatch(PARALLELISM);

	numberOfRestarts = new CountDownLatch(2);
	checkpointsToWaitFor = new CountDownLatch(10);

	syncSavepointId.set(-1);

	savepointDirectory = temporaryFolder.newFolder().toPath();

	Assume.assumeTrue(
			"ClusterClient is not an instance of MiniClusterClient",
			miniClusterResource.getClusterClient() instanceof MiniClusterClient);

	clusterClient = (MiniClusterClient) miniClusterResource.getClusterClient();

	jobGraph = new JobGraph();

	final ExecutionConfig config = new ExecutionConfig();
	config.setRestartStrategy(restartStrategy);
	jobGraph.setExecutionConfig(config);

	final JobVertex vertex = new JobVertex("testVertex");
	vertex.setInvokableClass(invokable);
	vertex.setParallelism(PARALLELISM);
	jobGraph.addVertex(vertex);

	jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
			Collections.singletonList(vertex.getID()),
			Collections.singletonList(vertex.getID()),
			Collections.singletonList(vertex.getID()),
			new CheckpointCoordinatorConfiguration(
					CHECKPOINT_INTERVAL,
					60_000,
					10,
					1,
					CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
					true,
					false,
					false,
					0),
			null));

	ClientUtils.submitJob(clusterClient, jobGraph);
	assertTrue(invokeLatch.await(60, TimeUnit.SECONDS));
	waitForJob();
}
 
Example 14
Source File: JobMaster.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
public JobMaster(
		RpcService rpcService,
		JobMasterConfiguration jobMasterConfiguration,
		ResourceID resourceId,
		JobGraph jobGraph,
		HighAvailabilityServices highAvailabilityService,
		SlotPoolFactory slotPoolFactory,
		SchedulerFactory schedulerFactory,
		JobManagerSharedServices jobManagerSharedServices,
		HeartbeatServices heartbeatServices,
		JobManagerJobMetricGroupFactory jobMetricGroupFactory,
		OnCompletionActions jobCompletionActions,
		FatalErrorHandler fatalErrorHandler,
		ClassLoader userCodeLoader) throws Exception {

	super(rpcService, AkkaRpcServiceUtils.createRandomName(JOB_MANAGER_NAME));

	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.resourceId = checkNotNull(resourceId);
	this.jobGraph = checkNotNull(jobGraph);
	this.rpcTimeout = jobMasterConfiguration.getRpcTimeout();
	this.highAvailabilityServices = checkNotNull(highAvailabilityService);
	this.blobWriter = jobManagerSharedServices.getBlobWriter();
	this.scheduledExecutorService = jobManagerSharedServices.getScheduledExecutorService();
	this.jobCompletionActions = checkNotNull(jobCompletionActions);
	this.fatalErrorHandler = checkNotNull(fatalErrorHandler);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.heartbeatServices = checkNotNull(heartbeatServices);
	this.jobMetricGroupFactory = checkNotNull(jobMetricGroupFactory);

	final String jobName = jobGraph.getName();
	final JobID jid = jobGraph.getJobID();

	log.info("Initializing job {} ({}).", jobName, jid);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
			jobGraph.getSerializedExecutionConfig()
					.deserializeValue(userCodeLoader)
					.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		jobManagerSharedServices.getRestartStrategyFactory(),
		jobGraph.isCheckpointingEnabled());

	log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobName, jid);

	resourceManagerLeaderRetriever = highAvailabilityServices.getResourceManagerLeaderRetriever();

	this.slotPool = checkNotNull(slotPoolFactory).createSlotPool(jobGraph.getJobID());

	this.scheduler = checkNotNull(schedulerFactory).createScheduler(slotPool);

	this.registeredTaskManagers = new HashMap<>(4);

	this.backPressureStatsTracker = checkNotNull(jobManagerSharedServices.getBackPressureStatsTracker());
	this.lastInternalSavepoint = null;

	this.jobManagerJobMetricGroup = jobMetricGroupFactory.create(jobGraph);
	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup);
	this.jobStatusListener = null;

	this.resourceManagerConnection = null;
	this.establishedResourceManagerConnection = null;

	this.accumulators = new HashMap<>();
}
 
Example 15
Source File: SchedulerBase.java    From flink with Apache License 2.0 4 votes vote down vote up
public SchedulerBase(
	final Logger log,
	final JobGraph jobGraph,
	final BackPressureStatsTracker backPressureStatsTracker,
	final Executor ioExecutor,
	final Configuration jobMasterConfiguration,
	final SlotProvider slotProvider,
	final ScheduledExecutorService futureExecutor,
	final ClassLoader userCodeLoader,
	final CheckpointRecoveryFactory checkpointRecoveryFactory,
	final Time rpcTimeout,
	final RestartStrategyFactory restartStrategyFactory,
	final BlobWriter blobWriter,
	final JobManagerJobMetricGroup jobManagerJobMetricGroup,
	final Time slotRequestTimeout,
	final ShuffleMaster<?> shuffleMaster,
	final JobMasterPartitionTracker partitionTracker,
	final ExecutionVertexVersioner executionVertexVersioner,
	final boolean legacyScheduling) throws Exception {

	this.log = checkNotNull(log);
	this.jobGraph = checkNotNull(jobGraph);
	this.backPressureStatsTracker = checkNotNull(backPressureStatsTracker);
	this.ioExecutor = checkNotNull(ioExecutor);
	this.jobMasterConfiguration = checkNotNull(jobMasterConfiguration);
	this.slotProvider = checkNotNull(slotProvider);
	this.futureExecutor = checkNotNull(futureExecutor);
	this.userCodeLoader = checkNotNull(userCodeLoader);
	this.checkpointRecoveryFactory = checkNotNull(checkpointRecoveryFactory);
	this.rpcTimeout = checkNotNull(rpcTimeout);

	final RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration =
		jobGraph.getSerializedExecutionConfig()
			.deserializeValue(userCodeLoader)
			.getRestartStrategy();

	this.restartStrategy = RestartStrategyResolving.resolve(restartStrategyConfiguration,
		restartStrategyFactory,
		jobGraph.isCheckpointingEnabled());

	if (legacyScheduling) {
		log.info("Using restart strategy {} for {} ({}).", this.restartStrategy, jobGraph.getName(), jobGraph.getJobID());
	}

	this.blobWriter = checkNotNull(blobWriter);
	this.jobManagerJobMetricGroup = checkNotNull(jobManagerJobMetricGroup);
	this.slotRequestTimeout = checkNotNull(slotRequestTimeout);
	this.executionVertexVersioner = checkNotNull(executionVertexVersioner);
	this.legacyScheduling = legacyScheduling;

	this.executionGraph = createAndRestoreExecutionGraph(jobManagerJobMetricGroup, checkNotNull(shuffleMaster), checkNotNull(partitionTracker));
	this.schedulingTopology = executionGraph.getSchedulingTopology();

	final StateLocationRetriever stateLocationRetriever =
		executionVertexId -> getExecutionVertex(executionVertexId).getPreferredLocationBasedOnState();
	final InputsLocationsRetriever inputsLocationsRetriever = new ExecutionGraphToInputsLocationsRetrieverAdapter(executionGraph);
	this.preferredLocationsRetriever = new DefaultPreferredLocationsRetriever(stateLocationRetriever, inputsLocationsRetriever);

	this.coordinatorMap = createCoordinatorMap();
}
 
Example 16
Source File: JobMasterStopWithSavepointIT.java    From flink with Apache License 2.0 4 votes vote down vote up
private void setUpJobGraph(
		final Class<? extends AbstractInvokable> invokable,
		final RestartStrategies.RestartStrategyConfiguration restartStrategy) throws Exception {

	finishingLatch = new OneShotLatch();

	invokeLatch = new CountDownLatch(PARALLELISM);

	numberOfRestarts = new CountDownLatch(2);
	checkpointsToWaitFor = new CountDownLatch(10);

	syncSavepointId.set(-1);

	savepointDirectory = temporaryFolder.newFolder().toPath();

	Assume.assumeTrue(
			"ClusterClient is not an instance of MiniClusterClient",
			miniClusterResource.getClusterClient() instanceof MiniClusterClient);

	clusterClient = (MiniClusterClient) miniClusterResource.getClusterClient();
	clusterClient.setDetached(true);

	jobGraph = new JobGraph();

	final ExecutionConfig config = new ExecutionConfig();
	config.setRestartStrategy(restartStrategy);
	jobGraph.setExecutionConfig(config);

	final JobVertex vertex = new JobVertex("testVertex");
	vertex.setInvokableClass(invokable);
	vertex.setParallelism(PARALLELISM);
	jobGraph.addVertex(vertex);

	jobGraph.setSnapshotSettings(new JobCheckpointingSettings(
			Collections.singletonList(vertex.getID()),
			Collections.singletonList(vertex.getID()),
			Collections.singletonList(vertex.getID()),
			new CheckpointCoordinatorConfiguration(
					CHECKPOINT_INTERVAL,
					60_000,
					10,
					1,
					CheckpointRetentionPolicy.NEVER_RETAIN_AFTER_TERMINATION,
					true,
					false,
					0),
			null));

	clusterClient.submitJob(jobGraph, ClassLoader.getSystemClassLoader());
	invokeLatch.await(60, TimeUnit.SECONDS);
	waitForJob();
}
 
Example 17
Source File: ExecutionEnvironment.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Sets the restart strategy configuration. The configuration specifies which restart strategy
 * will be used for the execution graph in case of a restart.
 *
 * @param restartStrategyConfiguration Restart strategy configuration to be set
 */
@PublicEvolving
public void setRestartStrategy(RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration) {
	config.setRestartStrategy(restartStrategyConfiguration);
}
 
Example 18
Source File: ExecutionConfig.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Sets the restart strategy to be used for recovery.
 *
 * <pre>{@code
 * ExecutionConfig config = env.getConfig();
 *
 * config.setRestartStrategy(RestartStrategies.fixedDelayRestart(
 * 	10,  // number of retries
 * 	1000 // delay between retries));
 * }</pre>
 *
 * @param restartStrategyConfiguration Configuration defining the restart strategy to use
 */
@PublicEvolving
public void setRestartStrategy(RestartStrategies.RestartStrategyConfiguration restartStrategyConfiguration) {
	this.restartStrategyConfiguration = Preconditions.checkNotNull(restartStrategyConfiguration);
}
 
Example 19
Source File: ExecutionEnvironment.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Returns the specified restart strategy configuration.
 *
 * @return The restart strategy configuration to be used
 */
@PublicEvolving
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	return config.getRestartStrategy();
}
 
Example 20
Source File: StreamExecutionEnvironment.java    From flink with Apache License 2.0 2 votes vote down vote up
/**
 * Returns the specified restart strategy configuration.
 *
 * @return The restart strategy configuration to be used
 */
@PublicEvolving
public RestartStrategies.RestartStrategyConfiguration getRestartStrategy() {
	return config.getRestartStrategy();
}