org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory Java Examples

The following examples show how to use org.apache.flink.runtime.executiongraph.failover.FailoverStrategy.Factory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ConcurrentFailoverStrategyExecutionGraphTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createSampleGraph(
	JobID jid,
	Factory failoverStrategy,
	RestartStrategy restartStrategy,
	SlotProvider slotProvider,
	int parallelism) throws Exception {

	final JobInformation jobInformation = new DummyJobInformation(
		jid,
		"test job");

	// build a simple execution graph with on job vertex, parallelism 2
	final Time timeout = Time.seconds(10L);
	final ExecutionGraph graph = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		getClass().getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);
	graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

	return graph;
}
Example #2
Source File: ConcurrentFailoverStrategyExecutionGraphTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionGraph createSampleGraph(
	JobID jid,
	Factory failoverStrategy,
	RestartStrategy restartStrategy,
	SlotProvider slotProvider,
	int parallelism) throws Exception {

	final JobInformation jobInformation = new DummyJobInformation(
		jid,
		"test job");

	// build a simple execution graph with on job vertex, parallelism 2
	final Time timeout = Time.seconds(10L);
	final ExecutionGraph graph = new ExecutionGraph(
		jobInformation,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		getClass().getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);

	JobVertex jv = new JobVertex("test vertex");
	jv.setInvokableClass(NoOpInvokable.class);
	jv.setParallelism(parallelism);

	JobGraph jg = new JobGraph(jid, "testjob", jv);
	graph.attachJobGraph(jg.getVerticesSortedTopologicallyFromSources());

	return graph;
}
Example #3
Source File: FailoverRegionTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that two failover regions failover at the same time, they will not influence each other
 * @throws Exception
 */
@Test
public void testMultiRegionFailoverAtSameTime() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 16);

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");

	v1.setParallelism(2);
	v2.setParallelism(2);
	v3.setParallelism(2);
	v4.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);

	ExecutionGraph eg = new ExecutionGraph(
			new DummyJobInformation(
				jobId,
				jobName),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			AkkaUtils.getDefaultTimeout(),
			new InfiniteDelayRestartStrategy(10),
			new RestartPipelinedRegionStrategy.Factory(),
			slotProvider);
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	eg.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	eg.scheduleForExecution();
	RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();

	ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0];
	ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1];
	ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
	ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1];
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());

	ev11.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	ev31.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState());

	ev32.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev31), JobStatus.RUNNING, 1000);

	ev12.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev11), JobStatus.RUNNING, 1000);
}
Example #4
Source File: FailoverRegionTest.java    From flink with Apache License 2.0 4 votes vote down vote up 
/**
 * Tests that two regions failover at the same time, they will not influence each other.
 * @throws Exception if fail to create dummy job information, fail to schedule for execution
 * or timeout before region switches to expected status.
 */
@Test
public void testMultiRegionFailoverAtSameTime() throws Exception {
	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";

	final SimpleSlotProvider slotProvider = new SimpleSlotProvider(jobId, 16);

	JobVertex v1 = new JobVertex("vertex1");
	JobVertex v2 = new JobVertex("vertex2");
	JobVertex v3 = new JobVertex("vertex3");
	JobVertex v4 = new JobVertex("vertex4");

	v1.setParallelism(2);
	v2.setParallelism(2);
	v3.setParallelism(2);
	v4.setParallelism(2);

	v1.setInvokableClass(AbstractInvokable.class);
	v2.setInvokableClass(AbstractInvokable.class);
	v3.setInvokableClass(AbstractInvokable.class);
	v4.setInvokableClass(AbstractInvokable.class);

	v2.connectNewDataSetAsInput(v1, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);
	v4.connectNewDataSetAsInput(v2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);
	v4.connectNewDataSetAsInput(v3, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	List<JobVertex> ordered = Arrays.asList(v1, v2, v3, v4);

	ExecutionGraph eg = new ExecutionGraph(
			new DummyJobInformation(
				jobId,
				jobName),
			TestingUtils.defaultExecutor(),
			TestingUtils.defaultExecutor(),
			AkkaUtils.getDefaultTimeout(),
			new InfiniteDelayRestartStrategy(10),
			new RestartPipelinedRegionStrategy.Factory(),
			slotProvider);
	try {
		eg.attachJobGraph(ordered);
	}
	catch (JobException e) {
		e.printStackTrace();
		fail("Job failed with exception: " + e.getMessage());
	}
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	eg.scheduleForExecution();
	RestartPipelinedRegionStrategy strategy = (RestartPipelinedRegionStrategy)eg.getFailoverStrategy();

	ExecutionVertex ev11 = eg.getJobVertex(v1.getID()).getTaskVertices()[0];
	ExecutionVertex ev12 = eg.getJobVertex(v1.getID()).getTaskVertices()[1];
	ExecutionVertex ev31 = eg.getJobVertex(v3.getID()).getTaskVertices()[0];
	ExecutionVertex ev32 = eg.getJobVertex(v3.getID()).getTaskVertices()[1];
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.RUNNING, strategy.getFailoverRegion(ev31).getState());

	ev11.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	ev31.getCurrentExecutionAttempt().fail(new Exception("new fail"));
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev11).getState());
	assertEquals(JobStatus.CANCELLING, strategy.getFailoverRegion(ev31).getState());

	ev32.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev31), JobStatus.RUNNING, 1000);

	ev12.getCurrentExecutionAttempt().completeCancelling();
	waitUntilFailoverRegionState(strategy.getFailoverRegion(ev11), JobStatus.RUNNING, 1000);
}