Java Code Examples for org.apache.flink.runtime.jobgraph.JobVertex#setStrictlyCoLocatedWith()

The following examples show how to use org.apache.flink.runtime.jobgraph.JobVertex#setStrictlyCoLocatedWith() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: JobExecutionITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example 2
Source File: JobExecutionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example 3
Source File: JobExecutionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example 4
Source File: PipelinedFailoverRegionBuildingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testBlockingAllToAllTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(13);

	target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(region1 == region2);
}
 
Example 5
Source File: PipelinedFailoverRegionBuildingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 
Example 6
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testBlockingAllToAllTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(13);

	target.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion region1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion region2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(region1 == region2);
}
 
Example 7
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 
Example 8
Source File: DefaultSchedulerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void coLocationConstraintIsResetOnTaskRecovery() {
	final JobGraph jobGraph = nonParallelSourceSinkJobGraph();
	final JobVertex source = jobGraph.getVerticesSortedTopologicallyFromSources().get(0);
	final JobVertex sink = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);

	final SlotSharingGroup ssg = new SlotSharingGroup();
	source.setSlotSharingGroup(ssg);
	sink.setSlotSharingGroup(ssg);
	sink.setStrictlyCoLocatedWith(source);

	final JobID jobId = jobGraph.getJobID();
	final DefaultScheduler scheduler = createSchedulerAndStartScheduling(jobGraph);

	final ExecutionVertex sourceVertex = scheduler.getExecutionVertex(new ExecutionVertexID(source.getID(), 0));
	final ExecutionAttemptID sourceAttemptId = sourceVertex.getCurrentExecutionAttempt().getAttemptId();
	final ExecutionVertex sinkVertex = scheduler.getExecutionVertex(new ExecutionVertexID(sink.getID(), 0));
	final ExecutionAttemptID sinkAttemptId = sinkVertex.getCurrentExecutionAttempt().getAttemptId();

	// init the location constraint manually because the testExecutionSlotAllocator does not do it
	sourceVertex.getLocationConstraint().setSlotRequestId(new SlotRequestId());
	assertThat(sourceVertex.getLocationConstraint().getSlotRequestId(), is(notNullValue()));

	final String exceptionMessage = "expected exception";
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, sourceAttemptId, ExecutionState.FAILED, new RuntimeException(exceptionMessage)));
	scheduler.updateTaskExecutionState(new TaskExecutionState(jobId, sinkAttemptId, ExecutionState.CANCELED));
	taskRestartExecutor.triggerScheduledTasks();

	assertThat(sourceVertex.getLocationConstraint().getSlotRequestId(), is(nullValue()));
}
 
Example 9
Source File: ExecutionGraphCoLocationRestartTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		testingSlotProvider,
		new TestRestartStrategy(
			1,
			false),
		groupVertex,
		groupVertex2);

	// enable the queued scheduling for the slot pool
	eg.setQueuedSchedulingAllowed(true);
	eg.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}
 
Example 10
Source File: ExecutionGraphCoLocationRestartTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(groupVertex, groupVertex2)
		.setSlotProvider(testingSlotProvider)
		.setRestartStrategy(
			new TestRestartStrategy(
				1,
				false))
		.allowQueuedScheduling()
		.build();

	// enable the queued scheduling for the slot pool
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}
 
Example 11
Source File: ExecutionGraphCoLocationRestartTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Test
public void testConstraintsAfterRestart() throws Exception {

	final long timeout = 5000L;

	//setting up
	testingSlotProvider.addTaskManager(NUM_TASKS);

	JobVertex groupVertex = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);
	JobVertex groupVertex2 = ExecutionGraphTestUtils.createNoOpVertex(NUM_TASKS);

	SlotSharingGroup sharingGroup = new SlotSharingGroup();
	groupVertex.setSlotSharingGroup(sharingGroup);
	groupVertex2.setSlotSharingGroup(sharingGroup);
	groupVertex.setStrictlyCoLocatedWith(groupVertex2);

	//initiate and schedule job
	final ExecutionGraph eg = TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(new JobGraph(groupVertex, groupVertex2))
		.setSlotProvider(testingSlotProvider)
		.setRestartStrategy(new TestRestartStrategy(1, false))
		.build();

	// enable the queued scheduling for the slot pool
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	assertEquals(JobStatus.CREATED, eg.getState());

	eg.scheduleForExecution();

	Predicate<AccessExecution> isDeploying = ExecutionGraphTestUtils.isInExecutionState(ExecutionState.DEPLOYING);
	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	assertEquals(JobStatus.RUNNING, eg.getState());

	//sanity checks
	validateConstraints(eg);

	eg.getAllExecutionVertices().iterator().next().fail(new FlinkException("Test exception"));

	assertEquals(JobStatus.FAILING, eg.getState());

	for (ExecutionVertex vertex : eg.getAllExecutionVertices()) {
		vertex.getCurrentExecutionAttempt().completeCancelling();
	}

	// wait until we have restarted
	ExecutionGraphTestUtils.waitUntilJobStatus(eg, JobStatus.RUNNING, timeout);

	ExecutionGraphTestUtils.waitForAllExecutionsPredicate(
		eg,
		isDeploying,
		timeout);

	//checking execution vertex properties
	validateConstraints(eg);

	ExecutionGraphTestUtils.finishAllVertices(eg);

	assertThat(eg.getState(), is(FINISHED));
}