org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup Java Examples

/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup(
		sender.getID(), receiver.getID());

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 

public static Execution getExecution(
		final JobVertexID jid,
		final int subtaskIndex,
		final int numTasks,
		final SlotSharingGroup slotSharingGroup,
		@Nullable final TaskManagerLocation... locations) throws Exception {

	final ExecutionJobVertex ejv = getExecutionJobVertex(
		jid,
		numTasks,
		slotSharingGroup,
		new DirectScheduledExecutorService(),
		ScheduleMode.LAZY_FROM_SOURCES);
	final TestExecutionVertex ev = new TestExecutionVertex(
		ejv,
		subtaskIndex,
		new IntermediateResult[0],
		DEFAULT_TIMEOUT);

	if (locations != null) {
		ev.setPreferredLocationFutures(mapToPreferredLocationFutures(locations));
	}

	return ev.getCurrentExecutionAttempt();
}
 

private ExecutionGraph createExecutionGraphWithCoLocationConstraint() throws Exception {
	JobVertex[] jobVertices = new JobVertex[2];
	int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, PIPELINED);

	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	jobVertices[0].setSlotSharingGroup(slotSharingGroup);
	jobVertices[1].setSlotSharingGroup(slotSharingGroup);

	CoLocationGroup coLocationGroup = new CoLocationGroup();
	coLocationGroup.addVertex(jobVertices[0]);
	coLocationGroup.addVertex(jobVertices[1]);
	jobVertices[0].updateCoLocationGroup(coLocationGroup);
	jobVertices[1].updateCoLocationGroup(coLocationGroup);

	return createSimpleTestGraph(
		taskManagerGateway,
		triggeredRestartStrategy,
		jobVertices);
}
 

/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 

@Test
public void testImmediateReleaseOneLevel() {
	try {
		JobVertexID vid = new JobVertexID();

		SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
		SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();

		Instance instance = SchedulerTestUtils.getRandomInstance(1);
		
		SharedSlot sharedSlot = instance.allocateSharedSlot(assignment);

		SimpleSlot sub = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid);
		sub.releaseSlot();
		
		assertTrue(sub.isReleased());
		assertTrue(sharedSlot.isReleased());
		
		assertEquals(1, instance.getNumberOfAvailableSlots());
		assertEquals(0, instance.getNumberOfAllocatedSlots());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 

private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 

@Test
public void testSlotSharingGroupWithSpecifiedResources() {
	final ResourceSpec resource1 = ResourceSpec.newBuilder(0.1, 100).build();
	final ResourceSpec resource2 = ResourceSpec.newBuilder(0.2, 200).build();

	final JobVertex v1 = new JobVertex("v1");
	final JobVertex v2 = new JobVertex("v2");

	v1.setResources(resource1, resource1);
	v2.setResources(resource2, resource2);

	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	v1.setSlotSharingGroup(group1);
	assertEquals(resource1, group1.getResourceSpec());

	v2.setSlotSharingGroup(group1);
	assertEquals(resource1.merge(resource2), group1.getResourceSpec());

	// v1 is moved from group1 to group2
	v1.setSlotSharingGroup(group2);
	assertEquals(resource1, group2.getResourceSpec());
	assertEquals(resource2, group1.getResourceSpec());
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

private void setSlotSharing() {
	final Map<String, SlotSharingGroup> specifiedSlotSharingGroups = new HashMap<>();
	final Map<JobVertexID, SlotSharingGroup> vertexRegionSlotSharingGroups = buildVertexRegionSlotSharingGroups();

	for (Entry<Integer, JobVertex> entry : jobVertices.entrySet()) {

		final JobVertex vertex = entry.getValue();
		final String slotSharingGroupKey = streamGraph.getStreamNode(entry.getKey()).getSlotSharingGroup();

		final SlotSharingGroup effectiveSlotSharingGroup;
		if (slotSharingGroupKey == null) {
			effectiveSlotSharingGroup = null;
		} else if (slotSharingGroupKey.equals(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP)) {
			// fallback to the region slot sharing group by default
			effectiveSlotSharingGroup = vertexRegionSlotSharingGroups.get(vertex.getID());
		} else {
			effectiveSlotSharingGroup = specifiedSlotSharingGroups.computeIfAbsent(
				slotSharingGroupKey, k -> new SlotSharingGroup());
		}

		vertex.setSlotSharingGroup(effectiveSlotSharingGroup);
	}
}
 

private JobGraph createJobGraph(final boolean containsCoLocationConstraint) {
	final JobVertex[] jobVertices = new JobVertex[3];
	final int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[2] = createNoOpVertex("v3", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, PIPELINED);
	jobVertices[2].connectNewDataSetAsInput(jobVertices[1], ALL_TO_ALL, BLOCKING);

	if (containsCoLocationConstraint) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		jobVertices[0].setSlotSharingGroup(slotSharingGroup);
		jobVertices[1].setSlotSharingGroup(slotSharingGroup);

		final CoLocationGroup coLocationGroup = new CoLocationGroup();
		coLocationGroup.addVertex(jobVertices[0]);
		coLocationGroup.addVertex(jobVertices[1]);
		jobVertices[0].updateCoLocationGroup(coLocationGroup);
		jobVertices[1].updateCoLocationGroup(coLocationGroup);
	}

	return new JobGraph(jobVertices);
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 

/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup(
		sender.getID(), receiver.getID());

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 

@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private static JobGraph createJobGraph(
		ScheduleMode scheduleMode,
		ResultPartitionType resultPartitionType,
		ExecutionMode executionMode) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	JobVertex source = new JobVertex("source");
	source.setInvokableClass(LongValueSource.class);
	source.setParallelism(PARALLELISM);
	source.setSlotSharingGroup(slotSharingGroup);

	JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(ResultVerifyingSink.class);
	sink.setParallelism(PARALLELISM);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
	JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(scheduleMode);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setExecutionMode(executionMode);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 

private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private static JobGraph createJobGraph() {
	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(TestSourceInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(group1);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(TestSinkInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(group2);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES);

	return jobGraph;
}
 

public static ExecutionVertexSchedulingRequirements from(final ExecutionVertex executionVertex) {

		final ExecutionVertexID executionVertexId = executionVertex.getID();

		final AllocationID latestPriorAllocation = executionVertex.getLatestPriorAllocation();
		final SlotSharingGroup slotSharingGroup = executionVertex.getJobVertex().getSlotSharingGroup();

		return new ExecutionVertexSchedulingRequirements.Builder()
			.withExecutionVertexId(executionVertexId)
			.withPreviousAllocationId(latestPriorAllocation)
			.withTaskResourceProfile(executionVertex.getResourceProfile())
			.withPhysicalSlotResourceProfile(getPhysicalSlotResourceProfile(executionVertex))
			.withSlotSharingGroupId(slotSharingGroup == null ? null : slotSharingGroup.getSlotSharingGroupId())
			.withCoLocationConstraint(executionVertex.getLocationConstraint())
			.build();
	}
 

private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 

private static JobGraph createJobGraph() {
	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(TestSourceInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(group1);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(TestSinkInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(group2);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES);

	return jobGraph;
}
 

private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 

private void setSlotSharingAndCoLocation() {
	final HashMap<String, SlotSharingGroup> slotSharingGroups = new HashMap<>();
	final HashMap<String, Tuple2<SlotSharingGroup, CoLocationGroup>> coLocationGroups = new HashMap<>();

	for (Entry<Integer, JobVertex> entry : jobVertices.entrySet()) {

		final StreamNode node = streamGraph.getStreamNode(entry.getKey());
		final JobVertex vertex = entry.getValue();

		// configure slot sharing group
		final String slotSharingGroupKey = node.getSlotSharingGroup();
		final SlotSharingGroup sharingGroup;

		if (slotSharingGroupKey != null) {
			sharingGroup = slotSharingGroups.computeIfAbsent(
					slotSharingGroupKey, (k) -> new SlotSharingGroup());
			vertex.setSlotSharingGroup(sharingGroup);
		} else {
			sharingGroup = null;
		}

		// configure co-location constraint
		final String coLocationGroupKey = node.getCoLocationGroup();
		if (coLocationGroupKey != null) {
			if (sharingGroup == null) {
				throw new IllegalStateException("Cannot use a co-location constraint without a slot sharing group");
			}

			Tuple2<SlotSharingGroup, CoLocationGroup> constraint = coLocationGroups.computeIfAbsent(
					coLocationGroupKey, (k) -> new Tuple2<>(sharingGroup, new CoLocationGroup()));

			if (constraint.f0 != sharingGroup) {
				throw new IllegalStateException("Cannot co-locate operators from different slot sharing groups");
			}

			vertex.updateCoLocationGroup(constraint.f1);
			constraint.f1.addVertex(vertex);
		}
	}
}
 

/**
 * Associates this vertex with a slot sharing group for scheduling. Different vertices in the same
 * slot sharing group can run one subtask each in the same slot.
 * 
 * @param grp The slot sharing group to associate the vertex with.
 */
public void setSlotSharingGroup(SlotSharingGroup grp) {
	if (this.slotSharingGroup != null) {
		this.slotSharingGroup.removeVertexFromGroup(id);
	}

	this.slotSharingGroup = grp;
	if (grp != null) {
		grp.addVertexToGroup(id);
	}
}
 

/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 

private ExecutionGraph createExecutionGraph(boolean addCollocationConstraints) throws Exception {
	JobVertex[] jobVertices = new JobVertex[3];
	int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[2] = createNoOpVertex("v3", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, BLOCKING);
	jobVertices[2].connectNewDataSetAsInput(jobVertices[1], POINTWISE, PIPELINED);

	if (addCollocationConstraints) {
		SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		jobVertices[1].setSlotSharingGroup(slotSharingGroup);
		jobVertices[2].setSlotSharingGroup(slotSharingGroup);

		CoLocationGroup coLocationGroup = new CoLocationGroup();
		coLocationGroup.addVertex(jobVertices[1]);
		coLocationGroup.addVertex(jobVertices[2]);
		jobVertices[1].updateCoLocationGroup(coLocationGroup);
		jobVertices[2].updateCoLocationGroup(coLocationGroup);
	}

	return createSimpleTestGraph(
		new JobID(),
		taskManagerGateway,
		triggeredRestartStrategy,
		jobVertices);
}
 

private static void setManagedMemoryFractionForSlotSharingGroup(
		final SlotSharingGroup slotSharingGroup,
		final Map<JobVertexID, Integer> vertexHeadOperators,
		final Map<JobVertexID, Set<Integer>> vertexOperators,
		final Map<Integer, StreamConfig> operatorConfigs,
		final Map<Integer, Map<Integer, StreamConfig>> vertexChainedConfigs,
		final java.util.function.Function<Integer, ResourceSpec> operatorResourceRetriever,
		final java.util.function.Function<Integer, Integer> operatorManagedMemoryWeightRetriever) {

	final int groupManagedMemoryWeight = slotSharingGroup.getJobVertexIds().stream()
		.flatMap(vid -> vertexOperators.get(vid).stream())
		.mapToInt(operatorManagedMemoryWeightRetriever::apply)
		.sum();

	for (JobVertexID jobVertexID : slotSharingGroup.getJobVertexIds()) {
		for (int operatorNodeId : vertexOperators.get(jobVertexID)) {
			final StreamConfig operatorConfig = operatorConfigs.get(operatorNodeId);
			final ResourceSpec operatorResourceSpec = operatorResourceRetriever.apply(operatorNodeId);
			final int operatorManagedMemoryWeight = operatorManagedMemoryWeightRetriever.apply(operatorNodeId);
			setManagedMemoryFractionForOperator(
				operatorResourceSpec,
				slotSharingGroup.getResourceSpec(),
				operatorManagedMemoryWeight,
				groupManagedMemoryWeight,
				operatorConfig);
		}

		// need to refresh the chained task configs because they are serialized
		final int headOperatorNodeId = vertexHeadOperators.get(jobVertexID);
		final StreamConfig vertexConfig = operatorConfigs.get(headOperatorNodeId);
		vertexConfig.setTransitiveChainedTaskConfigs(vertexChainedConfigs.get(headOperatorNodeId));
	}
}