org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup Java Examples

The following examples show how to use org.apache.flink.runtime.jobmanager.scheduler.SlotSharingGroup. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: PartialConsumePipelinedResultTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup(
		sender.getID(), receiver.getID());

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 
Example #2
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static Execution getExecution(
		final JobVertexID jid,
		final int subtaskIndex,
		final int numTasks,
		final SlotSharingGroup slotSharingGroup,
		@Nullable final TaskManagerLocation... locations) throws Exception {

	final ExecutionJobVertex ejv = getExecutionJobVertex(
		jid,
		numTasks,
		slotSharingGroup,
		new DirectScheduledExecutorService(),
		ScheduleMode.LAZY_FROM_SOURCES);
	final TestExecutionVertex ev = new TestExecutionVertex(
		ejv,
		subtaskIndex,
		new IntermediateResult[0],
		DEFAULT_TIMEOUT);

	if (locations != null) {
		ev.setPreferredLocationFutures(mapToPreferredLocationFutures(locations));
	}

	return ev.getCurrentExecutionAttempt();
}
 
Example #3
Source File: DefaultExecutionTopologyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraphWithCoLocationConstraint() throws Exception {
	JobVertex[] jobVertices = new JobVertex[2];
	int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, PIPELINED);

	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	jobVertices[0].setSlotSharingGroup(slotSharingGroup);
	jobVertices[1].setSlotSharingGroup(slotSharingGroup);

	CoLocationGroup coLocationGroup = new CoLocationGroup();
	coLocationGroup.addVertex(jobVertices[0]);
	coLocationGroup.addVertex(jobVertices[1]);
	jobVertices[0].updateCoLocationGroup(coLocationGroup);
	jobVertices[1].updateCoLocationGroup(coLocationGroup);

	return createSimpleTestGraph(
		taskManagerGateway,
		triggeredRestartStrategy,
		jobVertices);
}
 
Example #4
Source File: PartialConsumePipelinedResultTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 
Example #5
Source File: SharedSlotsTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Test
public void testImmediateReleaseOneLevel() {
	try {
		JobVertexID vid = new JobVertexID();

		SlotSharingGroup sharingGroup = new SlotSharingGroup(vid);
		SlotSharingGroupAssignment assignment = sharingGroup.getTaskAssignment();

		Instance instance = SchedulerTestUtils.getRandomInstance(1);
		
		SharedSlot sharedSlot = instance.allocateSharedSlot(assignment);

		SimpleSlot sub = assignment.addSharedSlotAndAllocateSubSlot(sharedSlot, Locality.UNCONSTRAINED, vid);
		sub.releaseSlot();
		
		assertTrue(sub.isReleased());
		assertTrue(sharedSlot.isReleased());
		
		assertEquals(1, instance.getNumberOfAvailableSlots());
		assertEquals(0, instance.getNumberOfAllocatedSlots());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example #6
Source File: JobRecoveryITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #7
Source File: TaskExecutorITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 
Example #8
Source File: VertexSlotSharingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testSlotSharingGroupWithSpecifiedResources() {
	final ResourceSpec resource1 = ResourceSpec.newBuilder(0.1, 100).build();
	final ResourceSpec resource2 = ResourceSpec.newBuilder(0.2, 200).build();

	final JobVertex v1 = new JobVertex("v1");
	final JobVertex v2 = new JobVertex("v2");

	v1.setResources(resource1, resource1);
	v2.setResources(resource2, resource2);

	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	v1.setSlotSharingGroup(group1);
	assertEquals(resource1, group1.getResourceSpec());

	v2.setSlotSharingGroup(group1);
	assertEquals(resource1.merge(resource2), group1.getResourceSpec());

	// v1 is moved from group1 to group2
	v1.setSlotSharingGroup(group2);
	assertEquals(resource1, group2.getResourceSpec());
	assertEquals(resource2, group1.getResourceSpec());
}
 
Example #9
Source File: JobExecutionITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example #10
Source File: StreamingJobGraphGenerator.java    From flink with Apache License 2.0 6 votes vote down vote up
private void setSlotSharing() {
	final Map<String, SlotSharingGroup> specifiedSlotSharingGroups = new HashMap<>();
	final Map<JobVertexID, SlotSharingGroup> vertexRegionSlotSharingGroups = buildVertexRegionSlotSharingGroups();

	for (Entry<Integer, JobVertex> entry : jobVertices.entrySet()) {

		final JobVertex vertex = entry.getValue();
		final String slotSharingGroupKey = streamGraph.getStreamNode(entry.getKey()).getSlotSharingGroup();

		final SlotSharingGroup effectiveSlotSharingGroup;
		if (slotSharingGroupKey == null) {
			effectiveSlotSharingGroup = null;
		} else if (slotSharingGroupKey.equals(StreamGraphGenerator.DEFAULT_SLOT_SHARING_GROUP)) {
			// fallback to the region slot sharing group by default
			effectiveSlotSharingGroup = vertexRegionSlotSharingGroups.get(vertex.getID());
		} else {
			effectiveSlotSharingGroup = specifiedSlotSharingGroups.computeIfAbsent(
				slotSharingGroupKey, k -> new SlotSharingGroup());
		}

		vertex.setSlotSharingGroup(effectiveSlotSharingGroup);
	}
}
 
Example #11
Source File: DefaultLogicalTopologyTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(final boolean containsCoLocationConstraint) {
	final JobVertex[] jobVertices = new JobVertex[3];
	final int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[2] = createNoOpVertex("v3", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, PIPELINED);
	jobVertices[2].connectNewDataSetAsInput(jobVertices[1], ALL_TO_ALL, BLOCKING);

	if (containsCoLocationConstraint) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		jobVertices[0].setSlotSharingGroup(slotSharingGroup);
		jobVertices[1].setSlotSharingGroup(slotSharingGroup);

		final CoLocationGroup coLocationGroup = new CoLocationGroup();
		coLocationGroup.addVertex(jobVertices[0]);
		coLocationGroup.addVertex(jobVertices[1]);
		jobVertices[0].updateCoLocationGroup(coLocationGroup);
		jobVertices[1].updateCoLocationGroup(coLocationGroup);
	}

	return new JobGraph(jobVertices);
}
 
Example #12
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 
Example #13
Source File: PartialConsumePipelinedResultTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Tests a fix for FLINK-1930.
 *
 * <p>When consuming a pipelined result only partially, is is possible that local channels
 * release the buffer pool, which is associated with the result partition, too early. If the
 * producer is still producing data when this happens, it runs into an IllegalStateException,
 * because of the destroyed buffer pool.
 *
 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1930">FLINK-1930</a>
 */
@Test
public void testPartialConsumePipelinedResultReceiver() throws Exception {
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(SlowBufferSender.class);
	sender.setParallelism(PARALLELISM);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SingleBufferReceiver.class);
	receiver.setParallelism(PARALLELISM);

	// The partition needs to be pipelined, otherwise the original issue does not occur, because
	// the sender and receiver are not online at the same time.
	receiver.connectNewDataSetAsInput(
		sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph("Partial Consume of Pipelined Result", sender, receiver);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup(
		sender.getID(), receiver.getID());

	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	MINI_CLUSTER_RESOURCE.getMiniCluster().executeJobBlocking(jobGraph);
}
 
Example #14
Source File: SchedulingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #15
Source File: SchedulingITCase.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #16
Source File: ShuffleCompressionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static JobGraph createJobGraph(
		ScheduleMode scheduleMode,
		ResultPartitionType resultPartitionType,
		ExecutionMode executionMode) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	JobVertex source = new JobVertex("source");
	source.setInvokableClass(LongValueSource.class);
	source.setParallelism(PARALLELISM);
	source.setSlotSharingGroup(slotSharingGroup);

	JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(ResultVerifyingSink.class);
	sink.setParallelism(PARALLELISM);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, resultPartitionType);
	JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(scheduleMode);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setExecutionMode(executionMode);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #17
Source File: TaskExecutorITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Blocking receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(BlockingOperator.class);
	BlockingOperator.reset();

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setSlotSharingGroup(slotSharingGroup);

	return new JobGraph("Blocking test job with slot sharing", sender, receiver);
}
 
Example #18
Source File: JobRecoveryITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #19
Source File: JobExecutionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example #20
Source File: SchedulingITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
@Nonnull
private JobGraph createJobGraph(long delay, int parallelism) throws IOException {
	SlotSharingGroup slotSharingGroup = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(OneTimeFailingInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(slotSharingGroup);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(NoOpInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(slotSharingGroup);

	sink.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);
	JobGraph jobGraph = new JobGraph(source, sink);

	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, delay));
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #21
Source File: FileBufferReaderITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static JobGraph createJobGraph() {
	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(TestSourceInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(group1);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(TestSinkInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(group2);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES);

	return jobGraph;
}
 
Example #22
Source File: ExecutionVertexSchedulingRequirementsMapper.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ExecutionVertexSchedulingRequirements from(final ExecutionVertex executionVertex) {

		final ExecutionVertexID executionVertexId = executionVertex.getID();

		final AllocationID latestPriorAllocation = executionVertex.getLatestPriorAllocation();
		final SlotSharingGroup slotSharingGroup = executionVertex.getJobVertex().getSlotSharingGroup();

		return new ExecutionVertexSchedulingRequirements.Builder()
			.withExecutionVertexId(executionVertexId)
			.withPreviousAllocationId(latestPriorAllocation)
			.withTaskResourceProfile(executionVertex.getResourceProfile())
			.withPhysicalSlotResourceProfile(getPhysicalSlotResourceProfile(executionVertex))
			.withSlotSharingGroupId(slotSharingGroup == null ? null : slotSharingGroup.getSlotSharingGroupId())
			.withCoLocationConstraint(executionVertex.getLocationConstraint())
			.build();
	}
 
Example #23
Source File: JobExecutionITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createJobGraph(int parallelism) {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(parallelism);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(parallelism);
	receiver.setInvokableClass(TestingAbstractInvokables.Receiver.class);

	// In order to make testCoLocationConstraintJobExecution fail, one needs to
	// remove the co-location constraint and the slot sharing groups, because then
	// the receivers will have to wait for the senders to finish and the slot
	// assignment order to the receivers is non-deterministic (depending on the
	// order in which the senders finish).
	final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
	receiver.setSlotSharingGroup(slotSharingGroup);
	sender.setSlotSharingGroup(slotSharingGroup);
	receiver.setStrictlyCoLocatedWith(sender);

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);

	return jobGraph;
}
 
Example #24
Source File: FileBufferReaderITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private static JobGraph createJobGraph() {
	final SlotSharingGroup group1 = new SlotSharingGroup();
	final SlotSharingGroup group2 = new SlotSharingGroup();

	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(TestSourceInvokable.class);
	source.setParallelism(parallelism);
	source.setSlotSharingGroup(group1);

	final JobVertex sink = new JobVertex("sink");
	sink.setInvokableClass(TestSinkInvokable.class);
	sink.setParallelism(parallelism);
	sink.setSlotSharingGroup(group2);

	sink.connectNewDataSetAsInput(source, DistributionPattern.ALL_TO_ALL, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(source, sink);
	jobGraph.setScheduleMode(ScheduleMode.LAZY_FROM_SOURCES);

	return jobGraph;
}
 
Example #25
Source File: JobRecoveryITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private JobGraph createjobGraph(boolean slotSharingEnabled) throws IOException {
	final JobVertex sender = new JobVertex("Sender");
	sender.setParallelism(PARALLELISM);
	sender.setInvokableClass(TestingAbstractInvokables.Sender.class);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setParallelism(PARALLELISM);
	receiver.setInvokableClass(FailingOnceReceiver.class);
	FailingOnceReceiver.reset();

	if (slotSharingEnabled) {
		final SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		receiver.setSlotSharingGroup(slotSharingGroup);
		sender.setSlotSharingGroup(slotSharingGroup);
	}

	receiver.connectNewDataSetAsInput(sender, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final ExecutionConfig executionConfig = new ExecutionConfig();
	executionConfig.setRestartStrategy(RestartStrategies.fixedDelayRestart(1, 0L));

	final JobGraph jobGraph = new JobGraph(getClass().getSimpleName(), sender, receiver);
	jobGraph.setExecutionConfig(executionConfig);

	return jobGraph;
}
 
Example #26
Source File: StreamingJobGraphGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up
private void setSlotSharingAndCoLocation() {
	final HashMap<String, SlotSharingGroup> slotSharingGroups = new HashMap<>();
	final HashMap<String, Tuple2<SlotSharingGroup, CoLocationGroup>> coLocationGroups = new HashMap<>();

	for (Entry<Integer, JobVertex> entry : jobVertices.entrySet()) {

		final StreamNode node = streamGraph.getStreamNode(entry.getKey());
		final JobVertex vertex = entry.getValue();

		// configure slot sharing group
		final String slotSharingGroupKey = node.getSlotSharingGroup();
		final SlotSharingGroup sharingGroup;

		if (slotSharingGroupKey != null) {
			sharingGroup = slotSharingGroups.computeIfAbsent(
					slotSharingGroupKey, (k) -> new SlotSharingGroup());
			vertex.setSlotSharingGroup(sharingGroup);
		} else {
			sharingGroup = null;
		}

		// configure co-location constraint
		final String coLocationGroupKey = node.getCoLocationGroup();
		if (coLocationGroupKey != null) {
			if (sharingGroup == null) {
				throw new IllegalStateException("Cannot use a co-location constraint without a slot sharing group");
			}

			Tuple2<SlotSharingGroup, CoLocationGroup> constraint = coLocationGroups.computeIfAbsent(
					coLocationGroupKey, (k) -> new Tuple2<>(sharingGroup, new CoLocationGroup()));

			if (constraint.f0 != sharingGroup) {
				throw new IllegalStateException("Cannot co-locate operators from different slot sharing groups");
			}

			vertex.updateCoLocationGroup(constraint.f1);
			constraint.f1.addVertex(vertex);
		}
	}
}
 
Example #27
Source File: JobVertex.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Associates this vertex with a slot sharing group for scheduling. Different vertices in the same
 * slot sharing group can run one subtask each in the same slot.
 * 
 * @param grp The slot sharing group to associate the vertex with.
 */
public void setSlotSharingGroup(SlotSharingGroup grp) {
	if (this.slotSharingGroup != null) {
		this.slotSharingGroup.removeVertexFromGroup(id);
	}

	this.slotSharingGroup = grp;
	if (grp != null) {
		grp.addVertexToGroup(id);
	}
}
 
Example #28
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This test checks that are strictly co-located vertices are in the same failover region,
 * even through they are connected via a blocking pattern.
 * This is currently an assumption / limitation of the scheduler.
 */
@Test
public void testPipelinedOneToOneTopologyWithCoLocation() throws Exception {
	final JobVertex source = new JobVertex("source");
	source.setInvokableClass(NoOpInvokable.class);
	source.setParallelism(10);

	final JobVertex target = new JobVertex("target");
	target.setInvokableClass(NoOpInvokable.class);
	target.setParallelism(10);

	target.connectNewDataSetAsInput(source, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final SlotSharingGroup sharingGroup = new SlotSharingGroup();
	source.setSlotSharingGroup(sharingGroup);
	target.setSlotSharingGroup(sharingGroup);

	source.setStrictlyCoLocatedWith(target);

	final JobGraph jobGraph = new JobGraph("test job", source, target);
	final ExecutionGraph eg = createExecutionGraph(jobGraph);

	RestartPipelinedRegionStrategy failoverStrategy = (RestartPipelinedRegionStrategy) eg.getFailoverStrategy();
	FailoverRegion sourceRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[0]);
	FailoverRegion sourceRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(source.getID()).getTaskVertices()[1]);
	FailoverRegion targetRegion1 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[0]);
	FailoverRegion targetRegion2 = failoverStrategy.getFailoverRegion(eg.getJobVertex(target.getID()).getTaskVertices()[1]);

	// we use 'assertTrue' here rather than 'assertEquals' because we want to test
	// for referential equality, to be on the safe side
	assertTrue(sourceRegion1 == sourceRegion2);
	assertTrue(sourceRegion2 == targetRegion1);
	assertTrue(targetRegion1 == targetRegion2);
}
 
Example #29
Source File: DefaultFailoverTopologyTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private ExecutionGraph createExecutionGraph(boolean addCollocationConstraints) throws Exception {
	JobVertex[] jobVertices = new JobVertex[3];
	int parallelism = 3;
	jobVertices[0] = createNoOpVertex("v1", parallelism);
	jobVertices[1] = createNoOpVertex("v2", parallelism);
	jobVertices[2] = createNoOpVertex("v3", parallelism);
	jobVertices[1].connectNewDataSetAsInput(jobVertices[0], ALL_TO_ALL, BLOCKING);
	jobVertices[2].connectNewDataSetAsInput(jobVertices[1], POINTWISE, PIPELINED);

	if (addCollocationConstraints) {
		SlotSharingGroup slotSharingGroup = new SlotSharingGroup();
		jobVertices[1].setSlotSharingGroup(slotSharingGroup);
		jobVertices[2].setSlotSharingGroup(slotSharingGroup);

		CoLocationGroup coLocationGroup = new CoLocationGroup();
		coLocationGroup.addVertex(jobVertices[1]);
		coLocationGroup.addVertex(jobVertices[2]);
		jobVertices[1].updateCoLocationGroup(coLocationGroup);
		jobVertices[2].updateCoLocationGroup(coLocationGroup);
	}

	return createSimpleTestGraph(
		new JobID(),
		taskManagerGateway,
		triggeredRestartStrategy,
		jobVertices);
}
 
Example #30
Source File: StreamingJobGraphGenerator.java    From flink with Apache License 2.0 5 votes vote down vote up
private static void setManagedMemoryFractionForSlotSharingGroup(
		final SlotSharingGroup slotSharingGroup,
		final Map<JobVertexID, Integer> vertexHeadOperators,
		final Map<JobVertexID, Set<Integer>> vertexOperators,
		final Map<Integer, StreamConfig> operatorConfigs,
		final Map<Integer, Map<Integer, StreamConfig>> vertexChainedConfigs,
		final java.util.function.Function<Integer, ResourceSpec> operatorResourceRetriever,
		final java.util.function.Function<Integer, Integer> operatorManagedMemoryWeightRetriever) {

	final int groupManagedMemoryWeight = slotSharingGroup.getJobVertexIds().stream()
		.flatMap(vid -> vertexOperators.get(vid).stream())
		.mapToInt(operatorManagedMemoryWeightRetriever::apply)
		.sum();

	for (JobVertexID jobVertexID : slotSharingGroup.getJobVertexIds()) {
		for (int operatorNodeId : vertexOperators.get(jobVertexID)) {
			final StreamConfig operatorConfig = operatorConfigs.get(operatorNodeId);
			final ResourceSpec operatorResourceSpec = operatorResourceRetriever.apply(operatorNodeId);
			final int operatorManagedMemoryWeight = operatorManagedMemoryWeightRetriever.apply(operatorNodeId);
			setManagedMemoryFractionForOperator(
				operatorResourceSpec,
				slotSharingGroup.getResourceSpec(),
				operatorManagedMemoryWeight,
				groupManagedMemoryWeight,
				operatorConfig);
		}

		// need to refresh the chained task configs because they are serialized
		final int headOperatorNodeId = vertexHeadOperators.get(jobVertexID);
		final StreamConfig vertexConfig = operatorConfigs.get(headOperatorNodeId);
		vertexConfig.setTransitiveChainedTaskConfigs(vertexChainedConfigs.get(headOperatorNodeId));
	}
}