org.apache.flink.runtime.jobmaster.slotpool.SlotProvider Java Examples

The following examples show how to use org.apache.flink.runtime.jobmaster.slotpool.SlotProvider. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * This constructor is for tests only, because it does not include class loading information.
 */
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		new RestartAllStrategy.Factory(),
		slotProvider);
}
 
Example #2
Source File: ExecutionGraphTestUtils.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an execution graph containing the given vertices and the given restart strategy.
 */
public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		TaskManagerGateway taskManagerGateway,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	int numSlotsNeeded = 0;
	for (JobVertex vertex : vertices) {
		numSlotsNeeded += vertex.getParallelism();
	}

	SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded, taskManagerGateway);

	return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
}
 
Example #3
Source File: SchedulerNGFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
JobMasterPartitionTracker partitionTracker) throws Exception;
 
Example #4
Source File: ExecutionGraphSchedulingTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 
Example #5
Source File: SlotProviderStrategy.java    From flink with Apache License 2.0 6 votes vote down vote up
static SlotProviderStrategy from(
	ScheduleMode scheduleMode,
	SlotProvider slotProvider,
	Time allocationTimeout,
	boolean allowQueuedScheduling) {

	switch (scheduleMode) {
		case LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST:
			return new BatchSlotProviderStrategy(slotProvider, allowQueuedScheduling);
		case LAZY_FROM_SOURCES:
		case EAGER:
			return new NormalSlotProviderStrategy(slotProvider, allocationTimeout, allowQueuedScheduling);
		default:
			throw new IllegalArgumentException(String.format("Unknown scheduling mode: %s", scheduleMode));
	}
}
 
Example #6
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Creates an execution graph containing the given vertices and the given restart strategy.
 */
public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		TaskManagerGateway taskManagerGateway,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	int numSlotsNeeded = 0;
	for (JobVertex vertex : vertices) {
		numSlotsNeeded += vertex.getParallelism();
	}

	SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded, taskManagerGateway);

	return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
}
 
Example #7
Source File: ExecutionGraphRestartTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph newExecutionGraph(RestartStrategy restartStrategy, SlotProvider slotProvider) throws IOException {
	final ExecutionGraph executionGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"Test job",
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		slotProvider);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	return executionGraph;
}
 
Example #8
Source File: PipelinedFailoverRegionBuildingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 
Example #9
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ExecutionGraph createExecutionGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(jid);
	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return new TestingExecutionGraphBuilder(vertices)
		.setFutureExecutor(executor)
		.setIoExecutor(executor)
		.setSlotProvider(slotProvider)
		.setAllocationTimeout(timeout)
		.setRpcTimeout(timeout)
		.setRestartStrategy(restartStrategy)
		.build();
}
 
Example #10
Source File: ExecutionVertexInputConstraintTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(
		List<JobVertex> orderedVertices,
		InputDependencyConstraint inputDependencyConstraint) throws Exception {

	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";
	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20);

	for (JobVertex vertex : orderedVertices) {
		vertex.setInputDependencyConstraint(inputDependencyConstraint);
	}

	ExecutionGraph eg = new ExecutionGraph(
		new DummyJobInformation(
			jobId,
			jobName),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		TestRestartStrategy.directExecuting(),
		new RestartAllStrategy.Factory(),
		slotProvider);
	eg.attachJobGraph(orderedVertices);

	return eg;
}
 
Example #11
Source File: ExecutionGraphSuspendTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 
Example #12
Source File: ExecutionGraphSuspendTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 
Example #13
Source File: ExecutionVertexInputConstraintTest.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(
		List<JobVertex> orderedVertices,
		InputDependencyConstraint inputDependencyConstraint) throws Exception {

	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";
	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20);

	for (JobVertex vertex : orderedVertices) {
		vertex.setInputDependencyConstraint(inputDependencyConstraint);
	}

	ExecutionGraph eg = new ExecutionGraph(
		new DummyJobInformation(
			jobId,
			jobName),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		TestRestartStrategy.directExecuting(),
		new RestartAllStrategy.Factory(),
		slotProvider);
	eg.attachJobGraph(orderedVertices);

	return eg;
}
 
Example #14
Source File: DefaultSchedulerFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ExecutionSlotAllocatorFactory createExecutionSlotAllocatorFactory(
		final ScheduleMode scheduleMode,
		final SlotProvider slotProvider,
		final Time slotRequestTimeout,
		final SchedulingStrategyFactory schedulingStrategyFactory) {

	if (schedulingStrategyFactory instanceof PipelinedRegionSchedulingStrategy.Factory) {
		return new OneSlotPerExecutionSlotAllocatorFactory(
			slotProvider,
			scheduleMode != ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST,
			slotRequestTimeout);
	} else {
		final SlotProviderStrategy slotProviderStrategy = SlotProviderStrategy.from(
			scheduleMode,
			slotProvider,
			slotRequestTimeout);

		return new DefaultExecutionSlotAllocatorFactory(slotProviderStrategy);
	}
}
 
Example #15
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 6 votes vote down vote up
public static ExecutionGraph createExecutionGraph(
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(new JobGraph(vertices))
		.setFutureExecutor(executor)
		.setIoExecutor(executor)
		.setSlotProvider(slotProvider)
		.setAllocationTimeout(timeout)
		.setRpcTimeout(timeout)
		.setRestartStrategy(restartStrategy)
		.build();
}
 
Example #16
Source File: ExecutionGraphSchedulingTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 
Example #17
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	final ExecutionGraph executionGraph = new ExecutionGraph(
		new JobInformation(
			TEST_JOB_ID,
			"Test job",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList()),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		failoverStrategyFactory,
		slotProvider);

	executionGraph.start(mainThreadExecutor);
	executionGraph.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	return executionGraph;
}
 
Example #18
Source File: ExecutionGraph.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Void> scheduleLazy(SlotProvider slotProvider) {

		final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>(numVerticesTotal);
		// simply take the vertices without inputs.
		for (ExecutionJobVertex ejv : verticesInCreationOrder) {
			if (ejv.getJobVertex().isInputVertex()) {
				final CompletableFuture<Void> schedulingJobVertexFuture = ejv.scheduleAll(
					slotProvider,
					allowQueuedScheduling,
					LocationPreferenceConstraint.ALL, // since it is an input vertex, the input based location preferences should be empty
					Collections.emptySet());

				schedulingFutures.add(schedulingJobVertexFuture);
			}
		}

		return FutureUtils.waitForAll(schedulingFutures);
	}
 
Example #19
Source File: ExecutionJobVertex.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Schedules all execution vertices of this ExecutionJobVertex.
 *
 * @param slotProvider to allocate the slots from
 * @param queued if the allocations can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds set with all previous allocation ids in the job graph.
 *                                                 Can be empty if the allocation ids are not required for scheduling.
 * @return Future which is completed once all {@link Execution} could be deployed
 */
public CompletableFuture<Void> scheduleAll(
		SlotProvider slotProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds) {

	final ExecutionVertex[] vertices = this.taskVertices;

	final ArrayList<CompletableFuture<Void>> scheduleFutures = new ArrayList<>(vertices.length);

	// kick off the tasks
	for (ExecutionVertex ev : vertices) {
		scheduleFutures.add(ev.scheduleForExecution(
			slotProvider,
			queued,
			locationPreferenceConstraint,
			allPreviousExecutionGraphAllocationIds));
	}

	return FutureUtils.waitForAll(scheduleFutures);
}
 
Example #20
Source File: ExecutionGraph.java    From flink with Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 
Example #21
Source File: SchedulerNGFactory.java    From flink with Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
PartitionTracker partitionTracker) throws Exception;
 
Example #22
Source File: ExecutionGraphTestUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ExecutionGraph createExecutionGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		JobVertex... vertices) throws Exception {

		return createExecutionGraph(jid, slotProvider, restartStrategy, executor, Time.seconds(10L), vertices);
}
 
Example #23
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * SlotPool#failAllocation should not fail with a {@link java.util.ConcurrentModificationException}
 * if there is a concurrent scheduling operation. See FLINK-13421.
 */
@Test
public void slotPoolExecutionGraph_ConcurrentSchedulingAndAllocationFailure_ShouldNotFailWithConcurrentModificationException() throws Exception {
	final SlotSharingGroup group = new SlotSharingGroup();
	final JobVertex vertex1 = createNoOpVertex("vertex1", 1);
	vertex1.setSlotSharingGroup(group);
	final JobVertex vertex2 = createNoOpVertex("vertex2", 3);
	vertex2.setSlotSharingGroup(group);
	final JobVertex vertex3 = createNoOpVertex("vertex3", 1);
	vertex3.setSlotSharingGroup(group);
	vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	try (SlotPool slotPool = createSlotPoolImpl()) {
		final SlotProvider slots = createSchedulerWithSlots(slotPool, new LocalTaskManagerLocation(), 2);

		final AllocationID allocationId = slotPool.getAvailableSlotsInformation().iterator().next().getAllocationId();

		final JobGraph jobGraph = new JobGraph(TEST_JOB_ID, "Test Job", vertex1, vertex2, vertex3);
		jobGraph.setScheduleMode(ScheduleMode.EAGER);
		final ExecutionGraph eg = TestingExecutionGraphBuilder
			.newBuilder()
			.setJobGraph(jobGraph)
			.setSlotProvider(slots)
			.setAllocationTimeout(Time.minutes(60))
			.build();

		startAndScheduleExecutionGraph(eg);

		slotPool.failAllocation(
			allocationId,
			new Exception("test exception"));

		eg.waitUntilTerminal();
	}
}
 
Example #24
Source File: LegacyJobVertexIdTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testIntroduceLegacyJobVertexIds() throws Exception {
	JobVertexID defaultId = new JobVertexID();
	JobVertexID legacyId1 = new JobVertexID();
	JobVertexID legacyId2 = new JobVertexID();

	JobVertex jobVertex = new JobVertex("test", defaultId, Arrays.asList(legacyId1, legacyId2), new ArrayList<OperatorID>(), new ArrayList<OperatorID>());
	jobVertex.setInvokableClass(AbstractInvokable.class);

	ExecutionGraph executionGraph = new ExecutionGraph(
		mock(ScheduledExecutorService.class),
		mock(Executor.class),
		new JobID(),
		"test",
		mock(Configuration.class),
		mock(SerializedValue.class),
		Time.seconds(1),
		mock(RestartStrategy.class),
		mock(SlotProvider.class));

	ExecutionJobVertex executionJobVertex =
			new ExecutionJobVertex(executionGraph, jobVertex, 1, Time.seconds(1));

	Map<JobVertexID, ExecutionJobVertex> idToVertex = new HashMap<>();
	idToVertex.put(executionJobVertex.getJobVertexId(), executionJobVertex);

	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertNull(idToVertex.get(legacyId1));
	Assert.assertNull(idToVertex.get(legacyId2));

	idToVertex = ExecutionJobVertex.includeLegacyJobVertexIDs(idToVertex);

	Assert.assertEquals(3, idToVertex.size());
	Assert.assertEquals(executionJobVertex, idToVertex.get(defaultId));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId1));
	Assert.assertEquals(executionJobVertex, idToVertex.get(legacyId2));
}
 
Example #25
Source File: ExecutionGraphSuspendTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 
Example #26
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider,
		ClassLoader userClassLoader,
		BlobWriter blobWriter,
		Time allocationTimeout) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		JobManagerOptions.MAX_ATTEMPTS_HISTORY_SIZE.defaultValue(),
		failoverStrategy,
		slotProvider,
		userClassLoader,
		blobWriter,
		allocationTimeout,
		new NotReleasingPartitionReleaseStrategy.Factory(),
		NettyShuffleMaster.INSTANCE,
		new PartitionTrackerImpl(
			jobInformation.getJobId(),
			NettyShuffleMaster.INSTANCE,
			ignored -> Optional.empty()),
		ScheduleMode.LAZY_FROM_SOURCES,
		false);
}
 
Example #27
Source File: ExecutionGraph.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * This constructor is for tests only, because it sets default values for many fields.
 */
@VisibleForTesting
ExecutionGraph(
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		JobID jobId,
		String jobName,
		Configuration jobConfig,
		SerializedValue<ExecutionConfig> serializedConfig,
		Time timeout,
		RestartStrategy restartStrategy,
		SlotProvider slotProvider) throws IOException {

	this(
		new JobInformation(
			jobId,
			jobName,
			serializedConfig,
			jobConfig,
			Collections.emptyList(),
			Collections.emptyList()),
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		slotProvider);
}
 
Example #28
Source File: ExecutionGraphRestartTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * SlotPool#failAllocation should not fail with a {@link java.util.ConcurrentModificationException}
 * if there is a concurrent scheduling operation. See FLINK-13421.
 */
@Test
public void slotPoolExecutionGraph_ConcurrentSchedulingAndAllocationFailure_ShouldNotFailWithConcurrentModificationException() throws Exception {
	final SlotSharingGroup group = new SlotSharingGroup();
	final JobVertex vertex1 = createNoOpVertex("vertex1", 1);
	vertex1.setSlotSharingGroup(group);
	final JobVertex vertex2 = createNoOpVertex("vertex2", 3);
	vertex2.setSlotSharingGroup(group);
	final JobVertex vertex3 = createNoOpVertex("vertex3", 1);
	vertex3.setSlotSharingGroup(group);
	vertex3.connectNewDataSetAsInput(vertex2, DistributionPattern.ALL_TO_ALL, ResultPartitionType.PIPELINED);

	try (SlotPool slotPool = createSlotPoolImpl()) {
		final SlotProvider slots = createSchedulerWithSlots(2, slotPool, new LocalTaskManagerLocation());

		final AllocationID allocationId = slotPool.getAvailableSlotsInformation().iterator().next().getAllocationId();

		final ExecutionGraph eg = new ExecutionGraphTestUtils.TestingExecutionGraphBuilder(TEST_JOB_ID, vertex1, vertex2, vertex3)
			.setSlotProvider(slots)
			.setAllocationTimeout(Time.minutes(60))
			.setScheduleMode(ScheduleMode.EAGER)
			.setAllowQueuedScheduling(true)
			.build();

		eg.start(mainThreadExecutor);

		eg.scheduleForExecution();

		slotPool.failAllocation(
			allocationId,
			new Exception("test exception"));

		eg.waitUntilTerminal();
	}
}
 
Example #29
Source File: SchedulerTestingUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static DefaultScheduler createScheduler(
		final JobGraph jobGraph,
		final SlotProvider slotProvider,
		final Time slotRequestTimeout) throws Exception {

	return newSchedulerBuilderWithDefaultSlotAllocator(jobGraph, slotProvider, slotRequestTimeout)
		.build();
}
 
Example #30
Source File: ExecutionGraphDeploymentTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private Tuple2<ExecutionGraph, Map<ExecutionAttemptID, Execution>> setupExecution(JobVertex v1, int dop1, JobVertex v2, int dop2) throws Exception {
	v1.setParallelism(dop1);
	v2.setParallelism(dop2);

	v1.setInvokableClass(BatchTask.class);
	v2.setInvokableClass(BatchTask.class);

	final ArrayDeque<CompletableFuture<LogicalSlot>> slotFutures = new ArrayDeque<>();
	for (int i = 0; i < dop1 + dop2; i++) {
		slotFutures.addLast(CompletableFuture.completedFuture(new TestingLogicalSlotBuilder().createTestingLogicalSlot()));
	}

	final SlotProvider slotProvider = new TestingSlotProvider(ignore -> slotFutures.removeFirst());

	DirectScheduledExecutorService executorService = new DirectScheduledExecutorService();

	// execution graph that executes actions synchronously
	ExecutionGraph eg = createExecutionGraphWithoutQueuedScheduling(new JobID(), slotProvider, executorService, TestingUtils.defaultExecutor());
	checkJobOffloaded(eg);

	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	List<JobVertex> ordered = Arrays.asList(v1, v2);
	eg.attachJobGraph(ordered);

	// schedule, this triggers mock deployment
	eg.scheduleForExecution();

	Map<ExecutionAttemptID, Execution> executions = eg.getRegisteredExecutions();
	assertEquals(dop1 + dop2, executions.size());

	return new Tuple2<>(eg, executions);
}