Java Code Examples for org.apache.flink.runtime.jobmaster.slotpool.SlotProvider

The following examples show how to use org.apache.flink.runtime.jobmaster.slotpool.SlotProvider. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: ExecutionGraph.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This constructor is for tests only, because it does not include class loading information.
 */
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		new RestartAllStrategy.Factory(),
		slotProvider);
}
 
Example 2
Source Project: flink   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static ExecutionGraph createExecutionGraph(
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return TestingExecutionGraphBuilder
		.newBuilder()
		.setJobGraph(new JobGraph(vertices))
		.setFutureExecutor(executor)
		.setIoExecutor(executor)
		.setSlotProvider(slotProvider)
		.setAllocationTimeout(timeout)
		.setRpcTimeout(timeout)
		.setRestartStrategy(restartStrategy)
		.build();
}
 
Example 3
Source Project: Flink-CEPplus   Source File: ExecutionGraph.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Void> scheduleLazy(SlotProvider slotProvider) {

		final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>(numVerticesTotal);
		// simply take the vertices without inputs.
		for (ExecutionJobVertex ejv : verticesInCreationOrder) {
			if (ejv.getJobVertex().isInputVertex()) {
				final CompletableFuture<Void> schedulingJobVertexFuture = ejv.scheduleAll(
					slotProvider,
					allowQueuedScheduling,
					LocationPreferenceConstraint.ALL, // since it is an input vertex, the input based location preferences should be empty
					Collections.emptySet());

				schedulingFutures.add(schedulingJobVertexFuture);
			}
		}

		return FutureUtils.waitForAll(schedulingFutures);
	}
 
Example 4
Source Project: Flink-CEPplus   Source File: ExecutionJobVertex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Schedules all execution vertices of this ExecutionJobVertex.
 *
 * @param slotProvider to allocate the slots from
 * @param queued if the allocations can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds set with all previous allocation ids in the job graph.
 *                                                 Can be empty if the allocation ids are not required for scheduling.
 * @return Future which is completed once all {@link Execution} could be deployed
 */
public CompletableFuture<Void> scheduleAll(
		SlotProvider slotProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds) {

	final ExecutionVertex[] vertices = this.taskVertices;

	final ArrayList<CompletableFuture<Void>> scheduleFutures = new ArrayList<>(vertices.length);

	// kick off the tasks
	for (ExecutionVertex ev : vertices) {
		scheduleFutures.add(ev.scheduleForExecution(
			slotProvider,
			queued,
			locationPreferenceConstraint,
			allPreviousExecutionGraphAllocationIds));
	}

	return FutureUtils.waitForAll(scheduleFutures);
}
 
Example 5
private static ExecutionGraph createExecutionGraph(
		List<JobVertex> orderedVertices,
		InputDependencyConstraint inputDependencyConstraint) throws Exception {

	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";
	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20);

	for (JobVertex vertex : orderedVertices) {
		vertex.setInputDependencyConstraint(inputDependencyConstraint);
	}

	ExecutionGraph eg = new ExecutionGraph(
		new DummyJobInformation(
			jobId,
			jobName),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		TestRestartStrategy.directExecuting(),
		new RestartAllStrategy.Factory(),
		slotProvider);
	eg.attachJobGraph(orderedVertices);

	return eg;
}
 
Example 6
Source Project: Flink-CEPplus   Source File: ExecutionGraphSuspendTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 
Example 7
Source Project: Flink-CEPplus   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph newExecutionGraph(RestartStrategy restartStrategy, SlotProvider slotProvider) throws IOException {
	final ExecutionGraph executionGraph = new ExecutionGraph(
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		new JobID(),
		"Test job",
		new Configuration(),
		new SerializedValue<>(new ExecutionConfig()),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		slotProvider);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	return executionGraph;
}
 
Example 8
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 
Example 9
Source Project: Flink-CEPplus   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an execution graph containing the given vertices and the given restart strategy.
 */
public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		TaskManagerGateway taskManagerGateway,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	int numSlotsNeeded = 0;
	for (JobVertex vertex : vertices) {
		numSlotsNeeded += vertex.getParallelism();
	}

	SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded, taskManagerGateway);

	return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
}
 
Example 10
Source Project: flink   Source File: SchedulerNGFactory.java    License: Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
PartitionTracker partitionTracker) throws Exception;
 
Example 11
Source Project: flink   Source File: ExecutionGraph.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
ExecutionGraph(
		JobInformation jobInformation,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		Time timeout,
		RestartStrategy restartStrategy,
		FailoverStrategy.Factory failoverStrategy,
		SlotProvider slotProvider) throws IOException {
	this(
		jobInformation,
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		failoverStrategy,
		slotProvider,
		ExecutionGraph.class.getClassLoader(),
		VoidBlobWriter.getInstance(),
		timeout);
}
 
Example 12
Source Project: flink   Source File: SlotProviderStrategy.java    License: Apache License 2.0 6 votes vote down vote up
static SlotProviderStrategy from(
	ScheduleMode scheduleMode,
	SlotProvider slotProvider,
	Time allocationTimeout,
	boolean allowQueuedScheduling) {

	switch (scheduleMode) {
		case LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST:
			return new BatchSlotProviderStrategy(slotProvider, allowQueuedScheduling);
		case LAZY_FROM_SOURCES:
		case EAGER:
			return new NormalSlotProviderStrategy(slotProvider, allocationTimeout, allowQueuedScheduling);
		default:
			throw new IllegalArgumentException(String.format("Unknown scheduling mode: %s", scheduleMode));
	}
}
 
Example 13
Source Project: flink   Source File: ExecutionVertexInputConstraintTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(
		List<JobVertex> orderedVertices,
		InputDependencyConstraint inputDependencyConstraint) throws Exception {

	final JobID jobId = new JobID();
	final String jobName = "Test Job Sample Name";
	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, 20);

	for (JobVertex vertex : orderedVertices) {
		vertex.setInputDependencyConstraint(inputDependencyConstraint);
	}

	ExecutionGraph eg = new ExecutionGraph(
		new DummyJobInformation(
			jobId,
			jobName),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		TestRestartStrategy.directExecuting(),
		new RestartAllStrategy.Factory(),
		slotProvider);
	eg.attachJobGraph(orderedVertices);

	return eg;
}
 
Example 14
Source Project: flink   Source File: ExecutionGraphSuspendTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createExecutionGraph(TaskManagerGateway gateway, int parallelism) throws Exception {
	final JobID jobId = new JobID();

	final JobVertex vertex = new JobVertex("vertex");
	vertex.setInvokableClass(NoOpInvokable.class);
	vertex.setParallelism(parallelism);

	final SlotProvider slotProvider = new SimpleSlotProvider(jobId, parallelism, gateway);

	ExecutionGraph simpleTestGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		jobId,
		slotProvider,
		new FixedDelayRestartStrategy(0, 0),
		vertex);
	simpleTestGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	return simpleTestGraph;
}
 
Example 15
Source Project: flink   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionGraph createSimpleExecutionGraph(
	final RestartStrategy restartStrategy,
	final FailoverStrategy.Factory failoverStrategyFactory,
	final SlotProvider slotProvider,
	final JobGraph jobGraph) throws IOException, JobException {

	final ExecutionGraph executionGraph = new ExecutionGraph(
		new JobInformation(
			TEST_JOB_ID,
			"Test job",
			new SerializedValue<>(new ExecutionConfig()),
			new Configuration(),
			Collections.emptyList(),
			Collections.emptyList()),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		AkkaUtils.getDefaultTimeout(),
		restartStrategy,
		failoverStrategyFactory,
		slotProvider);

	executionGraph.start(mainThreadExecutor);
	executionGraph.attachJobGraph(jobGraph.getVerticesSortedTopologicallyFromSources());

	return executionGraph;
}
 
Example 16
Source Project: flink   Source File: ExecutionGraphSchedulingTest.java    License: Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph, SlotProvider slotProvider, Time timeout) throws Exception {
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		new Configuration(),
		executor,
		executor,
		slotProvider,
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 
Example 17
Source Project: flink   Source File: PipelinedFailoverRegionBuildingTest.java    License: Apache License 2.0 6 votes vote down vote up
private ExecutionGraph createExecutionGraph(JobGraph jobGraph) throws JobException, JobExecutionException {
	// configure the pipelined failover strategy
	final Configuration jobManagerConfig = new Configuration();
	jobManagerConfig.setString(
			JobManagerOptions.EXECUTION_FAILOVER_STRATEGY,
			FailoverStrategyLoader.LEGACY_PIPELINED_REGION_RESTART_STRATEGY_NAME);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		jobGraph,
		jobManagerConfig,
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		PipelinedFailoverRegionBuildingTest.class.getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new NoRestartStrategy(),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpPartitionTracker.INSTANCE);
}
 
Example 18
Source Project: flink   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates an execution graph containing the given vertices and the given restart strategy.
 */
public static ExecutionGraph createSimpleTestGraph(
		JobID jid,
		TaskManagerGateway taskManagerGateway,
		RestartStrategy restartStrategy,
		JobVertex... vertices) throws Exception {

	int numSlotsNeeded = 0;
	for (JobVertex vertex : vertices) {
		numSlotsNeeded += vertex.getParallelism();
	}

	SlotProvider slotProvider = new SimpleSlotProvider(jid, numSlotsNeeded, taskManagerGateway);

	return createSimpleTestGraph(jid, slotProvider, restartStrategy, vertices);
}
 
Example 19
Source Project: flink   Source File: ExecutionGraphTestUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static ExecutionGraph createExecutionGraph(
		JobID jid,
		SlotProvider slotProvider,
		RestartStrategy restartStrategy,
		ScheduledExecutorService executor,
		Time timeout,
		JobVertex... vertices) throws Exception {

	checkNotNull(jid);
	checkNotNull(restartStrategy);
	checkNotNull(vertices);
	checkNotNull(timeout);

	return new TestingExecutionGraphBuilder(vertices)
		.setFutureExecutor(executor)
		.setIoExecutor(executor)
		.setSlotProvider(slotProvider)
		.setAllocationTimeout(timeout)
		.setRpcTimeout(timeout)
		.setRestartStrategy(restartStrategy)
		.build();
}
 
Example 20
Source Project: flink   Source File: SchedulerNGFactory.java    License: Apache License 2.0 6 votes vote down vote up
SchedulerNG createInstance(
Logger log,
JobGraph jobGraph,
BackPressureStatsTracker backPressureStatsTracker,
Executor ioExecutor,
Configuration jobMasterConfiguration,
SlotProvider slotProvider,
ScheduledExecutorService futureExecutor,
ClassLoader userCodeLoader,
CheckpointRecoveryFactory checkpointRecoveryFactory,
Time rpcTimeout,
BlobWriter blobWriter,
JobManagerJobMetricGroup jobManagerJobMetricGroup,
Time slotRequestTimeout,
ShuffleMaster<?> shuffleMaster,
JobMasterPartitionTracker partitionTracker) throws Exception;
 
Example 21
Source Project: flink   Source File: DefaultSchedulerFactory.java    License: Apache License 2.0 6 votes vote down vote up
private static ExecutionSlotAllocatorFactory createExecutionSlotAllocatorFactory(
		final ScheduleMode scheduleMode,
		final SlotProvider slotProvider,
		final Time slotRequestTimeout,
		final SchedulingStrategyFactory schedulingStrategyFactory) {

	if (schedulingStrategyFactory instanceof PipelinedRegionSchedulingStrategy.Factory) {
		return new OneSlotPerExecutionSlotAllocatorFactory(
			slotProvider,
			scheduleMode != ScheduleMode.LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST,
			slotRequestTimeout);
	} else {
		final SlotProviderStrategy slotProviderStrategy = SlotProviderStrategy.from(
			scheduleMode,
			slotProvider,
			slotRequestTimeout);

		return new DefaultExecutionSlotAllocatorFactory(slotProviderStrategy);
	}
}
 
Example 22
Source Project: Flink-CEPplus   Source File: ExecutionGraph.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This constructor is for tests only, because it sets default values for many fields.
 */
@VisibleForTesting
ExecutionGraph(
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		JobID jobId,
		String jobName,
		Configuration jobConfig,
		SerializedValue<ExecutionConfig> serializedConfig,
		Time timeout,
		RestartStrategy restartStrategy,
		SlotProvider slotProvider) throws IOException {

	this(
		new JobInformation(
			jobId,
			jobName,
			serializedConfig,
			jobConfig,
			Collections.emptyList(),
			Collections.emptyList()),
		futureExecutor,
		ioExecutor,
		timeout,
		restartStrategy,
		slotProvider);
}
 
Example 23
Source Project: Flink-CEPplus   Source File: ExecutionVertex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Schedules the current execution of this ExecutionVertex.
 *
 * @param slotProvider to allocate the slots from
 * @param queued if the allocation can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds set with all previous allocation ids in the job graph.
 *                                                 Can be empty if the allocation ids are not required for scheduling.
 * @return Future which is completed once the execution is deployed. The future
 * can also completed exceptionally.
 */
public CompletableFuture<Void> scheduleForExecution(
		SlotProvider slotProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds) {
	return this.currentExecution.scheduleForExecution(
		slotProvider,
		queued,
		locationPreferenceConstraint,
		allPreviousExecutionGraphAllocationIds);
}
 
Example 24
Source Project: Flink-CEPplus   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Void> scheduleForExecution() {
	final ExecutionGraph executionGraph = getVertex().getExecutionGraph();
	final SlotProvider resourceProvider = executionGraph.getSlotProvider();
	final boolean allowQueued = executionGraph.isQueuedSchedulingAllowed();
	return scheduleForExecution(
		resourceProvider,
		allowQueued,
		LocationPreferenceConstraint.ANY,
		Collections.emptySet());
}
 
Example 25
Source Project: Flink-CEPplus   Source File: ExecutionJobVertex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Acquires a slot for all the execution vertices of this ExecutionJobVertex. The method returns
 * pairs of the slots and execution attempts, to ease correlation between vertices and execution
 * attempts.
 *
 * <p>If this method throws an exception, it makes sure to release all so far requested slots.
 *
 * @param resourceProvider The resource provider from whom the slots are requested.
 * @param queued if the allocation can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds the allocation ids of all previous executions in the execution job graph.
 * @param allocationTimeout timeout for allocating the individual slots
 */
public Collection<CompletableFuture<Execution>> allocateResourcesForAll(
		SlotProvider resourceProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds,
		Time allocationTimeout) {
	final ExecutionVertex[] vertices = this.taskVertices;

	@SuppressWarnings("unchecked")
	final CompletableFuture<Execution>[] slots = new CompletableFuture[vertices.length];

	// try to acquire a slot future for each execution.
	// we store the execution with the future just to be on the safe side
	for (int i = 0; i < vertices.length; i++) {
		// allocate the next slot (future)
		final Execution exec = vertices[i].getCurrentExecutionAttempt();
		final CompletableFuture<Execution> allocationFuture = exec.allocateAndAssignSlotForExecution(
			resourceProvider,
			queued,
			locationPreferenceConstraint,
			allPreviousExecutionGraphAllocationIds,
			allocationTimeout);
		slots[i] = allocationFuture;
	}

	// all good, we acquired all slots
	return Arrays.asList(slots);
}
 
Example 26
Source Project: Flink-CEPplus   Source File: ExecutionGraphBuilder.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Builds the ExecutionGraph from the JobGraph.
 * If a prior execution graph exists, the JobGraph will be attached. If no prior execution
 * graph exists, then the JobGraph will become attach to a new empty execution graph.
 */
public static ExecutionGraph buildGraph(
		@Nullable ExecutionGraph prior,
		JobGraph jobGraph,
		Configuration jobManagerConfig,
		ScheduledExecutorService futureExecutor,
		Executor ioExecutor,
		SlotProvider slotProvider,
		ClassLoader classLoader,
		CheckpointRecoveryFactory recoveryFactory,
		Time rpcTimeout,
		RestartStrategy restartStrategy,
		MetricGroup metrics,
		BlobWriter blobWriter,
		Time allocationTimeout,
		Logger log)
	throws JobExecutionException, JobException {

	return buildGraph(
		prior,
		jobGraph,
		jobManagerConfig,
		futureExecutor,
		ioExecutor,
		slotProvider,
		classLoader,
		recoveryFactory,
		rpcTimeout,
		restartStrategy,
		metrics,
		-1,
		blobWriter,
		allocationTimeout,
		log);
}
 
Example 27
Source Project: flink   Source File: SlotProviderStrategy.java    License: Apache License 2.0 5 votes vote down vote up
public static SlotProviderStrategy from(
	ScheduleMode scheduleMode,
	SlotProvider slotProvider,
	Time allocationTimeout) {

	switch (scheduleMode) {
		case LAZY_FROM_SOURCES_WITH_BATCH_SLOT_REQUEST:
			return new BatchSlotProviderStrategy(slotProvider);
		case LAZY_FROM_SOURCES:
		case EAGER:
			return new NormalSlotProviderStrategy(slotProvider, allocationTimeout);
		default:
			throw new IllegalArgumentException(String.format("Unknown scheduling mode: %s", scheduleMode));
	}
}
 
Example 28
Source Project: flink   Source File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		VoidBlobWriter.getInstance(),
		timeout,
		log,
		NettyShuffleMaster.INSTANCE,
		NoOpJobMasterPartitionTracker.INSTANCE);
}
 
Example 29
Source Project: Flink-CEPplus   Source File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a simple 2 vertex graph with a parallel source and a parallel target.
 */
private ExecutionGraph createTestGraph(int parallelism, boolean allToAll) throws Exception {

	JobVertex source = new JobVertex("source", sourceVertexId);
	source.setParallelism(parallelism);
	source.setInvokableClass(NoOpInvokable.class);

	JobVertex target = new JobVertex("source", targetVertexId);
	target.setParallelism(parallelism);
	target.setInvokableClass(NoOpInvokable.class);

	DistributionPattern connectionPattern = allToAll ? DistributionPattern.ALL_TO_ALL : DistributionPattern.POINTWISE;
	target.connectNewDataSetAsInput(source, connectionPattern, ResultPartitionType.PIPELINED);

	JobGraph testJob = new JobGraph(jobId, "test job", source, target);

	final Time timeout = Time.seconds(10L);
	return ExecutionGraphBuilder.buildGraph(
		null,
		testJob,
		new Configuration(),
		TestingUtils.defaultExecutor(),
		TestingUtils.defaultExecutor(),
		mock(SlotProvider.class),
		getClass().getClassLoader(),
		new StandaloneCheckpointRecoveryFactory(),
		timeout,
		new FixedDelayRestartStrategy(10, 0L),
		new UnregisteredMetricsGroup(),
		1,
		VoidBlobWriter.getInstance(),
		timeout,
		log);
}
 
Example 30
Source Project: flink   Source File: DefaultSchedulerBatchSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
private SlotProvider createSlotProvider(SlotPool slotPool, ComponentMainThreadExecutor mainThreadExecutor) {
	final SchedulerImpl scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	return scheduler;
}