Java Code Examples for org.apache.flink.runtime.jobgraph.JobGraph#setAllowQueuedScheduling()

@Override
public ClusterClient<ApplicationId> deployJobCluster(
	ClusterSpecification clusterSpecification,
	JobGraph jobGraph,
	boolean detached) throws ClusterDeploymentException {

	// this is required because the slots are allocated lazily
	jobGraph.setAllowQueuedScheduling(true);

	try {
		return deployInternal(
			clusterSpecification,
			"Flink per-job cluster",
			getYarnJobClusterEntrypoint(),
			jobGraph,
			detached);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
	}
}
 

public CompletableFuture<JobSubmissionResult> submitJob(JobGraph jobGraph) {
	final CompletableFuture<DispatcherGateway> dispatcherGatewayFuture = getDispatcherGatewayFuture();

	// we have to allow queued scheduling in Flip-6 mode because we need to request slots
	// from the ResourceManager
	jobGraph.setAllowQueuedScheduling(true);

	final CompletableFuture<InetSocketAddress> blobServerAddressFuture = createBlobServerAddress(dispatcherGatewayFuture);

	final CompletableFuture<Void> jarUploadFuture = uploadAndSetJobFiles(blobServerAddressFuture, jobGraph);

	final CompletableFuture<Acknowledge> acknowledgeCompletableFuture = jarUploadFuture
		.thenCombine(
			dispatcherGatewayFuture,
			(Void ack, DispatcherGateway dispatcherGateway) -> dispatcherGateway.submitJob(jobGraph, rpcTimeout))
		.thenCompose(Function.identity());

	return acknowledgeCompletableFuture.thenApply(
		(Acknowledge ignored) -> new JobSubmissionResult(jobGraph.getJobID()));
}
 

@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
	final PackagedProgram packagedProgram = createPackagedProgram();
	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	try {
		final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(
			packagedProgram,
			configuration,
			defaultParallelism,
			jobId);
		jobGraph.setAllowQueuedScheduling(true);
		jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

		return jobGraph;
	} catch (Exception e) {
		throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
	}
}
 

@Override
public ClusterClient<ApplicationId> deployJobCluster(
	ClusterSpecification clusterSpecification,
	JobGraph jobGraph,
	boolean detached) throws ClusterDeploymentException {

	// this is required because the slots are allocated lazily
	jobGraph.setAllowQueuedScheduling(true);

	try {
		return deployInternal(
			clusterSpecification,
			"Flink per-job cluster",
			getYarnJobClusterEntrypoint(),
			jobGraph,
			detached);
	} catch (Exception e) {
		throw new ClusterDeploymentException("Could not deploy Yarn job cluster.", e);
	}
}
 

@Override
public JobGraph retrieveJobGraph(Configuration configuration) throws FlinkException {
	final PackagedProgram packagedProgram = createPackagedProgram();
	final int defaultParallelism = configuration.getInteger(CoreOptions.DEFAULT_PARALLELISM);
	try {
		final JobGraph jobGraph = PackagedProgramUtils.createJobGraph(
			packagedProgram,
			configuration,
			defaultParallelism,
			jobId);
		jobGraph.setAllowQueuedScheduling(true);
		jobGraph.setSavepointRestoreSettings(savepointRestoreSettings);

		return jobGraph;
	} catch (Exception e) {
		throw new FlinkException("Could not create the JobGraph from the provided user code jar.", e);
	}
}
 

/**
 * Tests that an ongoing scheduling operation does not fail the {@link ExecutionGraph}
 * if it gets concurrently cancelled.
 */
@Test
public void testSchedulingOperationCancellationWhenCancel() throws Exception {
	final JobVertex jobVertex = new JobVertex("NoOp JobVertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(2);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);
	jobGraph.setAllowQueuedScheduling(true);

	final CompletableFuture<LogicalSlot> slotFuture1 = new CompletableFuture<>();
	final CompletableFuture<LogicalSlot> slotFuture2 = new CompletableFuture<>();
	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(2);
	slotProvider.addSlots(jobVertex.getID(), new CompletableFuture[]{slotFuture1, slotFuture2});
	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	final TestingLogicalSlot slot = createTestingSlot();
	final CompletableFuture<?> releaseFuture = slot.getReleaseFuture();
	slotFuture1.complete(slot);

	// cancel should change the state of all executions to CANCELLED
	executionGraph.cancel();

	// complete the now CANCELLED execution --> this should cause a failure
	slotFuture2.complete(new TestingLogicalSlotBuilder().createTestingLogicalSlot());

	Thread.sleep(1L);
	// release the first slot to finish the cancellation
	releaseFuture.complete(null);

	// NOTE: This test will only occasionally fail without the fix since there is
	// a race between the releaseFuture and the slotFuture2
	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.CANCELED));
}
 

@Nonnull
static JobGraph createNonEmptyJobGraph() {
	final JobVertex noOpVertex = new JobVertex("NoOp vertex");
	noOpVertex.setInvokableClass(NoOpInvokable.class);
	final JobGraph jobGraph = new JobGraph(noOpVertex);
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

private JobGraph createTestJobGraph(
		String jobName,
		int senderParallelism,
		int receiverParallelism) {

	// The sender and receiver invokable logic ensure that each subtask gets the expected data
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(RoundRobinSubtaskIndexSender.class);
	sender.getConfiguration().setInteger(RoundRobinSubtaskIndexSender.CONFIG_KEY, receiverParallelism);
	sender.setParallelism(senderParallelism);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SubtaskIndexReceiver.class);
	receiver.getConfiguration().setInteger(SubtaskIndexReceiver.CONFIG_KEY, senderParallelism);
	receiver.setParallelism(receiverParallelism);

	receiver.connectNewDataSetAsInput(
			sender,
			DistributionPattern.ALL_TO_ALL,
			ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(jobName, sender, receiver);

	// We need to allow queued scheduling, because there are not enough slots available
	// to run all tasks at once. We queue tasks and then let them finish/consume the blocking
	// result one after the other.
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

private JobGraph createTestJobGraph(
		String jobName,
		int senderParallelism,
		int receiverParallelism) {

	// The sender and receiver invokable logic ensure that each subtask gets the expected data
	final JobVertex sender = new JobVertex("Sender");
	sender.setInvokableClass(RoundRobinSubtaskIndexSender.class);
	sender.getConfiguration().setInteger(RoundRobinSubtaskIndexSender.CONFIG_KEY, receiverParallelism);
	sender.setParallelism(senderParallelism);

	final JobVertex receiver = new JobVertex("Receiver");
	receiver.setInvokableClass(SubtaskIndexReceiver.class);
	receiver.getConfiguration().setInteger(SubtaskIndexReceiver.CONFIG_KEY, senderParallelism);
	receiver.setParallelism(receiverParallelism);

	receiver.connectNewDataSetAsInput(
			sender,
			DistributionPattern.ALL_TO_ALL,
			ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(jobName, sender, receiver);

	// We need to allow queued scheduling, because there are not enough slots available
	// to run all tasks at once. We queue tasks and then let them finish/consume the blocking
	// result one after the other.
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

@Nonnull
private JobGraph createSingleVertexJobGraph() {
	final JobVertex jobVertex = new JobVertex("Test vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);

	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	return jobGraph;
}
 

private JobGraph producerConsumerJobGraph() {
	final JobVertex producer = new JobVertex("Producer");
	producer.setInvokableClass(NoOpInvokable.class);
	final JobVertex consumer = new JobVertex("Consumer");
	consumer.setInvokableClass(NoOpInvokable.class);

	consumer.connectNewDataSetAsInput(producer, DistributionPattern.POINTWISE, ResultPartitionType.BLOCKING);

	final JobGraph jobGraph = new JobGraph(producer, consumer);
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

/**
 * Tests that a partially completed eager scheduling operation fails if a
 * completed slot is released. See FLINK-9099.
 */
@Test
public void testSlotReleasingFailsSchedulingOperation() throws Exception {
	final int parallelism = 2;

	final JobVertex jobVertex = new JobVertex("Testing job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);
	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);

	final LogicalSlot slot = createSingleLogicalSlot(new DummySlotOwner(), new SimpleAckingTaskManagerGateway(), new SlotRequestId());
	slotProvider.addSlot(jobVertex.getID(), 0, CompletableFuture.completedFuture(slot));

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	slotProvider.addSlot(jobVertex.getID(), 1, slotFuture);

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());
	executionGraph.scheduleForExecution();

	assertThat(executionGraph.getState(), is(JobStatus.RUNNING));

	final ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertex.getID());
	final ExecutionVertex[] taskVertices = executionJobVertex.getTaskVertices();
	assertThat(taskVertices[0].getExecutionState(), is(ExecutionState.SCHEDULED));
	assertThat(taskVertices[1].getExecutionState(), is(ExecutionState.SCHEDULED));

	// fail the single allocated slot --> this should fail the scheduling operation
	slot.releaseSlot(new FlinkException("Test failure"));

	assertThat(executionGraph.getTerminationFuture().get(), is(JobStatus.FAILED));
}
 

/**
 * Tests that we can submit a job to the Dispatcher which then spawns a
 * new JobManagerRunner.
 */
@Test
public void testJobSubmissionWithPartialResourceConfigured() throws Exception {
	dispatcher = createAndStartDispatcher(heartbeatServices, haServices, new ExpectedJobIdJobManagerRunnerFactory(TEST_JOB_ID, createdJobManagerRunnerLatch));

	CompletableFuture<UUID> leaderFuture = dispatcherLeaderElectionService.isLeader(UUID.randomUUID());

	// wait for the leader to be elected
	leaderFuture.get();

	DispatcherGateway dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	ResourceSpec resourceSpec = ResourceSpec.newBuilder().setCpuCores(2).build();

	final JobVertex firstVertex = new JobVertex("firstVertex");
	firstVertex.setInvokableClass(NoOpInvokable.class);
	firstVertex.setResources(resourceSpec, resourceSpec);

	final JobVertex secondVertex = new JobVertex("secondVertex");
	secondVertex.setInvokableClass(NoOpInvokable.class);

	JobGraph jobGraphWithTwoVertices = new JobGraph(TEST_JOB_ID, "twoVerticesJob", firstVertex, secondVertex);
	jobGraphWithTwoVertices.setAllowQueuedScheduling(true);

	CompletableFuture<Acknowledge> acknowledgeFuture = dispatcherGateway.submitJob(jobGraphWithTwoVertices, TIMEOUT);

	try {
		acknowledgeFuture.get();
		fail("job submission should have failed");
	}
	catch (ExecutionException e) {
		assertTrue(ExceptionUtils.findThrowable(e, JobSubmissionException.class).isPresent());
	}
}
 

@Before
public void setUp() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobGraph = new JobGraph(TEST_JOB_ID, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	fatalErrorHandler = new TestingFatalErrorHandler();
	heartbeatServices = new HeartbeatServices(1000L, 10000L);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();

	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	jobMasterLeaderElectionService = new TestingLeaderElectionService();

	haServices = new TestingHighAvailabilityServices();
	haServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	haServices.setSubmittedJobGraphStore(submittedJobGraphStore);
	haServices.setJobMasterLeaderElectionService(TEST_JOB_ID, jobMasterLeaderElectionService);
	haServices.setCheckpointRecoveryFactory(new StandaloneCheckpointRecoveryFactory());
	haServices.setResourceManagerLeaderRetriever(new SettableLeaderRetrievalService());
	runningJobsRegistry = haServices.getRunningJobsRegistry();

	configuration = new Configuration();

	configuration.setString(
		BlobServerOptions.STORAGE_DIRECTORY,
		temporaryFolder.newFolder().getAbsolutePath());

	createdJobManagerRunnerLatch = new CountDownLatch(2);
	blobServer = new BlobServer(configuration, new VoidBlobStore());
}
 

@Nonnull
static JobGraph createNonEmptyJobGraph() {
	final JobVertex noOpVertex = new JobVertex("NoOp vertex");
	noOpVertex.setInvokableClass(NoOpInvokable.class);
	final JobGraph jobGraph = new JobGraph(noOpVertex);
	jobGraph.setAllowQueuedScheduling(true);

	return jobGraph;
}
 

/**
 * Tests that all slots are being returned to the {@link SlotOwner} if the
 * {@link ExecutionGraph} is being cancelled. See FLINK-9908
 */
@Test
public void testCancellationOfIncompleteScheduling() throws Exception {
	final int parallelism = 10;

	final JobVertex jobVertex = new JobVertex("Test job vertex");
	jobVertex.setInvokableClass(NoOpInvokable.class);
	jobVertex.setParallelism(parallelism);

	final JobGraph jobGraph = new JobGraph(jobVertex);
	jobGraph.setAllowQueuedScheduling(true);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);

	final TestingSlotOwner slotOwner = new TestingSlotOwner();
	final SimpleAckingTaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

	final ConcurrentMap<SlotRequestId, Integer> slotRequestIds = new ConcurrentHashMap<>(parallelism);

	final TestingSlotProvider slotProvider = new TestingSlotProvider(
		(SlotRequestId slotRequestId) -> {
			slotRequestIds.put(slotRequestId, 1);
			// return 50/50 fulfilled and unfulfilled requests
			return slotRequestIds.size() % 2 == 0 ?
				CompletableFuture.completedFuture(
					createSingleLogicalSlot(slotOwner, taskManagerGateway, slotRequestId)) :
				new CompletableFuture<>();
		});

	final ExecutionGraph executionGraph = createExecutionGraph(jobGraph, slotProvider);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());
	final Set<SlotRequestId> slotRequestIdsToReturn = ConcurrentHashMap.newKeySet(slotRequestIds.size());

	executionGraph.scheduleForExecution();

	slotRequestIdsToReturn.addAll(slotRequestIds.keySet());

	slotOwner.setReturnAllocatedSlotConsumer(logicalSlot -> {
		slotRequestIdsToReturn.remove(logicalSlot.getSlotRequestId());
	});

	slotProvider.setSlotCanceller(slotRequestIdsToReturn::remove);

	// make sure that we complete cancellations of deployed tasks
	taskManagerGateway.setCancelConsumer(
		(ExecutionAttemptID executionAttemptId) -> {
			final Execution execution = executionGraph.getRegisteredExecutions().get(executionAttemptId);

			// if the execution was cancelled in state SCHEDULING, then it might already have been removed
			if (execution != null) {
				execution.completeCancelling();
			}
		}
	);

	executionGraph.cancel();
	assertThat(slotRequestIdsToReturn, is(empty()));
}
 

/**
 * Executes the JobGraph of the on a mini cluster of CLusterUtil with a user
 * specified name.
 *
 * @param jobName
 *            name of the job
 * @return The result of the job execution, containing elapsed time and accumulators.
 */
@Override
public JobExecutionResult execute(String jobName) throws Exception {
	// transform the streaming program into a JobGraph
	StreamGraph streamGraph = getStreamGraph();
	streamGraph.setJobName(jobName);

	JobGraph jobGraph = streamGraph.getJobGraph();
	jobGraph.setAllowQueuedScheduling(true);

	Configuration configuration = new Configuration();
	configuration.addAll(jobGraph.getJobConfiguration());
	configuration.setString(TaskManagerOptions.MANAGED_MEMORY_SIZE, "0");

	// add (and override) the settings with what the user defined
	configuration.addAll(this.configuration);

	if (!configuration.contains(RestOptions.BIND_PORT)) {
		configuration.setString(RestOptions.BIND_PORT, "0");
	}

	int numSlotsPerTaskManager = configuration.getInteger(TaskManagerOptions.NUM_TASK_SLOTS, jobGraph.getMaximumParallelism());

	MiniClusterConfiguration cfg = new MiniClusterConfiguration.Builder()
		.setConfiguration(configuration)
		.setNumSlotsPerTaskManager(numSlotsPerTaskManager)
		.build();

	if (LOG.isInfoEnabled()) {
		LOG.info("Running job on local embedded Flink mini cluster");
	}

	MiniCluster miniCluster = new MiniCluster(cfg);

	try {
		miniCluster.start();
		configuration.setInteger(RestOptions.PORT, miniCluster.getRestAddress().get().getPort());

		return miniCluster.executeJobBlocking(jobGraph);
	}
	finally {
		transformations.clear();
		miniCluster.close();
	}
}
 

@Test
public void testRequestNextInputSplit() throws Exception {
	final List<TestingInputSplit> expectedInputSplits = Arrays.asList(
		new TestingInputSplit(1),
		new TestingInputSplit(42),
		new TestingInputSplit(1337));

	// build one node JobGraph
	InputSplitSource<TestingInputSplit> inputSplitSource = new TestingInputSplitSource(expectedInputSplits);

	JobVertex source = new JobVertex("vertex1");
	source.setParallelism(1);
	source.setInputSplitSource(inputSplitSource);
	source.setInvokableClass(AbstractInvokable.class);

	final JobGraph testJobGraph = new JobGraph(source);
	testJobGraph.setAllowQueuedScheduling(true);

	configuration.setLong(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_ATTEMPTS, 1);
	configuration.setString(ConfigConstants.RESTART_STRATEGY_FIXED_DELAY_DELAY, "0 s");

	final JobManagerSharedServices jobManagerSharedServices =
		new TestingJobManagerSharedServicesBuilder()
			.setRestartStrategyFactory(RestartStrategyFactory.createRestartStrategyFactory(configuration))
			.build();

	final JobMaster jobMaster = createJobMaster(
		configuration,
		testJobGraph,
		haServices,
		jobManagerSharedServices);

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

		ExecutionGraph eg = jobMaster.getExecutionGraph();

		ExecutionVertex ev = eg.getAllExecutionVertices().iterator().next();

		final SupplierWithException<SerializedInputSplit, Exception> inputSplitSupplier = () -> jobMasterGateway.requestNextInputSplit(
			source.getID(),
			ev.getCurrentExecutionAttempt().getAttemptId()).get();

		List<InputSplit> actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		final Matcher<Iterable<? extends InputSplit>> expectedInputSplitsMatcher = containsInAnyOrder(expectedInputSplits.toArray(EMPTY_TESTING_INPUT_SPLITS));
		assertThat(actualInputSplits, expectedInputSplitsMatcher);

		final long maxWaitMillis = 2000L;
		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		CompletableFuture.runAsync(() -> eg.failGlobal(new Exception("Testing exception")), eg.getJobMasterMainThreadExecutor()).get();

		ExecutionGraphTestUtils.waitUntilExecutionVertexState(ev, ExecutionState.SCHEDULED, maxWaitMillis);

		actualInputSplits = getInputSplits(
			expectedInputSplits.size(),
			inputSplitSupplier);

		assertThat(actualInputSplits, expectedInputSplitsMatcher);
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
	}
}
 

/**
 * This test verifies that if one slot future fails, the deployment will be aborted.
 */
@Test
public void testOneSlotFailureAbortsDeploy() throws Exception {

	//                                            [pipelined]
	//  we construct a simple graph    (source) ----------------> (target)

	final int parallelism = 6;

	final JobVertex sourceVertex = new JobVertex("source");
	sourceVertex.setParallelism(parallelism);
	sourceVertex.setInvokableClass(NoOpInvokable.class);

	final JobVertex targetVertex = new JobVertex("target");
	targetVertex.setParallelism(parallelism);
	targetVertex.setInvokableClass(NoOpInvokable.class);

	targetVertex.connectNewDataSetAsInput(sourceVertex, DistributionPattern.POINTWISE, ResultPartitionType.PIPELINED);

	final JobID jobId = new JobID();
	final JobGraph jobGraph = new JobGraph(jobId, "test", sourceVertex, targetVertex);
	jobGraph.setScheduleMode(ScheduleMode.EAGER);
	jobGraph.setAllowQueuedScheduling(true);

	//
	//  Create the slots, futures, and the slot provider

	final InteractionsCountingTaskManagerGateway taskManager = createTaskManager();
	final BlockingQueue<AllocationID> returnedSlots = new ArrayBlockingQueue<>(parallelism);
	final TestingSlotOwner slotOwner = new TestingSlotOwner();
	slotOwner.setReturnAllocatedSlotConsumer(
		(LogicalSlot logicalSlot) -> returnedSlots.offer(logicalSlot.getAllocationId()));

	final LogicalSlot[] sourceSlots = new LogicalSlot[parallelism];
	final LogicalSlot[] targetSlots = new LogicalSlot[parallelism];

	@SuppressWarnings({"unchecked", "rawtypes"})
	final CompletableFuture<LogicalSlot>[] sourceFutures = new CompletableFuture[parallelism];
	@SuppressWarnings({"unchecked", "rawtypes"})
	final CompletableFuture<LogicalSlot>[] targetFutures = new CompletableFuture[parallelism];

	for (int i = 0; i < parallelism; i++) {
		sourceSlots[i] = createSingleLogicalSlot(slotOwner, taskManager, new SlotRequestId());
		targetSlots[i] = createSingleLogicalSlot(slotOwner, taskManager, new SlotRequestId());

		sourceFutures[i] = new CompletableFuture<>();
		targetFutures[i] = new CompletableFuture<>();
	}

	ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(parallelism);
	slotProvider.addSlots(sourceVertex.getID(), sourceFutures);
	slotProvider.addSlots(targetVertex.getID(), targetFutures);

	final ExecutionGraph eg = createExecutionGraph(jobGraph, slotProvider);
	eg.start(ComponentMainThreadExecutorServiceAdapter.forMainThread());

	//
	//  we complete some of the futures

	for (int i = 0; i < parallelism; i += 2) {
		sourceFutures[i].complete(sourceSlots[i]);
		targetFutures[i].complete(targetSlots[i]);
	}

	//  kick off the scheduling
	eg.scheduleForExecution();

	// fail one slot
	sourceFutures[1].completeExceptionally(new TestRuntimeException());

	// wait until the job failed as a whole
	eg.getTerminationFuture().get(2000, TimeUnit.MILLISECONDS);

	// wait until all slots are back
	for (int i = 0; i < parallelism; i++) {
		returnedSlots.poll(2000L, TimeUnit.MILLISECONDS);
	}

	// no deployment calls must have happened
	assertThat(taskManager.getSubmitTaskCount(), is(0));

	// all completed futures must have been returns
	for (int i = 0; i < parallelism; i += 2) {
		assertFalse(sourceSlots[i].isAlive());
		assertFalse(targetSlots[i].isAlive());
	}
}
 

@Before
public void setup() throws Exception {
	final JobVertex testVertex = new JobVertex("testVertex");
	testVertex.setInvokableClass(NoOpInvokable.class);
	jobId = new JobID();
	jobGraph = new JobGraph(jobId, "testJob", testVertex);
	jobGraph.setAllowQueuedScheduling(true);

	configuration = new Configuration();
	configuration.setString(BlobServerOptions.STORAGE_DIRECTORY, temporaryFolder.newFolder().getAbsolutePath());

	highAvailabilityServices = new TestingHighAvailabilityServices();
	dispatcherLeaderElectionService = new TestingLeaderElectionService();
	highAvailabilityServices.setDispatcherLeaderElectionService(dispatcherLeaderElectionService);
	clearedJobLatch = new OneShotLatch();
	runningJobsRegistry = new SingleRunningJobsRegistry(jobId, clearedJobLatch);
	highAvailabilityServices.setRunningJobsRegistry(runningJobsRegistry);
	submittedJobGraphStore = new FaultySubmittedJobGraphStore();
	highAvailabilityServices.setSubmittedJobGraphStore(submittedJobGraphStore);

	storedHABlobFuture = new CompletableFuture<>();
	deleteAllHABlobsFuture = new CompletableFuture<>();

	final TestingBlobStore testingBlobStore = new TestingBlobStoreBuilder()
		.setPutFunction(
			putArguments -> storedHABlobFuture.complete(putArguments.f2))
		.setDeleteAllFunction(deleteAllHABlobsFuture::complete)
		.createTestingBlobStore();

	cleanupJobFuture = new CompletableFuture<>();
	terminationFuture = new CompletableFuture<>();

	blobServer = new TestingBlobServer(configuration, testingBlobStore, cleanupJobFuture);

	// upload a blob to the blob server
	permanentBlobKey = blobServer.putPermanent(jobId, new byte[256]);
	jobGraph.addUserJarBlobKey(permanentBlobKey);
	blobFile = blobServer.getStorageLocation(jobId, permanentBlobKey);

	resultFuture = new CompletableFuture<>();

	fatalErrorHandler = new TestingFatalErrorHandler();

	failJobMasterCreationWith = new AtomicReference<>();

	TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	dispatcher = new TestingDispatcher(
		rpcService,
		Dispatcher.DISPATCHER_NAME + UUID.randomUUID(),
		configuration,
		highAvailabilityServices,
		() -> CompletableFuture.completedFuture(resourceManagerGateway),
		blobServer,
		new HeartbeatServices(1000L, 1000L),
		UnregisteredMetricGroups.createUnregisteredJobManagerMetricGroup(),
		null,
		new MemoryArchivedExecutionGraphStore(),
		new TestingJobManagerRunnerFactory(new CompletableFuture<>(), resultFuture, terminationFuture, failJobMasterCreationWith),
		fatalErrorHandler);

	dispatcher.start();

	dispatcherGateway = dispatcher.getSelfGateway(DispatcherGateway.class);

	dispatcherLeaderElectionService.isLeader(UUID.randomUUID()).get();

	assertThat(blobFile.exists(), is(true));

	// verify that we stored the blob also in the BlobStore
	assertThat(storedHABlobFuture.get(), equalTo(permanentBlobKey));
}