Java Code Examples for org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint

The following examples show how to use org.apache.flink.runtime.jobmanager.scheduler.LocationPreferenceConstraint. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: ExecutionGraph.java    License: Apache License 2.0 6 votes vote down vote up
private CompletableFuture<Void> scheduleLazy(SlotProvider slotProvider) {

		final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>(numVerticesTotal);
		// simply take the vertices without inputs.
		for (ExecutionJobVertex ejv : verticesInCreationOrder) {
			if (ejv.getJobVertex().isInputVertex()) {
				final CompletableFuture<Void> schedulingJobVertexFuture = ejv.scheduleAll(
					slotProvider,
					allowQueuedScheduling,
					LocationPreferenceConstraint.ALL, // since it is an input vertex, the input based location preferences should be empty
					Collections.emptySet());

				schedulingFutures.add(schedulingJobVertexFuture);
			}
		}

		return FutureUtils.waitForAll(schedulingFutures);
	}
 
Example 2
Source Project: Flink-CEPplus   Source File: ExecutionJobVertex.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Schedules all execution vertices of this ExecutionJobVertex.
 *
 * @param slotProvider to allocate the slots from
 * @param queued if the allocations can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds set with all previous allocation ids in the job graph.
 *                                                 Can be empty if the allocation ids are not required for scheduling.
 * @return Future which is completed once all {@link Execution} could be deployed
 */
public CompletableFuture<Void> scheduleAll(
		SlotProvider slotProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds) {

	final ExecutionVertex[] vertices = this.taskVertices;

	final ArrayList<CompletableFuture<Void>> scheduleFutures = new ArrayList<>(vertices.length);

	// kick off the tasks
	for (ExecutionVertex ev : vertices) {
		scheduleFutures.add(ev.scheduleForExecution(
			slotProvider,
			queued,
			locationPreferenceConstraint,
			allPreviousExecutionGraphAllocationIds));
	}

	return FutureUtils.waitForAll(scheduleFutures);
}
 
Example 3
@Test
public void testScheduleToDeploying() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		final LogicalSlot slot = new TestingLogicalSlot();

		CompletableFuture<LogicalSlot> future = CompletableFuture.completedFuture(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(
			new TestingSlotProvider(ignore -> future),
			false,
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 4
Source Project: Flink-CEPplus   Source File: ExecutionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that any preferred locations are calculated.
 */
@Test
public void testAnyPreferredLocationCalculation() throws ExecutionException, InterruptedException {
	final TaskManagerLocation taskManagerLocation1 = new LocalTaskManagerLocation();
	final TaskManagerLocation taskManagerLocation3 = new LocalTaskManagerLocation();

	final CompletableFuture<TaskManagerLocation> locationFuture1 = CompletableFuture.completedFuture(taskManagerLocation1);
	final CompletableFuture<TaskManagerLocation> locationFuture2 = new CompletableFuture<>();
	final CompletableFuture<TaskManagerLocation> locationFuture3 = CompletableFuture.completedFuture(taskManagerLocation3);

	final Execution execution = SchedulerTestUtils.getTestVertex(Arrays.asList(locationFuture1, locationFuture2, locationFuture3));

	CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture = execution.calculatePreferredLocations(LocationPreferenceConstraint.ANY);

	assertTrue(preferredLocationsFuture.isDone());

	final Collection<TaskManagerLocation> preferredLocations = preferredLocationsFuture.get();

	assertThat(preferredLocations, containsInAnyOrder(taskManagerLocation1, taskManagerLocation3));
}
 
Example 5
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testScheduleToDeploying() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();

		CompletableFuture<LogicalSlot> future = CompletableFuture.completedFuture(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider(ignore -> future), false),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 6
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that any preferred locations are calculated.
 */
@Test
public void testAnyPreferredLocationCalculation() throws ExecutionException, InterruptedException {
	final TaskManagerLocation taskManagerLocation1 = new LocalTaskManagerLocation();
	final TaskManagerLocation taskManagerLocation3 = new LocalTaskManagerLocation();

	final CompletableFuture<TaskManagerLocation> locationFuture1 = CompletableFuture.completedFuture(taskManagerLocation1);
	final CompletableFuture<TaskManagerLocation> locationFuture2 = new CompletableFuture<>();
	final CompletableFuture<TaskManagerLocation> locationFuture3 = CompletableFuture.completedFuture(taskManagerLocation3);

	final Execution execution = SchedulerTestUtils.getTestVertex(Arrays.asList(locationFuture1, locationFuture2, locationFuture3));

	CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture = execution.calculatePreferredLocations(LocationPreferenceConstraint.ANY);

	assertTrue(preferredLocationsFuture.isDone());

	final Collection<TaskManagerLocation> preferredLocations = preferredLocationsFuture.get();

	assertThat(preferredLocations, containsInAnyOrder(taskManagerLocation1, taskManagerLocation3));
}
 
Example 7
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testScheduleToDeploying() {
	try {
		final ExecutionJobVertex ejv = getExecutionJobVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();

		CompletableFuture<LogicalSlot> future = CompletableFuture.completedFuture(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());

		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider(ignore -> future)),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());
		assertEquals(ExecutionState.DEPLOYING, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 8
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that any preferred locations are calculated.
 */
@Test
public void testAnyPreferredLocationCalculation() throws Exception {
	final TaskManagerLocation taskManagerLocation1 = new LocalTaskManagerLocation();
	final TaskManagerLocation taskManagerLocation3 = new LocalTaskManagerLocation();

	final CompletableFuture<TaskManagerLocation> locationFuture1 = CompletableFuture.completedFuture(taskManagerLocation1);
	final CompletableFuture<TaskManagerLocation> locationFuture2 = new CompletableFuture<>();
	final CompletableFuture<TaskManagerLocation> locationFuture3 = CompletableFuture.completedFuture(taskManagerLocation3);

	final Execution execution = getExecution(Arrays.asList(locationFuture1, locationFuture2, locationFuture3));

	CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture = execution.calculatePreferredLocations(LocationPreferenceConstraint.ANY);

	assertTrue(preferredLocationsFuture.isDone());

	final Collection<TaskManagerLocation> preferredLocations = preferredLocationsFuture.get();

	assertThat(preferredLocations, containsInAnyOrder(taskManagerLocation1, taskManagerLocation3));
}
 
Example 9
Source Project: Flink-CEPplus   Source File: ExecutionVertex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Schedules the current execution of this ExecutionVertex.
 *
 * @param slotProvider to allocate the slots from
 * @param queued if the allocation can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds set with all previous allocation ids in the job graph.
 *                                                 Can be empty if the allocation ids are not required for scheduling.
 * @return Future which is completed once the execution is deployed. The future
 * can also completed exceptionally.
 */
public CompletableFuture<Void> scheduleForExecution(
		SlotProvider slotProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds) {
	return this.currentExecution.scheduleForExecution(
		slotProvider,
		queued,
		locationPreferenceConstraint,
		allPreviousExecutionGraphAllocationIds);
}
 
Example 10
Source Project: Flink-CEPplus   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Void> scheduleForExecution() {
	final ExecutionGraph executionGraph = getVertex().getExecutionGraph();
	final SlotProvider resourceProvider = executionGraph.getSlotProvider();
	final boolean allowQueued = executionGraph.isQueuedSchedulingAllowed();
	return scheduleForExecution(
		resourceProvider,
		allowQueued,
		LocationPreferenceConstraint.ANY,
		Collections.emptySet());
}
 
Example 11
Source Project: Flink-CEPplus   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
private void scheduleConsumer(ExecutionVertex consumerVertex) {
	try {
		final ExecutionGraph executionGraph = consumerVertex.getExecutionGraph();
		consumerVertex.scheduleForExecution(
			executionGraph.getSlotProvider(),
			executionGraph.isQueuedSchedulingAllowed(),
			LocationPreferenceConstraint.ANY, // there must be at least one known location
			Collections.emptySet());
	} catch (Throwable t) {
		consumerVertex.fail(new IllegalStateException("Could not schedule consumer " +
			"vertex " + consumerVertex, t));
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calculates the preferred locations based on the location preference constraint.
 *
 * @param locationPreferenceConstraint constraint for the location preference
 * @return Future containing the collection of preferred locations. This might not be completed if not all inputs
 * 		have been a resource assigned.
 */
@VisibleForTesting
public CompletableFuture<Collection<TaskManagerLocation>> calculatePreferredLocations(LocationPreferenceConstraint locationPreferenceConstraint) {
	final Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures = getVertex().getPreferredLocations();
	final CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture;

	switch(locationPreferenceConstraint) {
		case ALL:
			preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures);
			break;
		case ANY:
			final ArrayList<TaskManagerLocation> completedTaskManagerLocations = new ArrayList<>(preferredLocationFutures.size());

			for (CompletableFuture<TaskManagerLocation> preferredLocationFuture : preferredLocationFutures) {
				if (preferredLocationFuture.isDone() && !preferredLocationFuture.isCompletedExceptionally()) {
					final TaskManagerLocation taskManagerLocation = preferredLocationFuture.getNow(null);

					if (taskManagerLocation == null) {
						throw new FlinkRuntimeException("TaskManagerLocationFuture was completed with null. This indicates a programming bug.");
					}

					completedTaskManagerLocations.add(taskManagerLocation);
				}
			}

			preferredLocationsFuture = CompletableFuture.completedFuture(completedTaskManagerLocations);
			break;
		default:
			throw new RuntimeException("Unknown LocationPreferenceConstraint " + locationPreferenceConstraint + '.');
	}

	return preferredLocationsFuture;
}
 
Example 13
Source Project: Flink-CEPplus   Source File: ExecutionJobVertex.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Acquires a slot for all the execution vertices of this ExecutionJobVertex. The method returns
 * pairs of the slots and execution attempts, to ease correlation between vertices and execution
 * attempts.
 *
 * <p>If this method throws an exception, it makes sure to release all so far requested slots.
 *
 * @param resourceProvider The resource provider from whom the slots are requested.
 * @param queued if the allocation can be queued
 * @param locationPreferenceConstraint constraint for the location preferences
 * @param allPreviousExecutionGraphAllocationIds the allocation ids of all previous executions in the execution job graph.
 * @param allocationTimeout timeout for allocating the individual slots
 */
public Collection<CompletableFuture<Execution>> allocateResourcesForAll(
		SlotProvider resourceProvider,
		boolean queued,
		LocationPreferenceConstraint locationPreferenceConstraint,
		@Nonnull Set<AllocationID> allPreviousExecutionGraphAllocationIds,
		Time allocationTimeout) {
	final ExecutionVertex[] vertices = this.taskVertices;

	@SuppressWarnings("unchecked")
	final CompletableFuture<Execution>[] slots = new CompletableFuture[vertices.length];

	// try to acquire a slot future for each execution.
	// we store the execution with the future just to be on the safe side
	for (int i = 0; i < vertices.length; i++) {
		// allocate the next slot (future)
		final Execution exec = vertices[i].getCurrentExecutionAttempt();
		final CompletableFuture<Execution> allocationFuture = exec.allocateAndAssignSlotForExecution(
			resourceProvider,
			queued,
			locationPreferenceConstraint,
			allPreviousExecutionGraphAllocationIds,
			allocationTimeout);
		slots[i] = allocationFuture;
	}

	// all good, we acquired all slots
	return Arrays.asList(slots);
}
 
Example 14
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
		future.complete(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(new TestingSlotProvider((i) -> future), false, LocationPreferenceConstraint.ALL, Collections.emptySet());

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 15
@Test
public void testSlotReleasedWhenScheduledQueued() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		final CompletableFuture<LogicalSlot> future = new CompletableFuture<>();

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(
			new TestingSlotProvider(ignore -> future),
			true,
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());

		// future has not yet a slot
		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());

		future.complete(slot);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(slotProvider, false, LocationPreferenceConstraint.ANY, Collections.emptySet()).get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 
Example 17
Source Project: flink   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Void> scheduleForExecution() {
	final ExecutionGraph executionGraph = getVertex().getExecutionGraph();
	final SlotProviderStrategy resourceProvider = executionGraph.getSlotProviderStrategy();
	return scheduleForExecution(
		resourceProvider,
		LocationPreferenceConstraint.ANY,
		Collections.emptySet());
}
 
Example 18
Source Project: flink   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
private void scheduleConsumer(ExecutionVertex consumerVertex) {
	try {
		final ExecutionGraph executionGraph = consumerVertex.getExecutionGraph();
		consumerVertex.scheduleForExecution(
			executionGraph.getSlotProviderStrategy(),
			LocationPreferenceConstraint.ANY, // there must be at least one known location
			Collections.emptySet());
	} catch (Throwable t) {
		consumerVertex.fail(new IllegalStateException("Could not schedule consumer " +
			"vertex " + consumerVertex, t));
	}
}
 
Example 19
Source Project: flink   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calculates the preferred locations based on the location preference constraint.
 *
 * @param locationPreferenceConstraint constraint for the location preference
 * @return Future containing the collection of preferred locations. This might not be completed if not all inputs
 * 		have been a resource assigned.
 */
@VisibleForTesting
public CompletableFuture<Collection<TaskManagerLocation>> calculatePreferredLocations(LocationPreferenceConstraint locationPreferenceConstraint) {
	final Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures = getVertex().getPreferredLocations();
	final CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture;

	switch(locationPreferenceConstraint) {
		case ALL:
			preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures);
			break;
		case ANY:
			final ArrayList<TaskManagerLocation> completedTaskManagerLocations = new ArrayList<>(preferredLocationFutures.size());

			for (CompletableFuture<TaskManagerLocation> preferredLocationFuture : preferredLocationFutures) {
				if (preferredLocationFuture.isDone() && !preferredLocationFuture.isCompletedExceptionally()) {
					final TaskManagerLocation taskManagerLocation = preferredLocationFuture.getNow(null);

					if (taskManagerLocation == null) {
						throw new FlinkRuntimeException("TaskManagerLocationFuture was completed with null. This indicates a programming bug.");
					}

					completedTaskManagerLocations.add(taskManagerLocation);
				}
			}

			preferredLocationsFuture = CompletableFuture.completedFuture(completedTaskManagerLocations);
			break;
		default:
			throw new RuntimeException("Unknown LocationPreferenceConstraint " + locationPreferenceConstraint + '.');
	}

	return preferredLocationsFuture;
}
 
Example 20
Source Project: flink   Source File: SchedulingUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Schedule vertices lazy. That means only vertices satisfying its input constraint will be scheduled.
 *
 * @param vertices Topologically sorted vertices to schedule.
 * @param executionGraph The graph the given vertices belong to.
 */
public static CompletableFuture<Void> scheduleLazy(
		final Iterable<ExecutionVertex> vertices,
		final ExecutionGraph executionGraph) {

	executionGraph.assertRunningInJobMasterMainThread();

	final SlotProviderStrategy slotProviderStrategy = executionGraph.getSlotProviderStrategy();
	final Set<AllocationID> previousAllocations = computePriorAllocationIdsIfRequiredByScheduling(
		vertices, slotProviderStrategy.asSlotProvider());

	final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>();
	for (ExecutionVertex executionVertex : vertices) {
		// only schedule vertex when its input constraint is satisfied
		if (executionVertex.getJobVertex().getJobVertex().isInputVertex() ||
			executionVertex.checkInputDependencyConstraints()) {

			final CompletableFuture<Void> schedulingVertexFuture = executionVertex.scheduleForExecution(
				slotProviderStrategy,
				LocationPreferenceConstraint.ANY,
				previousAllocations);

			schedulingFutures.add(schedulingVertexFuture);
		}
	}

	return FutureUtils.waitForAll(schedulingFutures);
}
 
Example 21
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
		future.complete(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider((i) -> future), false),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 22
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSlotReleasedWhenScheduledQueued() {
	try {
		final ExecutionJobVertex ejv = getExecutionVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		final CompletableFuture<LogicalSlot> future = new CompletableFuture<>();

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider(ignore -> future), true),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());

		// future has not yet a slot
		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());

		future.complete(slot);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 23
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(
		executionGraph.getSlotProviderStrategy(),
		LocationPreferenceConstraint.ANY,
		Collections.emptySet())
		.get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 
Example 24
Source Project: flink   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
public CompletableFuture<Void> scheduleForExecution() {
	final ExecutionGraph executionGraph = getVertex().getExecutionGraph();
	final SlotProviderStrategy resourceProvider = executionGraph.getSlotProviderStrategy();
	return scheduleForExecution(
		resourceProvider,
		LocationPreferenceConstraint.ANY,
		Collections.emptySet());
}
 
Example 25
Source Project: flink   Source File: Execution.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Calculates the preferred locations based on the location preference constraint.
 *
 * @param locationPreferenceConstraint constraint for the location preference
 * @return Future containing the collection of preferred locations. This might not be completed if not all inputs
 * 		have been a resource assigned.
 */
@VisibleForTesting
public CompletableFuture<Collection<TaskManagerLocation>> calculatePreferredLocations(LocationPreferenceConstraint locationPreferenceConstraint) {
	final Collection<CompletableFuture<TaskManagerLocation>> preferredLocationFutures = getVertex().getPreferredLocations();
	final CompletableFuture<Collection<TaskManagerLocation>> preferredLocationsFuture;

	switch(locationPreferenceConstraint) {
		case ALL:
			preferredLocationsFuture = FutureUtils.combineAll(preferredLocationFutures);
			break;
		case ANY:
			final ArrayList<TaskManagerLocation> completedTaskManagerLocations = new ArrayList<>(preferredLocationFutures.size());

			for (CompletableFuture<TaskManagerLocation> preferredLocationFuture : preferredLocationFutures) {
				if (preferredLocationFuture.isDone() && !preferredLocationFuture.isCompletedExceptionally()) {
					final TaskManagerLocation taskManagerLocation = preferredLocationFuture.getNow(null);

					if (taskManagerLocation == null) {
						throw new FlinkRuntimeException("TaskManagerLocationFuture was completed with null. This indicates a programming bug.");
					}

					completedTaskManagerLocations.add(taskManagerLocation);
				}
			}

			preferredLocationsFuture = CompletableFuture.completedFuture(completedTaskManagerLocations);
			break;
		default:
			throw new RuntimeException("Unknown LocationPreferenceConstraint " + locationPreferenceConstraint + '.');
	}

	return preferredLocationsFuture;
}
 
Example 26
Source Project: flink   Source File: SchedulingUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Schedule vertices lazy. That means only vertices satisfying its input constraint will be scheduled.
 *
 * @param vertices Topologically sorted vertices to schedule.
 * @param executionGraph The graph the given vertices belong to.
 */
public static CompletableFuture<Void> scheduleLazy(
		final Iterable<ExecutionVertex> vertices,
		final ExecutionGraph executionGraph) {

	executionGraph.assertRunningInJobMasterMainThread();

	final SlotProviderStrategy slotProviderStrategy = executionGraph.getSlotProviderStrategy();
	final Set<AllocationID> previousAllocations = computePriorAllocationIdsIfRequiredByScheduling(
		vertices, slotProviderStrategy.asSlotProvider());

	final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>();
	for (ExecutionVertex executionVertex : vertices) {
		// only schedule vertex when its input constraint is satisfied
		if (executionVertex.getJobVertex().getJobVertex().isInputVertex() ||
			executionVertex.checkInputDependencyConstraints()) {

			final CompletableFuture<Void> schedulingVertexFuture = executionVertex.scheduleForExecution(
				slotProviderStrategy,
				LocationPreferenceConstraint.ANY,
				previousAllocations);

			schedulingFutures.add(schedulingVertexFuture);
		}
	}

	return FutureUtils.waitForAll(schedulingFutures);
}
 
Example 27
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSlotReleasedWhenScheduledImmediately() {
	try {
		final ExecutionJobVertex ejv = getExecutionJobVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		CompletableFuture<LogicalSlot> future = new CompletableFuture<>();
		future.complete(slot);

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider((i) -> future)),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 28
Source Project: flink   Source File: ExecutionVertexSchedulingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSlotReleasedWhenScheduledQueued() {
	try {
		final ExecutionJobVertex ejv = getExecutionJobVertex(new JobVertexID());
		final ExecutionVertex vertex = new ExecutionVertex(ejv, 0, new IntermediateResult[0],
				AkkaUtils.getDefaultTimeout());

		// a slot than cannot be deployed to
		final LogicalSlot slot = new TestingLogicalSlotBuilder().createTestingLogicalSlot();
		slot.releaseSlot(new Exception("Test Exception"));

		assertFalse(slot.isAlive());

		final CompletableFuture<LogicalSlot> future = new CompletableFuture<>();

		assertEquals(ExecutionState.CREATED, vertex.getExecutionState());
		// try to deploy to the slot
		vertex.scheduleForExecution(
			TestingSlotProviderStrategy.from(new TestingSlotProvider(ignore -> future)),
			LocationPreferenceConstraint.ALL,
			Collections.emptySet());

		// future has not yet a slot
		assertEquals(ExecutionState.SCHEDULED, vertex.getExecutionState());

		future.complete(slot);

		// will have failed
		assertEquals(ExecutionState.FAILED, vertex.getExecutionState());
	}
	catch (Exception e) {
		e.printStackTrace();
		fail(e.getMessage());
	}
}
 
Example 29
Source Project: flink   Source File: ExecutionTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that the task restore state is nulled after the {@link Execution} has been
 * deployed. See FLINK-9693.
 */
@Test
public void testTaskRestoreStateIsNulledAfterDeployment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();
	final ProgrammedSlotProvider slotProvider = createProgrammedSlotProvider(
		1,
		Collections.singleton(jobVertexId),
		slotOwner);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	ExecutionVertex executionVertex = executionJobVertex.getTaskVertices()[0];

	final Execution execution = executionVertex.getCurrentExecutionAttempt();

	final JobManagerTaskRestore taskRestoreState = new JobManagerTaskRestore(1L, new TaskStateSnapshot());
	execution.setInitialState(taskRestoreState);

	assertThat(execution.getTaskRestore(), is(notNullValue()));

	// schedule the execution vertex and wait for its deployment
	executionVertex.scheduleForExecution(
		executionGraph.getSlotProviderStrategy(),
		LocationPreferenceConstraint.ANY,
		Collections.emptySet())
		.get();

	assertThat(execution.getTaskRestore(), is(nullValue()));
}
 
Example 30
Source Project: Flink-CEPplus   Source File: ExecutionTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that slots are released if we cannot assign the allocated resource to the
 * Execution.
 */
@Test
public void testSlotReleaseOnFailedResourceAssignment() throws Exception {
	final JobVertex jobVertex = createNoOpJobVertex();
	final JobVertexID jobVertexId = jobVertex.getID();

	final CompletableFuture<LogicalSlot> slotFuture = new CompletableFuture<>();
	final ProgrammedSlotProvider slotProvider = new ProgrammedSlotProvider(1);
	slotProvider.addSlot(jobVertexId, 0, slotFuture);

	ExecutionGraph executionGraph = ExecutionGraphTestUtils.createSimpleTestGraph(
		new JobID(),
		slotProvider,
		new NoRestartStrategy(),
		jobVertex);

	executionGraph.start(TestingComponentMainThreadExecutorServiceAdapter.forMainThread());

	ExecutionJobVertex executionJobVertex = executionGraph.getJobVertex(jobVertexId);

	final Execution execution = executionJobVertex.getTaskVertices()[0].getCurrentExecutionAttempt();

	final SingleSlotTestingSlotOwner slotOwner = new SingleSlotTestingSlotOwner();

	final SimpleSlot slot = new SimpleSlot(
		slotOwner,
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway());

	final LogicalSlot otherSlot = new TestingLogicalSlot();

	CompletableFuture<Execution> allocationFuture = execution.allocateAndAssignSlotForExecution(
		slotProvider,
		false,
		LocationPreferenceConstraint.ALL,
		Collections.emptySet(),
		TestingUtils.infiniteTime());

	assertFalse(allocationFuture.isDone());

	assertEquals(ExecutionState.SCHEDULED, execution.getState());

	// assign a different resource to the execution
	assertTrue(execution.tryAssignResource(otherSlot));

	// completing now the future should cause the slot to be released
	slotFuture.complete(slot);

	assertEquals(slot, slotOwner.getReturnedSlotFuture().get());
}