Java Code Examples for org.apache.flink.runtime.clusterframework.types.AllocationID

The following examples show how to use org.apache.flink.runtime.clusterframework.types.AllocationID. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: flink   Source File: TaskSlotTableImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public boolean markSlotInactive(AllocationID allocationId, Time slotTimeout) throws SlotNotFoundException {
	checkStarted();

	TaskSlot<T> taskSlot = getTaskSlot(allocationId);

	if (taskSlot != null) {
		if (taskSlot.markInactive()) {
			// register a timeout to free the slot
			timerService.registerTimeout(allocationId, slotTimeout.getSize(), slotTimeout.getUnit());

			return true;
		} else {
			return false;
		}
	} else {
		throw new SlotNotFoundException(allocationId);
	}
}
 
Example 2
Source Project: flink   Source File: SlotManagerFailUnfulfillableTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUnfulfillableRequestsFailWhenOn() {
	// setup
	final ResourceProfile availableProfile = ResourceProfile.fromResources(2.0, 100);
	final ResourceProfile unfulfillableProfile = ResourceProfile.fromResources(2.0, 200);

	final List<Tuple3<JobID, AllocationID, Exception>> notifiedAllocationFailures = new ArrayList<>();
	final SlotManager slotManager = createSlotManagerNotStartingNewTMs(notifiedAllocationFailures);
	registerFreeSlot(slotManager, availableProfile);

	// test
	try {
		slotManager.registerSlotRequest(slotRequest(unfulfillableProfile));
		fail("this should cause an exception");
	} catch (ResourceManagerException exception) {
		assertTrue(ExceptionUtils.findThrowable(exception, UnfulfillableSlotRequestException.class).isPresent());
	}

	// assert
	assertEquals(0, notifiedAllocationFailures.size());
	assertEquals(0, slotManager.getNumberPendingSlotRequests());
}
 
Example 3
Source Project: flink   Source File: SlotManagerTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that if we have received a slot report with some allocated slots, then we don't accept
 * slot requests with allocated allocation ids.
 */
@Test
public void testDuplicatePendingSlotRequestAfterSlotReport() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceActions resourceManagerActions = mock(ResourceActions.class);
	final JobID jobId = new JobID();
	final AllocationID allocationId = new AllocationID();
	final ResourceProfile resourceProfile = new ResourceProfile(1.0, 1);
	final ResourceID resourceID = ResourceID.generate();
	final SlotID slotId = new SlotID(resourceID, 0);

	final TaskExecutorGateway taskExecutorGateway = mock(TaskExecutorGateway.class);
	final TaskExecutorConnection taskManagerConnection = new TaskExecutorConnection(resourceID, taskExecutorGateway);

	final SlotStatus slotStatus = new SlotStatus(slotId, resourceProfile, jobId, allocationId);
	final SlotReport slotReport = new SlotReport(slotStatus);

	final SlotRequest slotRequest = new SlotRequest(jobId, allocationId, resourceProfile, "foobar");

	try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {
		slotManager.registerTaskManager(taskManagerConnection, slotReport);

		assertFalse(slotManager.registerSlotRequest(slotRequest));
	}
}
 
Example 4
Source Project: flink   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example 5
Source Project: Flink-CEPplus   Source File: TaskLocalStateStoreImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example 6
Source Project: flink   Source File: TaskLocalStateStoreImpl.java    License: Apache License 2.0 6 votes vote down vote up
@VisibleForTesting
TaskLocalStateStoreImpl(
	@Nonnull JobID jobID,
	@Nonnull AllocationID allocationID,
	@Nonnull JobVertexID jobVertexID,
	@Nonnegative int subtaskIndex,
	@Nonnull LocalRecoveryConfig localRecoveryConfig,
	@Nonnull Executor discardExecutor,
	@Nonnull SortedMap<Long, TaskStateSnapshot> storedTaskStateByCheckpointID,
	@Nonnull Object lock) {

	this.jobID = jobID;
	this.allocationID = allocationID;
	this.jobVertexID = jobVertexID;
	this.subtaskIndex = subtaskIndex;
	this.discardExecutor = discardExecutor;
	this.localRecoveryConfig = localRecoveryConfig;
	this.storedTaskStateByCheckpointID = storedTaskStateByCheckpointID;
	this.lock = lock;
	this.disposed = false;
}
 
Example 7
Source Project: Flink-CEPplus   Source File: ExecutionVertexLocalityTest.java    License: Apache License 2.0 6 votes vote down vote up
private void initializeLocation(ExecutionVertex vertex, TaskManagerLocation location) throws Exception {
	// we need a bit of reflection magic to initialize the location without going through
	// scheduling paths. we choose to do that, rather than the alternatives:
	//  - mocking the scheduler created fragile tests that break whenever the scheduler is adjusted
	//  - exposing test methods in the ExecutionVertex leads to undesirable setters 

	SlotContext slot = new SimpleSlotContext(
		new AllocationID(),
		location,
		0,
		mock(TaskManagerGateway.class));

	SimpleSlot simpleSlot = new SimpleSlot(slot, mock(SlotOwner.class), 0);

	if (!vertex.getCurrentExecutionAttempt().tryAssignResource(simpleSlot)) {
		throw new FlinkException("Could not assign resource.");
	}
}
 
Example 8
Source Project: flink   Source File: SlotManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void checkSlotRequestTimeouts() {
	if (!pendingSlotRequests.isEmpty()) {
		long currentTime = System.currentTimeMillis();

		Iterator<Map.Entry<AllocationID, PendingSlotRequest>> slotRequestIterator = pendingSlotRequests.entrySet().iterator();

		while (slotRequestIterator.hasNext()) {
			PendingSlotRequest slotRequest = slotRequestIterator.next().getValue();

			if (currentTime - slotRequest.getCreationTimestamp() >= slotRequestTimeout.toMilliseconds()) {
				slotRequestIterator.remove();

				if (slotRequest.isAssigned()) {
					cancelPendingSlotRequest(slotRequest);
				}

				resourceActions.notifyAllocationFailure(
					slotRequest.getJobId(),
					slotRequest.getAllocationId(),
					new TimeoutException("The allocation could not be fulfilled in time."));
			}
		}
	}
}
 
Example 9
public void releaseLocalStateForAllocationId(@Nonnull AllocationID allocationID) {

		if (LOG.isDebugEnabled()) {
			LOG.debug("Releasing local state under allocation id {}.", allocationID);
		}

		Map<JobVertexSubtaskKey, OwnedTaskLocalStateStore> cleanupLocalStores;

		synchronized (lock) {
			if (closed) {
				return;
			}
			cleanupLocalStores = taskStateStoresByAllocationID.remove(allocationID);
		}

		if (cleanupLocalStores != null) {
			doRelease(cleanupLocalStores.values());
		}

		cleanupAllocationBaseDirs(allocationID);
	}
 
Example 10
Source Project: flink   Source File: SlotPoolBatchSlotRequestTest.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a batch slot request does not react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is not {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfAllocationFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(), new FlinkException("Failed request"));

		assertThat(slotFuture.isDone(), is(false));
	}
}
 
Example 11
Source Project: flink   Source File: SlotSharingManager.java    License: Apache License 2.0 6 votes vote down vote up
private double calculateTaskExecutorUtilization(Map<AllocationID, MultiTaskSlot> map, AbstractID groupId) {
	int numberValidSlots = 0;
	int numberFreeSlots = 0;

	for (MultiTaskSlot multiTaskSlot : map.values()) {
		if (isNotReleasing(multiTaskSlot)) {
			numberValidSlots++;

			if (doesNotContain(groupId, multiTaskSlot)) {
				numberFreeSlots++;
			}
		}
	}

	return (double) (numberValidSlots - numberFreeSlots) / numberValidSlots;
}
 
Example 12
Source Project: flink   Source File: SlotPoolImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void close() {
	log.info("Stopping SlotPool.");
	// cancel all pending allocations
	Set<AllocationID> allocationIds = pendingRequests.keySetB();

	for (AllocationID allocationId : allocationIds) {
		resourceManagerGateway.cancelSlotRequest(allocationId);
	}

	// release all registered slots by releasing the corresponding TaskExecutors
	for (ResourceID taskManagerResourceId : registeredTaskManagers) {
		final FlinkException cause = new FlinkException(
			"Releasing TaskManager " + taskManagerResourceId + ", because of stopping of SlotPool");
		releaseTaskManagerInternal(taskManagerResourceId, cause);
	}

	clear();
}
 
Example 13
Source Project: flink   Source File: SlotManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Cancels and removes a pending slot request with the given allocation id. If there is no such
 * pending request, then nothing is done.
 *
 * @param allocationId identifying the pending slot request
 * @return True if a pending slot request was found; otherwise false
 */
@Override
public boolean unregisterSlotRequest(AllocationID allocationId) {
	checkInit();

	PendingSlotRequest pendingSlotRequest = pendingSlotRequests.remove(allocationId);

	if (null != pendingSlotRequest) {
		LOG.debug("Cancel slot request {}.", allocationId);

		cancelPendingSlotRequest(pendingSlotRequest);

		return true;
	} else {
		LOG.debug("No pending slot request with allocation id {} found. Ignoring unregistration request.", allocationId);

		return false;
	}
}
 
Example 14
Source Project: flink   Source File: TaskLocalStateStoreImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void before() throws Exception {
	JobID jobID = new JobID();
	AllocationID allocationID = new AllocationID();
	JobVertexID jobVertexID = new JobVertexID();
	int subtaskIdx = 0;
	this.temporaryFolder = new TemporaryFolder();
	this.temporaryFolder.create();
	this.allocationBaseDirs = new File[]{temporaryFolder.newFolder(), temporaryFolder.newFolder()};
	this.internalSnapshotMap = new TreeMap<>();
	this.internalLock = new Object();

	LocalRecoveryDirectoryProviderImpl directoryProvider =
		new LocalRecoveryDirectoryProviderImpl(allocationBaseDirs, jobID, jobVertexID, subtaskIdx);

	LocalRecoveryConfig localRecoveryConfig = new LocalRecoveryConfig(false, directoryProvider);

	this.taskLocalStateStore = new TaskLocalStateStoreImpl(
		jobID,
		allocationID,
		jobVertexID,
		subtaskIdx,
		localRecoveryConfig,
		Executors.directExecutor(),
		internalSnapshotMap,
		internalLock);
}
 
Example 15
Source Project: Flink-CEPplus   Source File: TaskDeploymentDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
public TaskDeploymentDescriptor(
	JobID jobId,
	MaybeOffloaded<JobInformation> serializedJobInformation,
	MaybeOffloaded<TaskInformation> serializedTaskInformation,
	ExecutionAttemptID executionAttemptId,
	AllocationID allocationId,
	int subtaskIndex,
	int attemptNumber,
	int targetSlotNumber,
	@Nullable JobManagerTaskRestore taskRestore,
	Collection<ResultPartitionDeploymentDescriptor> resultPartitionDeploymentDescriptors,
	Collection<InputGateDeploymentDescriptor> inputGateDeploymentDescriptors) {

	this.jobId = Preconditions.checkNotNull(jobId);

	this.serializedJobInformation = Preconditions.checkNotNull(serializedJobInformation);
	this.serializedTaskInformation = Preconditions.checkNotNull(serializedTaskInformation);

	this.executionId = Preconditions.checkNotNull(executionAttemptId);
	this.allocationId = Preconditions.checkNotNull(allocationId);

	Preconditions.checkArgument(0 <= subtaskIndex, "The subtask index must be positive.");
	this.subtaskIndex = subtaskIndex;

	Preconditions.checkArgument(0 <= attemptNumber, "The attempt number must be positive.");
	this.attemptNumber = attemptNumber;

	Preconditions.checkArgument(0 <= targetSlotNumber, "The target slot number must be positive.");
	this.targetSlotNumber = targetSlotNumber;

	this.taskRestore = taskRestore;

	this.producedPartitions = Preconditions.checkNotNull(resultPartitionDeploymentDescriptors);
	this.inputGates = Preconditions.checkNotNull(inputGateDeploymentDescriptors);
}
 
Example 16
Source Project: flink   Source File: SchedulingUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the result of {@link #computePriorAllocationIds(Iterable)},
 * but only if the scheduling really requires it.
 * Otherwise this method simply returns an empty set.
 */
private static Set<AllocationID> computePriorAllocationIdsIfRequiredByScheduling(
		final Iterable<ExecutionVertex> vertices,
		final SlotProvider slotProvider) {
	// This is a temporary optimization to avoid computing all previous allocations if not required
	// This can go away when we progress with the implementation of the Scheduler.
	if (slotProvider instanceof Scheduler &&
		((Scheduler) slotProvider).requiresPreviousExecutionGraphAllocations()) {

		return computePriorAllocationIds(vertices);
	} else {
		return Collections.emptySet();
	}
}
 
Example 17
Source Project: flink   Source File: SlotStatus.java    License: Apache License 2.0 5 votes vote down vote up
public SlotStatus(
	SlotID slotID,
	ResourceProfile resourceProfile,
	JobID jobID,
	AllocationID allocationID) {
	this.slotID = checkNotNull(slotID, "slotID cannot be null");
	this.resourceProfile = checkNotNull(resourceProfile, "profile cannot be null");
	this.allocationID = allocationID;
	this.jobID = jobID;
}
 
Example 18
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that we ignore slot requests if the TaskExecutor is not
 * registered at a ResourceManager.
 */
@Test
public void testIgnoringSlotRequestsIfNotRegistered() throws Exception {
	final TaskExecutor taskExecutor = createTaskExecutor(1);

	taskExecutor.start();

	try {
		final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();

		final CompletableFuture<RegistrationResponse> registrationFuture = new CompletableFuture<>();
		final CompletableFuture<ResourceID> taskExecutorResourceIdFuture = new CompletableFuture<>();

		testingResourceManagerGateway.setRegisterTaskExecutorFunction(taskExecutorRegistration -> {
			taskExecutorResourceIdFuture.complete(taskExecutorRegistration.getResourceId());
			return registrationFuture;
		});

		rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
		resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

		final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

		final ResourceID resourceId = taskExecutorResourceIdFuture.get();

		final SlotID slotId = new SlotID(resourceId, 0);
		final CompletableFuture<Acknowledge> slotRequestResponse = taskExecutorGateway.requestSlot(slotId, jobId, new AllocationID(), ResourceProfile.ZERO, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);

		try {
			slotRequestResponse.get();
			fail("We should not be able to request slots before the TaskExecutor is registered at the ResourceManager.");
		} catch (ExecutionException ee) {
			assertThat(ExceptionUtils.stripExecutionException(ee), instanceOf(TaskManagerException.class));
		}
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}
 
Example 19
Source Project: flink   Source File: TaskSlotTable.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void notifyTimeout(AllocationID key, UUID ticket) {
	checkInit();

	if (slotActions != null) {
		slotActions.timeoutSlot(key, ticket);
	}
}
 
Example 20
Source Project: Flink-CEPplus   Source File: AllocatedSlot.java    License: Apache License 2.0 5 votes vote down vote up
public AllocatedSlot(
		AllocationID allocationId,
		TaskManagerLocation location,
		int physicalSlotNumber,
		ResourceProfile resourceProfile,
		TaskManagerGateway taskManagerGateway) {
	this.allocationId = checkNotNull(allocationId);
	this.taskManagerLocation = checkNotNull(location);
	this.physicalSlotNumber = physicalSlotNumber;
	this.resourceProfile = checkNotNull(resourceProfile);
	this.taskManagerGateway = checkNotNull(taskManagerGateway);

	payloadReference = new AtomicReference<>(null);
}
 
Example 21
Source Project: flink   Source File: TaskSlotTableImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int freeSlot(AllocationID allocationId, Throwable cause) throws SlotNotFoundException {
	checkStarted();

	TaskSlot<T> taskSlot = getTaskSlot(allocationId);

	if (taskSlot != null) {
		return freeSlotInternal(taskSlot, cause).isDone() ? taskSlot.getIndex() : -1;
	} else {
		throw new SlotNotFoundException(allocationId);
	}
}
 
Example 22
Source Project: flink   Source File: TaskSlotTableImpl.java    License: Apache License 2.0 5 votes vote down vote up
private CompletableFuture<Void> freeSlotInternal(TaskSlot<T> taskSlot, Throwable cause) {
	AllocationID allocationId = taskSlot.getAllocationId();

	if (LOG.isDebugEnabled()) {
		LOG.debug("Free slot {}.", taskSlot, cause);
	} else {
		LOG.info("Free slot {}.", taskSlot);
	}

	if (taskSlot.isEmpty()) {
		// remove the allocation id to task slot mapping
		allocatedSlots.remove(allocationId);

		// unregister a potential timeout
		timerService.unregisterTimeout(allocationId);

		JobID jobId = taskSlot.getJobId();
		Set<AllocationID> slots = slotsPerJob.get(jobId);

		if (slots == null) {
			throw new IllegalStateException("There are no more slots allocated for the job " + jobId +
				". This indicates a programming bug.");
		}

		slots.remove(allocationId);

		if (slots.isEmpty()) {
			slotsPerJob.remove(jobId);
		}

		taskSlots.remove(taskSlot.getIndex());
		budgetManager.release(taskSlot.getResourceProfile());
	}
	return taskSlot.closeAsync(cause);
}
 
Example 23
Source Project: flink   Source File: ResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void notifyAllocationFailure(JobID jobId, AllocationID allocationId, Exception cause) {
	validateRunsInMainThread();

	JobManagerRegistration jobManagerRegistration = jobManagerRegistrations.get(jobId);
	if (jobManagerRegistration != null) {
		jobManagerRegistration.getJobManagerGateway().notifyAllocationFailure(allocationId, cause);
	}
}
 
Example 24
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public boolean markSlotActive(AllocationID allocationId) throws SlotNotFoundException {
	final boolean result = super.markSlotActive(allocationId);

	if (result) {
		slotsToActivate.countDown();
	}

	return result;
}
 
Example 25
Source Project: flink   Source File: SlotPoolImpl.java    License: Apache License 2.0 5 votes vote down vote up
AllocatedSlot get(AllocationID allocationID) {
	SlotAndTimestamp slotAndTimestamp = availableSlots.get(allocationID);
	if (slotAndTimestamp != null) {
		return slotAndTimestamp.slot();
	} else {
		return null;
	}
}
 
Example 26
Source Project: flink   Source File: SimpleSlotContext.java    License: Apache License 2.0 5 votes vote down vote up
public SimpleSlotContext(
		AllocationID allocationId,
		TaskManagerLocation taskManagerLocation,
		int physicalSlotNumber,
		TaskManagerGateway taskManagerGateway,
		ResourceProfile resourceProfile) {
	this.allocationId = Preconditions.checkNotNull(allocationId);
	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
	this.physicalSlotNumber = physicalSlotNumber;
	this.taskManagerGateway = Preconditions.checkNotNull(taskManagerGateway);
	this.resourceProfile = resourceProfile;
}
 
Example 27
Source Project: Flink-CEPplus   Source File: SimpleSlotContext.java    License: Apache License 2.0 5 votes vote down vote up
public SimpleSlotContext(
		AllocationID allocationId,
		TaskManagerLocation taskManagerLocation,
		int physicalSlotNumber,
		TaskManagerGateway taskManagerGateway) {
	this(allocationId, taskManagerLocation, physicalSlotNumber, taskManagerGateway, ResourceProfile.UNKNOWN);
}
 
Example 28
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void freeNoLongerUsedSlots(AllocatedSlotReport allocatedSlotReport) {
	final Iterator<AllocationID> slotsTaskManagerSide = taskSlotTable.getActiveSlots(allocatedSlotReport.getJobId());
	final Set<AllocationID> activeSlots = Sets.newHashSet(slotsTaskManagerSide);
	final Set<AllocationID> reportedSlots = allocatedSlotReport.getAllocatedSlotInfos().stream()
			.map(AllocatedSlotInfo::getAllocationId).collect(Collectors.toSet());

	final Sets.SetView<AllocationID> difference = Sets.difference(activeSlots, reportedSlots);

	for (AllocationID allocationID : difference) {
		freeSlotInternal(
			allocationID,
			new FlinkException(
				String.format("%s is no longer allocated by job %s.", allocationID, allocatedSlotReport.getJobId())));
	}
}
 
Example 29
Source Project: flink   Source File: SchedulingUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Schedule vertices lazy. That means only vertices satisfying its input constraint will be scheduled.
 *
 * @param vertices Topologically sorted vertices to schedule.
 * @param executionGraph The graph the given vertices belong to.
 */
public static CompletableFuture<Void> scheduleLazy(
		final Iterable<ExecutionVertex> vertices,
		final ExecutionGraph executionGraph) {

	executionGraph.assertRunningInJobMasterMainThread();

	final SlotProviderStrategy slotProviderStrategy = executionGraph.getSlotProviderStrategy();
	final Set<AllocationID> previousAllocations = computePriorAllocationIdsIfRequiredByScheduling(
		vertices, slotProviderStrategy.asSlotProvider());

	final ArrayList<CompletableFuture<Void>> schedulingFutures = new ArrayList<>();
	for (ExecutionVertex executionVertex : vertices) {
		// only schedule vertex when its input constraint is satisfied
		if (executionVertex.getJobVertex().getJobVertex().isInputVertex() ||
			executionVertex.checkInputDependencyConstraints()) {

			final CompletableFuture<Void> schedulingVertexFuture = executionVertex.scheduleForExecution(
				slotProviderStrategy,
				LocationPreferenceConstraint.ANY,
				previousAllocations);

			schedulingFutures.add(schedulingVertexFuture);
		}
	}

	return FutureUtils.waitForAll(schedulingFutures);
}
 
Example 30
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void freeSlotInternal(AllocationID allocationId, Throwable cause) {
	checkNotNull(allocationId);

	log.debug("Free slot with allocation id {} because: {}", allocationId, cause.getMessage());

	try {
		final JobID jobId = taskSlotTable.getOwningJob(allocationId);

		final int slotIndex = taskSlotTable.freeSlot(allocationId, cause);

		if (slotIndex != -1) {

			if (isConnectedToResourceManager()) {
				// the slot was freed. Tell the RM about it
				ResourceManagerGateway resourceManagerGateway = establishedResourceManagerConnection.getResourceManagerGateway();

				resourceManagerGateway.notifySlotAvailable(
					establishedResourceManagerConnection.getTaskExecutorRegistrationId(),
					new SlotID(getResourceID(), slotIndex),
					allocationId);
			}

			if (jobId != null) {
				closeJobManagerConnectionIfNoAllocatedResources(jobId);
			}
		}
	} catch (SlotNotFoundException e) {
		log.debug("Could not free slot for allocation id {}.", allocationId, e);
	}

	localStateStoresManager.releaseLocalStateForAllocationId(allocationId);
}