org.apache.flink.runtime.clusterframework.types.ResourceProfile Java Examples

The following examples show how to use org.apache.flink.runtime.clusterframework.types.ResourceProfile. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: SlotManagerFailUnfulfillableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTurnOnKeepsRequestsWithStartingTMs() throws Exception {
	// setup
	final ResourceProfile availableProfile = new ResourceProfile(2.0, 100);
	final ResourceProfile newTmProfile = new ResourceProfile(2.0, 200);

	final SlotManager slotManager = createSlotManagerStartingNewTMs();
	slotManager.setFailUnfulfillableRequest(false);
	registerFreeSlot(slotManager, availableProfile);

	// test
	slotManager.registerSlotRequest(slotRequest(newTmProfile));
	slotManager.setFailUnfulfillableRequest(true);

	// assert
	assertEquals(1, slotManager.getNumberPendingSlotRequests());
}
 
Example #2
Source File: SlotSharingManager.java    From flink with Apache License 2.0 6 votes vote down vote up
private SingleTaskSlot(
		SlotRequestId slotRequestId,
		ResourceProfile resourceProfile,
		AbstractID groupId,
		MultiTaskSlot parent,
		Locality locality) {
	super(slotRequestId, groupId);

	this.resourceProfile = Preconditions.checkNotNull(resourceProfile);
	this.parent = Preconditions.checkNotNull(parent);

	Preconditions.checkNotNull(locality);
	singleLogicalSlotFuture = parent.getSlotContextFuture()
		.thenApply(
			(SlotContext slotContext) -> {
				LOG.trace("Fulfill single task slot [{}] with slot [{}].", slotRequestId, slotContext.getAllocationId());
				return new SingleLogicalSlot(
					slotRequestId,
					slotContext,
					slotSharingGroupId,
					locality,
					slotOwner);
			});
}
 
Example #3
Source File: SchedulerIsolatedTasksTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testNewPhysicalSlotAllocation() {
	final ResourceProfile taskResourceProfile = ResourceProfile.fromResources(0.5, 250);
	final ResourceProfile physicalSlotResourceProfile = ResourceProfile.fromResources(1.0, 300);

	testingSlotProvider.allocateSlot(
		new SlotRequestId(),
		new ScheduledUnit(new JobVertexID(), null, null),
		SlotProfile.priorAllocation(
			taskResourceProfile,
			physicalSlotResourceProfile,
			Collections.emptyList(),
			Collections.emptyList(),
			Collections.emptySet()),
		TestingUtils.infiniteTime());

	assertEquals(physicalSlotResourceProfile, testingSlotProvider.getSlotPool().getLastRequestedSlotResourceProfile());
}
 
Example #4
Source File: SlotManagerTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a second pending slot request is detected as a duplicate if the allocation ids are
 * the same.
 */
@Test
public void testDuplicatePendingSlotRequest() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final AtomicInteger numberAllocateResourceFunctionCalls = new AtomicInteger(0);
	final ResourceActions resourceManagerActions = new TestingResourceActionsBuilder()
		.setAllocateResourceConsumer(resourceProfile -> numberAllocateResourceFunctionCalls.incrementAndGet())
		.build();
	final AllocationID allocationId = new AllocationID();
	final ResourceProfile resourceProfile1 = new ResourceProfile(1.0, 2);
	final ResourceProfile resourceProfile2 = new ResourceProfile(2.0, 1);
	final SlotRequest slotRequest1 = new SlotRequest(new JobID(), allocationId, resourceProfile1, "foobar");
	final SlotRequest slotRequest2 = new SlotRequest(new JobID(), allocationId, resourceProfile2, "barfoo");

	try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {
		assertTrue(slotManager.registerSlotRequest(slotRequest1));
		assertFalse(slotManager.registerSlotRequest(slotRequest2));
	}

	// check that we have only called the resource allocation only for the first slot request,
	// since the second request is a duplicate
	assertThat(numberAllocateResourceFunctionCalls.get(), is(1));
}
 
Example #5
Source File: SlotManagerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Finds a matching slot for a given resource profile. A matching slot has at least as many
 * resources available as the given resource profile. If there is no such slot available, then
 * the method returns null.
 *
 * <p>Note: If you want to change the behaviour of the slot manager wrt slot allocation and
 * request fulfillment, then you should override this method.
 *
 * @param requestResourceProfile specifying the resource requirements for the a slot request
 * @return A matching slot which fulfills the given resource profile. {@link Optional#empty()}
 * if there is no such slot available.
 */
private Optional<TaskManagerSlot> findMatchingSlot(ResourceProfile requestResourceProfile) {
	final Optional<TaskManagerSlot> optionalMatchingSlot = slotMatchingStrategy.findMatchingSlot(
		requestResourceProfile,
		freeSlots.values(),
		this::getNumberRegisteredSlotsOf);

	optionalMatchingSlot.ifPresent(taskManagerSlot -> {
		// sanity check
		Preconditions.checkState(
			taskManagerSlot.getState() == TaskManagerSlot.State.FREE,
			"TaskManagerSlot %s is not in state FREE but %s.",
			taskManagerSlot.getSlotId(), taskManagerSlot.getState());

		freeSlots.remove(taskManagerSlot.getSlotId());
	});

	return optionalMatchingSlot;
}
 
Example #6
Source File: JobMasterPartitionReleaseTest.java    From flink with Apache License 2.0 6 votes vote down vote up
private void registerTaskExecutorAtJobMaster(
		TestingRpcService rpcService,
		JobMasterGateway jobMasterGateway,
		TaskExecutorGateway taskExecutorGateway,
		SettableLeaderRetrievalService rmLeaderRetrievalService) throws ExecutionException, InterruptedException {

	final AllocationIdsResourceManagerGateway resourceManagerGateway = new AllocationIdsResourceManagerGateway();
	rpcService.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
	rmLeaderRetrievalService.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());

	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	jobMasterGateway.registerTaskManager(taskExecutorGateway.getAddress(), localTaskManagerUnresolvedLocation, testingTimeout).get();

	final AllocationID allocationId = resourceManagerGateway.takeAllocationId();
	Collection<SlotOffer> slotOffers = Collections.singleton(new SlotOffer(allocationId, 0, ResourceProfile.UNKNOWN));

	jobMasterGateway.offerSlots(localTaskManagerUnresolvedLocation.getResourceID(), slotOffers, testingTimeout).get();
}
 
Example #7
Source File: SlotManagerImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a slot request with no free slots will trigger the resource allocation.
 */
@Test
public void testSlotRequestWithoutFreeSlots() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
	final SlotRequest slotRequest = new SlotRequest(
		new JobID(),
		new AllocationID(),
		resourceProfile,
		"localhost");

	CompletableFuture<WorkerResourceSpec> allocateResourceFuture = new CompletableFuture<>();
	ResourceActions resourceManagerActions = new TestingResourceActionsBuilder()
		.setAllocateResourceConsumer(allocateResourceFuture::complete)
		.build();

	try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {

		slotManager.registerSlotRequest(slotRequest);

		allocateResourceFuture.get();
	}
}
 
Example #8
Source File: SlotManagerFailUnfulfillableTest.java    From flink with Apache License 2.0 6 votes vote down vote up
@Test
public void testTurnOnKeepsPendingFulfillableRequests() throws Exception {
	// setup
	final ResourceProfile availableProfile = new ResourceProfile(2.0, 100);
	final ResourceProfile fulfillableProfile = new ResourceProfile(1.0, 100);

	final SlotManager slotManager = createSlotManagerNotStartingNewTMs();
	slotManager.setFailUnfulfillableRequest(false);
	registerFreeSlot(slotManager, availableProfile);

	slotManager.registerSlotRequest(slotRequest(fulfillableProfile));
	slotManager.registerSlotRequest(slotRequest(fulfillableProfile));

	// test
	slotManager.setFailUnfulfillableRequest(true);

	// assert
	assertEquals(1, slotManager.getNumberPendingSlotRequests());
}
 
Example #9
Source File: SlotManagerImplTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that the slot request fails if we cannot allocate more resources.
 */
@Test
public void testSlotRequestWithResourceAllocationFailure() throws Exception {
	final ResourceManagerId resourceManagerId = ResourceManagerId.generate();
	final ResourceProfile resourceProfile = ResourceProfile.fromResources(42.0, 1337);
	final SlotRequest slotRequest = new SlotRequest(
		new JobID(),
		new AllocationID(),
		resourceProfile,
		"localhost");

	ResourceActions resourceManagerActions = new TestingResourceActionsBuilder()
		.setAllocateResourceFunction(value -> false)
		.build();

	try (SlotManager slotManager = createSlotManager(resourceManagerId, resourceManagerActions)) {

		slotManager.registerSlotRequest(slotRequest);

		fail("The slot request should have failed with a ResourceManagerException.");

	} catch (ResourceManagerException e) {
		// expected exception
	}
}
 
Example #10
Source File: SlotManagerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up
private void fulfillPendingSlotRequestWithPendingTaskManagerSlot(PendingSlotRequest pendingSlotRequest) throws ResourceManagerException {
	ResourceProfile resourceProfile = pendingSlotRequest.getResourceProfile();
	Optional<PendingTaskManagerSlot> pendingTaskManagerSlotOptional = findFreeMatchingPendingTaskManagerSlot(resourceProfile);

	if (!pendingTaskManagerSlotOptional.isPresent()) {
		pendingTaskManagerSlotOptional = allocateResource(resourceProfile);
	}

	OptionalConsumer.of(pendingTaskManagerSlotOptional)
		.ifPresent(pendingTaskManagerSlot -> assignPendingTaskManagerSlot(pendingSlotRequest, pendingTaskManagerSlot))
		.ifNotPresent(() -> {
			// request can not be fulfilled by any free slot or pending slot that can be allocated,
			// check whether it can be fulfilled by allocated slots
			if (failUnfulfillableRequest && !isFulfillableByRegisteredOrPendingSlots(pendingSlotRequest.getResourceProfile())) {
				throw new UnfulfillableSlotRequestException(pendingSlotRequest.getAllocationId(), pendingSlotRequest.getResourceProfile());
			}
		});
}
 
Example #11
Source File: TestingResourceActions.java    From flink with Apache License 2.0 5 votes vote down vote up
public TestingResourceActions(
		@Nonnull BiConsumer<InstanceID, Exception> releaseResourceConsumer,
		@Nonnull FunctionWithException<ResourceProfile, Collection<ResourceProfile>, ResourceManagerException> allocateResourceFunction,
		@Nonnull Consumer<Tuple3<JobID, AllocationID, Exception>> notifyAllocationFailureConsumer) {
	this.releaseResourceConsumer = releaseResourceConsumer;
	this.allocateResourceFunction = allocateResourceFunction;
	this.notifyAllocationFailureConsumer = notifyAllocationFailureConsumer;
}
 
Example #12
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private void registerSlotRequest(
		TestingYarnResourceManager resourceManager,
		MockResourceManagerRuntimeServices rmServices,
		ResourceProfile resourceProfile,
		String taskHost) throws ExecutionException, InterruptedException {

	CompletableFuture<?> registerSlotRequestFuture = resourceManager.runInMainThread(() -> {
		rmServices.slotManager.registerSlotRequest(
			new SlotRequest(new JobID(), new AllocationID(), resourceProfile, taskHost));
		return null;
	});

	// wait for the registerSlotRequest completion
	registerSlotRequestFuture.get();
}
 
Example #13
Source File: AllocatedSlotsTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) {
	return new AllocatedSlot(
		allocationId,
		taskManagerLocation,
		0,
		ResourceProfile.ANY,
		new SimpleAckingTaskManagerGateway());
}
 
Example #14
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private Predicate<PendingRequest> canBeFulfilledWithAllocatedSlot(Set<ResourceProfile> allocatedResourceProfiles) {
	return pendingRequest -> {
		for (ResourceProfile allocatedResourceProfile : allocatedResourceProfiles) {
			if (allocatedResourceProfile.isMatching(pendingRequest.getResourceProfile())) {
				return true;
			}
		}

		return false;
	};
}
 
Example #15
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that a batch slot request won't time out if there exists a slot in the
 * SlotPool which fulfills the requested {@link ResourceProfile}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotTimeoutIfFulfillingSlotExists() throws Exception {
	final Time batchSlotTimeout = Time.milliseconds(2L);
	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();
	final ManualClock clock = new ManualClock();

	try (final TestingSlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
			.setClock(clock)
			.setBatchSlotTimeout(batchSlotTimeout)
			.build()) {

		SlotPoolUtils.offerSlots(slotPool, directMainThreadExecutor, Collections.singletonList(resourceProfile));

		final CompletableFuture<PhysicalSlot> firstSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);
		final CompletableFuture<PhysicalSlot> secondSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, ResourceProfile.UNKNOWN);
		final CompletableFuture<PhysicalSlot> thirdSlotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, smallerResourceProfile);

		final List<CompletableFuture<PhysicalSlot>> slotFutures = Arrays.asList(firstSlotFuture, secondSlotFuture, thirdSlotFuture);
		advanceTimeAndTriggerCheckBatchSlotTimeout(slotPool, clock, batchSlotTimeout);

		for (CompletableFuture<PhysicalSlot> slotFuture : slotFutures) {
			assertThat(slotFuture.isDone(), is(false));
		}
	}

}
 
Example #16
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void checkBatchSlotTimeout() {
	final Collection<PendingRequest> pendingBatchRequests = getPendingBatchRequests();

	if (!pendingBatchRequests.isEmpty()) {
		final Set<ResourceProfile> allocatedResourceProfiles = getAllocatedResourceProfiles();

		final Map<Boolean, List<PendingRequest>> fulfillableAndUnfulfillableRequests = pendingBatchRequests
			.stream()
			.collect(Collectors.partitioningBy(canBeFulfilledWithAllocatedSlot(allocatedResourceProfiles)));

		final List<PendingRequest> fulfillableRequests = fulfillableAndUnfulfillableRequests.get(true);
		final List<PendingRequest> unfulfillableRequests = fulfillableAndUnfulfillableRequests.get(false);

		final long currentTimestamp = clock.relativeTimeMillis();

		for (PendingRequest fulfillableRequest : fulfillableRequests) {
			fulfillableRequest.markFulfillable();
		}

		for (PendingRequest unfulfillableRequest : unfulfillableRequests) {
			unfulfillableRequest.markUnfulfillable(currentTimestamp);

			if (unfulfillableRequest.getUnfulfillableSince() + batchSlotTimeout.toMilliseconds() <= currentTimestamp) {
				timeoutPendingSlotRequest(unfulfillableRequest.getSlotRequestId());
			}
		}
	}

	scheduleRunAsync(this::checkBatchSlotTimeout, batchSlotTimeout);
}
 
Example #17
Source File: TaskExecutorTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testDynamicSlotAllocation() throws Exception {
	final AllocationID allocationId = new AllocationID();
	try (TaskExecutorTestingContext submissionContext = createTaskExecutorTestingContext(2)) {
		submissionContext.start();
		final CompletableFuture<Tuple3<ResourceID, InstanceID, SlotReport>> initialSlotReportFuture =
			new CompletableFuture<>();
		ResourceManagerId resourceManagerId = createAndRegisterResourceManager(initialSlotReportFuture);
		initialSlotReportFuture.get();
		final ResourceProfile resourceProfile = DEFAULT_RESOURCE_PROFILE
			.merge(ResourceProfile.newBuilder().setCpuCores(0.1).build());

		submissionContext.taskExecutor
			.getSelfGateway(TaskExecutorGateway.class)
			.requestSlot(
				SlotID.generateDynamicSlotID(ResourceID.generate()),
				jobId,
				allocationId,
				resourceProfile,
				submissionContext.jobMasterGateway.getAddress(),
				resourceManagerId,
				timeout)
			.get();

		ResourceID resourceId = ResourceID.generate();
		SlotReport slotReport = submissionContext.taskSlotTable.createSlotReport(resourceId);
		assertThat(slotReport, containsInAnyOrder(
			new SlotStatus(new SlotID(resourceId, 0), DEFAULT_RESOURCE_PROFILE),
			new SlotStatus(new SlotID(resourceId, 1), DEFAULT_RESOURCE_PROFILE),
			new SlotStatus(SlotID.generateDynamicSlotID(resourceId), resourceProfile, jobId, allocationId)));
	}
}
 
Example #18
Source File: JobMasterTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public CompletableFuture<PhysicalSlot> requestNewAllocatedSlot(
		@Nonnull SlotRequestId slotRequestId,
		@Nonnull ResourceProfile resourceProfile,
		@Nullable Time timeout) {
	return new CompletableFuture<>();
}
 
Example #19
Source File: SlotSharingManagerTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that slot context future failures will release the root slot
 */
@Test
public void testSlotContextFutureFailure() {
	final TestingAllocatedSlotActions allocatedSlotActions = new TestingAllocatedSlotActions();

	SlotSharingManager slotSharingManager = new SlotSharingManager(
		SLOT_SHARING_GROUP_ID,
		allocatedSlotActions,
		SLOT_OWNER);

	CompletableFuture<SlotContext> slotContextFuture = new CompletableFuture<>();

	assertTrue(slotSharingManager.isEmpty());

	SlotSharingManager.MultiTaskSlot rootSlot = slotSharingManager.createRootSlot(
		new SlotRequestId(),
		slotContextFuture,
		new SlotRequestId());

	SlotSharingManager.SingleTaskSlot singleTaskSlot = rootSlot.allocateSingleTaskSlot(
		new SlotRequestId(),
		ResourceProfile.UNKNOWN,
		new AbstractID(),
		Locality.LOCAL);

	slotContextFuture.completeExceptionally(new FlinkException("Test exception"));

	assertTrue(singleTaskSlot.getLogicalSlotFuture().isCompletedExceptionally());
	assertTrue(slotSharingManager.isEmpty());
	assertTrue(slotSharingManager.getResolvedRootSlots().isEmpty());
	assertTrue(slotSharingManager.getUnresolvedRootSlots().isEmpty());
}
 
Example #20
Source File: SlotManagerImplTest.java    From flink with Apache License 2.0 5 votes vote down vote up
private SlotReport createSingleAllocatedSlotReport(ResourceID resourceID, JobID jobId) {
	return createSlotReport(
		resourceID,
		1,
		ResourceProfile.ANY,
		(slotId, resourceProfile) -> new SlotStatus(slotId, resourceProfile, jobId, new AllocationID()));
}
 
Example #21
Source File: TaskExecutorRegistration.java    From flink with Apache License 2.0 5 votes vote down vote up
public TaskExecutorRegistration(
		final String taskExecutorAddress,
		final ResourceID resourceId,
		final int dataPort,
		final HardwareDescription hardwareDescription,
		final ResourceProfile defaultSlotResourceProfile,
		final ResourceProfile totalResourceProfile) {
	this.taskExecutorAddress = checkNotNull(taskExecutorAddress);
	this.resourceId = checkNotNull(resourceId);
	this.dataPort = dataPort;
	this.hardwareDescription = checkNotNull(hardwareDescription);
	this.defaultSlotResourceProfile = checkNotNull(defaultSlotResourceProfile);
	this.totalResourceProfile = checkNotNull(totalResourceProfile);
}
 
Example #22
Source File: TaskSlotTableTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that one can can mark allocated slots as active.
 */
@Test
public void testTryMarkSlotActive() throws SlotNotFoundException {
	final TaskSlotTable taskSlotTable = createTaskSlotTable(Collections.nCopies(3, ResourceProfile.UNKNOWN));

	try {
		taskSlotTable.start(new TestingSlotActionsBuilder().build());

		final JobID jobId1 = new JobID();
		final AllocationID allocationId1 = new AllocationID();
		taskSlotTable.allocateSlot(0, jobId1, allocationId1, SLOT_TIMEOUT);
		final AllocationID allocationId2 = new AllocationID();
		taskSlotTable.allocateSlot(1, jobId1, allocationId2, SLOT_TIMEOUT);
		final AllocationID allocationId3 = new AllocationID();
		final JobID jobId2 = new JobID();
		taskSlotTable.allocateSlot(2, jobId2, allocationId3, SLOT_TIMEOUT);

		taskSlotTable.markSlotActive(allocationId1);

		assertThat(taskSlotTable.isAllocated(0, jobId1, allocationId1), is(true));
		assertThat(taskSlotTable.isAllocated(1, jobId1, allocationId2), is(true));
		assertThat(taskSlotTable.isAllocated(2, jobId2, allocationId3), is(true));

		assertThat(IteratorUtils.toList(taskSlotTable.getActiveSlots(jobId1)), is(equalTo(Arrays.asList(allocationId1))));

		assertThat(taskSlotTable.tryMarkSlotActive(jobId1, allocationId1), is(true));
		assertThat(taskSlotTable.tryMarkSlotActive(jobId1, allocationId2), is(true));
		assertThat(taskSlotTable.tryMarkSlotActive(jobId1, allocationId3), is(false));

		assertThat(Sets.newHashSet(taskSlotTable.getActiveSlots(jobId1)), is(equalTo(new HashSet<>(Arrays.asList(allocationId2, allocationId1)))));
	} finally {
		taskSlotTable.stop();
	}
}
 
Example #23
Source File: SlotManagerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private Optional<PendingTaskManagerSlot> findFreeMatchingPendingTaskManagerSlot(ResourceProfile requiredResourceProfile) {
	for (PendingTaskManagerSlot pendingTaskManagerSlot : pendingSlots.values()) {
		if (pendingTaskManagerSlot.getAssignedPendingSlotRequest() == null && pendingTaskManagerSlot.getResourceProfile().isMatching(requiredResourceProfile)) {
			return Optional.of(pendingTaskManagerSlot);
		}
	}

	return Optional.empty();
}
 
Example #24
Source File: PhysicalSlotRequestBulkChecker.java    From flink with Apache License 2.0 5 votes vote down vote up
private static boolean areRequestsFulfillableWithSlots(
		final Collection<ResourceProfile> requestResourceProfiles,
		final Set<SlotInfo> slots) {

	final Set<SlotInfo> remainingSlots = new HashSet<>(slots);
	for (ResourceProfile requestResourceProfile : requestResourceProfiles) {
		final Optional<SlotInfo> matchedSlot = findMatchingSlotForRequest(requestResourceProfile, remainingSlots);
		if (matchedSlot.isPresent()) {
			remainingSlots.remove(matchedSlot.get());
		} else {
			return false;
		}
	}
	return true;
}
 
Example #25
Source File: SlotManager.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
private Optional<PendingTaskManagerSlot> findFreeMatchingPendingTaskManagerSlot(ResourceProfile requiredResourceProfile) {
	for (PendingTaskManagerSlot pendingTaskManagerSlot : pendingSlots.values()) {
		if (pendingTaskManagerSlot.getAssignedPendingSlotRequest() == null && pendingTaskManagerSlot.getResourceProfile().isMatching(requiredResourceProfile)) {
			return Optional.of(pendingTaskManagerSlot);
		}
	}

	return Optional.empty();
}
 
Example #26
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
protected void checkBatchSlotTimeout() {
	if (!batchSlotRequestTimeoutCheckEnabled) {
		return;
	}

	final Collection<PendingRequest> pendingBatchRequests = getPendingBatchRequests();

	if (!pendingBatchRequests.isEmpty()) {
		final Set<ResourceProfile> allocatedResourceProfiles = getAllocatedResourceProfiles();

		final Map<Boolean, List<PendingRequest>> fulfillableAndUnfulfillableRequests = pendingBatchRequests
			.stream()
			.collect(Collectors.partitioningBy(canBeFulfilledWithAllocatedSlot(allocatedResourceProfiles)));

		final List<PendingRequest> fulfillableRequests = fulfillableAndUnfulfillableRequests.get(true);
		final List<PendingRequest> unfulfillableRequests = fulfillableAndUnfulfillableRequests.get(false);

		final long currentTimestamp = clock.relativeTimeMillis();

		for (PendingRequest fulfillableRequest : fulfillableRequests) {
			fulfillableRequest.markFulfillable();
		}

		for (PendingRequest unfulfillableRequest : unfulfillableRequests) {
			unfulfillableRequest.markUnfulfillable(currentTimestamp);

			if (unfulfillableRequest.getUnfulfillableSince() + batchSlotTimeout.toMilliseconds() <= currentTimestamp) {
				timeoutPendingSlotRequest(unfulfillableRequest.getSlotRequestId());
			}
		}
	}

	scheduleRunAsync(this::checkBatchSlotTimeout, batchSlotTimeout);
}
 
Example #27
Source File: SlotManagerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
private boolean isFulfillableByRegisteredSlots(ResourceProfile resourceProfile) {
	for (TaskManagerSlot slot : slots.values()) {
		if (slot.getResourceProfile().isMatching(resourceProfile)) {
			return true;
		}
	}
	return false;
}
 
Example #28
Source File: SlotPoolUtils.java    From flink with Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles,
		TaskManagerGateway taskManagerGateway) {
	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	CompletableFuture.runAsync(
		() -> {
			slotPool.registerTaskManager(taskManagerLocation.getResourceID());

			final Collection<SlotOffer> slotOffers = IntStream
				.range(0, resourceProfiles.size())
				.mapToObj(i -> new SlotOffer(new AllocationID(), i, resourceProfiles.get(i)))
				.collect(Collectors.toList());

			final Collection<SlotOffer> acceptedOffers = slotPool.offerSlots(
				taskManagerLocation,
				taskManagerGateway,
				slotOffers);

			assertThat(acceptedOffers, is(slotOffers));
		},
		mainThreadExecutor
	).join();

	return taskManagerLocation.getResourceID();
}
 
Example #29
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.ANY));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example #30
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public CompletableFuture<PhysicalSlot> requestNewAllocatedSlot(
		@Nonnull SlotRequestId slotRequestId,
		@Nonnull ResourceProfile resourceProfile,
		@Nullable Time timeout) {

	componentMainThreadExecutor.assertRunningInMainThread();

	final PendingRequest pendingRequest = PendingRequest.createStreamingRequest(slotRequestId, resourceProfile);

	if (timeout != null) {
		// register request timeout
		FutureUtils
			.orTimeout(
				pendingRequest.getAllocatedSlotFuture(),
				timeout.toMilliseconds(),
				TimeUnit.MILLISECONDS,
				componentMainThreadExecutor)
			.whenComplete(
				(AllocatedSlot ignored, Throwable throwable) -> {
					if (throwable instanceof TimeoutException) {
						timeoutPendingSlotRequest(slotRequestId);
					}
				});
	}

	return requestNewAllocatedSlotInternal(pendingRequest)
		.thenApply((Function.identity()));
}