Java Code Examples for org.apache.flink.runtime.taskexecutor.slot.SlotOffer

The following examples show how to use org.apache.flink.runtime.taskexecutor.slot.SlotOffer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Flink-CEPplus   Source File: SlotPoolImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<SlotOffer> offerSlots(
		TaskManagerLocation taskManagerLocation,
		TaskManagerGateway taskManagerGateway,
		Collection<SlotOffer> offers) {

	ArrayList<SlotOffer> result = new ArrayList<>(offers.size());

	for (SlotOffer offer : offers) {
		if (offerSlot(
			taskManagerLocation,
			taskManagerGateway,
			offer)) {

			result.add(offer);
		}
	}

	return result;
}
 
Example 2
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<SlotOffer> offerSlots(TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) {
	hasReceivedSlotOffers.trigger();
	final Collection<SlotInfo> slotInfos = Optional.ofNullable(registeredSlots.get(taskManagerLocation.getResourceID()))
		.orElseThrow(() -> new FlinkRuntimeException("TaskManager not registered."));

	int slotIndex = slotInfos.size();

	for (SlotOffer offer : offers) {
		slotInfos.add(new SimpleSlotContext(
			offer.getAllocationId(),
			taskManagerLocation,
			slotIndex,
			taskManagerGateway));
		slotIndex++;
	}

	return offers;
}
 
Example 3
Source Project: flink   Source File: SlotPoolImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<SlotOffer> offerSlots(
		TaskManagerLocation taskManagerLocation,
		TaskManagerGateway taskManagerGateway,
		Collection<SlotOffer> offers) {

	ArrayList<SlotOffer> result = new ArrayList<>(offers.size());

	for (SlotOffer offer : offers) {
		if (offerSlot(
			taskManagerLocation,
			taskManagerGateway,
			offer)) {

			result.add(offer);
		}
	}

	return result;
}
 
Example 4
Source Project: flink   Source File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<SlotOffer>> offerSlots(
		final ResourceID taskManagerId,
		final Collection<SlotOffer> slots,
		final Time timeout) {

	Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);

	if (taskManager == null) {
		return FutureUtils.completedExceptionally(new Exception("Unknown TaskManager " + taskManagerId));
	}

	final TaskManagerLocation taskManagerLocation = taskManager.f0;
	final TaskExecutorGateway taskExecutorGateway = taskManager.f1;

	final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, getFencingToken());

	return CompletableFuture.completedFuture(
		slotPool.offerSlots(
			taskManagerLocation,
			rpcTaskManagerGateway,
			slots));
}
 
Example 5
Source Project: flink   Source File: ExecutionGraphRestartTest.java    License: Apache License 2.0 6 votes vote down vote up
private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 
Example 6
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 6 votes vote down vote up
private List<AllocationID> registerAndOfferSlots(TaskManagerLocation taskManagerLocation, SlotPoolImpl slotPool, int numberOfSlotsToRegister) {
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());
	final List<AllocationID> allocationIds = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(ignored -> new AllocationID())
		.collect(Collectors.toList());

	Collection<SlotOffer> slotOffers = IntStream.range(0, numberOfSlotsToRegister)
		.mapToObj(index -> new SlotOffer(allocationIds.get(index), index, ResourceProfile.ANY))
		.collect(Collectors.toList());

	slotPool.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		slotOffers);

	return allocationIds;
}
 
Example 7
Source Project: flink   Source File: JobMasterPartitionReleaseTest.java    License: Apache License 2.0 6 votes vote down vote up
private void registerTaskExecutorAtJobMaster(
		TestingRpcService rpcService,
		JobMasterGateway jobMasterGateway,
		TaskExecutorGateway taskExecutorGateway,
		SettableLeaderRetrievalService rmLeaderRetrievalService) throws ExecutionException, InterruptedException {

	final AllocationIdsResourceManagerGateway resourceManagerGateway = new AllocationIdsResourceManagerGateway();
	rpcService.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
	rmLeaderRetrievalService.notifyListener(resourceManagerGateway.getAddress(), resourceManagerGateway.getFencingToken().toUUID());

	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	jobMasterGateway.registerTaskManager(taskExecutorGateway.getAddress(), localTaskManagerUnresolvedLocation, testingTimeout).get();

	final AllocationID allocationId = resourceManagerGateway.takeAllocationId();
	Collection<SlotOffer> slotOffers = Collections.singleton(new SlotOffer(allocationId, 0, ResourceProfile.UNKNOWN));

	jobMasterGateway.offerSlots(localTaskManagerUnresolvedLocation.getResourceID(), slotOffers, testingTimeout).get();
}
 
Example 8
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<SlotOffer> offerSlots(TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) {
	hasReceivedSlotOffers.trigger();
	final Collection<SlotInfo> slotInfos = Optional.ofNullable(registeredSlots.get(taskManagerLocation.getResourceID()))
		.orElseThrow(() -> new FlinkRuntimeException("TaskManager not registered."));

	int slotIndex = slotInfos.size();

	for (SlotOffer offer : offers) {
		slotInfos.add(new SimpleSlotContext(
			offer.getAllocationId(),
			taskManagerLocation,
			slotIndex,
			taskManagerGateway));
		slotIndex++;
	}

	return offers;
}
 
Example 9
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Collection<SlotOffer> offerSlots(TaskManagerLocation taskManagerLocation, TaskManagerGateway taskManagerGateway, Collection<SlotOffer> offers) {
	hasReceivedSlotOffers.trigger();
	final Collection<SlotInfo> slotInfos = Optional.ofNullable(registeredSlots.get(taskManagerLocation.getResourceID()))
		.orElseThrow(() -> new FlinkRuntimeException("TaskManager not registered."));

	int slotIndex = slotInfos.size();

	for (SlotOffer offer : offers) {
		slotInfos.add(new SimpleSlotContext(
			offer.getAllocationId(),
			taskManagerLocation,
			slotIndex,
			taskManagerGateway));
		slotIndex++;
	}

	return offers;
}
 
Example 10
Source Project: flink   Source File: JobMaster.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public CompletableFuture<Collection<SlotOffer>> offerSlots(
		final ResourceID taskManagerId,
		final Collection<SlotOffer> slots,
		final Time timeout) {

	Tuple2<TaskManagerLocation, TaskExecutorGateway> taskManager = registeredTaskManagers.get(taskManagerId);

	if (taskManager == null) {
		return FutureUtils.completedExceptionally(new Exception("Unknown TaskManager " + taskManagerId));
	}

	final TaskManagerLocation taskManagerLocation = taskManager.f0;
	final TaskExecutorGateway taskExecutorGateway = taskManager.f1;

	final RpcTaskManagerGateway rpcTaskManagerGateway = new RpcTaskManagerGateway(taskExecutorGateway, getFencingToken());

	return CompletableFuture.completedFuture(
		slotPool.offerSlots(
			taskManagerLocation,
			rpcTaskManagerGateway,
			slots));
}
 
Example 11
Source Project: Flink-CEPplus   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void offerSlotsToJobManager(final JobID jobId) {
	final JobManagerConnection jobManagerConnection = jobManagerTable.get(jobId);

	if (jobManagerConnection == null) {
		log.debug("There is no job manager connection to the leader of job {}.", jobId);
	} else {
		if (taskSlotTable.hasAllocatedSlots(jobId)) {
			log.info("Offer reserved slots to the leader of job {}.", jobId);

			final JobMasterGateway jobMasterGateway = jobManagerConnection.getJobManagerGateway();

			final Iterator<TaskSlot> reservedSlotsIterator = taskSlotTable.getAllocatedSlots(jobId);
			final JobMasterId jobMasterId = jobManagerConnection.getJobMasterId();

			final Collection<SlotOffer> reservedSlots = new HashSet<>(2);

			while (reservedSlotsIterator.hasNext()) {
				SlotOffer offer = reservedSlotsIterator.next().generateSlotOffer();
				reservedSlots.add(offer);
			}

			CompletableFuture<Collection<SlotOffer>> acceptedSlotsFuture = jobMasterGateway.offerSlots(
				getResourceID(),
				reservedSlots,
				taskManagerConfiguration.getTimeout());

			acceptedSlotsFuture.whenCompleteAsync(
				handleAcceptedSlotOffers(jobId, jobMasterGateway, jobMasterId, reservedSlots),
				getMainThreadExecutor());
		} else {
			log.debug("There are no unassigned slots for the job {}.", jobId);
		}
	}
}
 
Example 12
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAllocateSimpleSlot() throws Exception {
	CompletableFuture<SlotRequest> slotRequestFuture = new CompletableFuture<>();
	resourceManagerGateway.setRequestSlotConsumer(slotRequestFuture::complete);

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = scheduler.allocateSlot(
			requestId,
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			timeout);
		assertFalse(future.isDone());

		final SlotRequest slotRequest = slotRequestFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final SlotOffer slotOffer = new SlotOffer(
			slotRequest.getAllocationId(),
			0,
			DEFAULT_TESTING_PROFILE);

		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));

		LogicalSlot slot = future.get(1, TimeUnit.SECONDS);
		assertTrue(future.isDone());
		assertTrue(slot.isAlive());
		assertEquals(taskManagerLocation, slot.getTaskManagerLocation());
	}
}
 
Example 13
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.ANY));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.ANY));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}
 
Example 14
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
private Collection<SlotOffer> registerSlotsAtJobMaster(
	int numberSlots,
	JobMasterGateway jobMasterGateway,
	TaskExecutorGateway taskExecutorGateway) throws ExecutionException, InterruptedException {
	return registerSlotsAtJobMaster(
		numberSlots,
		jobMasterGateway,
		taskExecutorGateway,
		new LocalUnresolvedTaskManagerLocation());
}
 
Example 15
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}
 
Example 16
Source Project: Flink-CEPplus   Source File: SlotPoolSlotSharingTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example 17
Source Project: Flink-CEPplus   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
private Collection<SlotOffer> registerSlotsAtJobMaster(
	int numberSlots,
	JobMasterGateway jobMasterGateway,
	TaskExecutorGateway taskExecutorGateway) throws ExecutionException, InterruptedException {
	return registerSlotsAtJobMaster(
		numberSlots,
		jobMasterGateway,
		taskExecutorGateway,
		new LocalTaskManagerLocation());
}
 
Example 18
Source Project: flink   Source File: SlotPoolPendingRequestFailureTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFailingAllocationFailsRemappedPendingSlotRequests() throws Exception {
	final List<AllocationID> allocations = new ArrayList<>();
	resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocations.add(slotRequest.getAllocationId()));

	try (SlotPoolImpl slotPool = setUpSlotPool()) {
		final CompletableFuture<PhysicalSlot> slotFuture1 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
		final CompletableFuture<PhysicalSlot> slotFuture2 = requestNewAllocatedSlot(slotPool, new SlotRequestId());

		final AllocationID allocationId1 = allocations.get(0);
		final AllocationID allocationId2 = allocations.get(1);

		final TaskManagerLocation location = new LocalTaskManagerLocation();
		final SlotOffer slotOffer = new SlotOffer(allocationId2, 0, ResourceProfile.ANY);
		slotPool.registerTaskManager(location.getResourceID());
		slotPool.offerSlot(location, new SimpleAckingTaskManagerGateway(), slotOffer);

		assertThat(slotFuture1.isDone(), is(true));
		assertThat(slotFuture2.isDone(), is(false));

		final FlinkException cause = new FlinkException("Fail pending slot request failure.");
		final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId1, cause);

		assertThat(responseFuture.isPresent(), is(false));

		try {
			slotFuture2.getNow(null);
			fail("Expected a slot allocation failure.");
		} catch (Throwable t) {
			assertThat(ExceptionUtils.stripCompletionException(t), equalTo(cause));
		}
	}
}
 
Example 19
Source Project: flink   Source File: SlotPoolUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles,
		TaskManagerGateway taskManagerGateway) {
	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	CompletableFuture.runAsync(
		() -> {
			slotPool.registerTaskManager(taskManagerLocation.getResourceID());

			final Collection<SlotOffer> slotOffers = IntStream
				.range(0, resourceProfiles.size())
				.mapToObj(i -> new SlotOffer(new AllocationID(), i, resourceProfiles.get(i)))
				.collect(Collectors.toList());

			final Collection<SlotOffer> acceptedOffers = slotPool.offerSlots(
				taskManagerLocation,
				taskManagerGateway,
				slotOffers);

			assertThat(acceptedOffers, is(slotOffers));
		},
		mainThreadExecutor
	).join();

	return taskManagerLocation.getResourceID();
}
 
Example 20
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}
 
Example 21
Source Project: flink   Source File: JobMasterTest.java    License: Apache License 2.0 5 votes vote down vote up
private Collection<SlotOffer> registerSlotsAtJobMaster(
	int numberSlots,
	JobMasterGateway jobMasterGateway,
	TaskExecutorGateway taskExecutorGateway) throws ExecutionException, InterruptedException {
	return registerSlotsAtJobMaster(
		numberSlots,
		jobMasterGateway,
		taskExecutorGateway,
		new LocalTaskManagerLocation());
}
 
Example 22
Source Project: flink   Source File: TaskExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void internalOfferSlotsToJobManager(JobTable.Connection jobManagerConnection) {
	final JobID jobId = jobManagerConnection.getJobId();

	if (taskSlotTable.hasAllocatedSlots(jobId)) {
		log.info("Offer reserved slots to the leader of job {}.", jobId);

		final JobMasterGateway jobMasterGateway = jobManagerConnection.getJobManagerGateway();

		final Iterator<TaskSlot<Task>> reservedSlotsIterator = taskSlotTable.getAllocatedSlots(jobId);
		final JobMasterId jobMasterId = jobManagerConnection.getJobMasterId();

		final Collection<SlotOffer> reservedSlots = new HashSet<>(2);

		while (reservedSlotsIterator.hasNext()) {
			SlotOffer offer = reservedSlotsIterator.next().generateSlotOffer();
			reservedSlots.add(offer);
		}

		CompletableFuture<Collection<SlotOffer>> acceptedSlotsFuture = jobMasterGateway.offerSlots(
			getResourceID(),
			reservedSlots,
			taskManagerConfiguration.getTimeout());

		acceptedSlotsFuture.whenCompleteAsync(
			handleAcceptedSlotOffers(jobId, jobMasterGateway, jobMasterId, reservedSlots),
			getMainThreadExecutor());
	} else {
		log.debug("There are no unassigned slots for the job {}.", jobId);
	}
}
 
Example 23
Source Project: flink   Source File: SlotPoolUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceID offerSlots(
		SlotPoolImpl slotPool,
		ComponentMainThreadExecutor mainThreadExecutor,
		List<ResourceProfile> resourceProfiles,
		TaskManagerGateway taskManagerGateway) {
	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	CompletableFuture.runAsync(
		() -> {
			slotPool.registerTaskManager(taskManagerLocation.getResourceID());

			final Collection<SlotOffer> slotOffers = IntStream
				.range(0, resourceProfiles.size())
				.mapToObj(i -> new SlotOffer(new AllocationID(), i, resourceProfiles.get(i)))
				.collect(Collectors.toList());

			final Collection<SlotOffer> acceptedOffers = slotPool.offerSlots(
				taskManagerLocation,
				taskManagerGateway,
				slotOffers);

			assertThat(acceptedOffers, is(slotOffers));
		},
		mainThreadExecutor
	).join();

	return taskManagerLocation.getResourceID();
}
 
Example 24
Source Project: Flink-CEPplus   Source File: SlotPoolImpl.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// check whether we have request waiting for this slot
	PendingRequest pendingRequest = pendingRequests.removeKeyB(allocationID);
	if (pendingRequest != null) {
		// we were waiting for this!
		allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot);

		if (!pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot)) {
			// we could not complete the pending slot future --> try to fulfill another pending request
			allocatedSlots.remove(pendingRequest.getSlotRequestId());
			tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
		} else {
			log.debug("Fulfilled slot request [{}] with allocated slot [{}].", pendingRequest.getSlotRequestId(), allocationID);
		}
	}
	else {
		// we were actually not waiting for this:
		//   - could be that this request had been fulfilled
		//   - we are receiving the slots from TaskManagers after becoming leaders
		tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
	}

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 
Example 25
Source Project: flink   Source File: SlotPoolImplTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testAllocateWithFreeSlot() throws Exception {
	final CompletableFuture<SlotRequest> slotRequestFuture = new CompletableFuture<>();
	resourceManagerGateway.setRequestSlotConsumer(slotRequestFuture::complete);

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		CompletableFuture<LogicalSlot> future1 = scheduler.allocateSlot(
			new SlotRequestId(),
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			timeout);
		assertFalse(future1.isDone());

		final SlotRequest slotRequest = slotRequestFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final SlotOffer slotOffer = new SlotOffer(
			slotRequest.getAllocationId(),
			0,
			DEFAULT_TESTING_PROFILE);

		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));

		LogicalSlot slot1 = future1.get(1, TimeUnit.SECONDS);
		assertTrue(future1.isDone());

		// return this slot to pool
		slot1.releaseSlot();

		CompletableFuture<LogicalSlot> future2 = scheduler.allocateSlot(
			new SlotRequestId(),
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			timeout);

		// second allocation fulfilled by previous slot returning
		LogicalSlot slot2 = future2.get(1, TimeUnit.SECONDS);
		assertTrue(future2.isDone());

		assertNotEquals(slot1, slot2);
		assertFalse(slot1.isAlive());
		assertTrue(slot2.isAlive());
		assertEquals(slot1.getTaskManagerLocation(), slot2.getTaskManagerLocation());
		assertEquals(slot1.getPhysicalSlotNumber(), slot2.getPhysicalSlotNumber());
	}
}
 
Example 26
Source Project: flink   Source File: SlotPoolInteractionsTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that extra slots are kept by the {@link SlotPoolImpl}.
 */
@Test
public void testExtraSlotsAreKept() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();

		TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		pool.connectToResourceManager(resourceManagerGateway);

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(getExecution()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());

		AllocationID allocationId = allocationIdFuture.get();
		final SlotOffer slotOffer = new SlotOffer(
			allocationId,
			0,
			DEFAULT_TESTING_PROFILE);
		final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

		testMainThreadExecutor.execute(() -> pool.registerTaskManager(taskManagerLocation.getResourceID()));

		assertTrue(testMainThreadExecutor.execute(() -> pool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer)));

		assertTrue(pool.containsAvailableSlot(allocationId));
	}
}
 
Example 27
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testOfferSlot() throws Exception {
	final CompletableFuture<SlotRequest> slotRequestFuture = new CompletableFuture<>();

	resourceManagerGateway.setRequestSlotConsumer(slotRequestFuture::complete);

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		CompletableFuture<LogicalSlot> future = scheduler.allocateSlot(
			new SlotRequestId(),
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			timeout);
		assertFalse(future.isDone());

		final SlotRequest slotRequest = slotRequestFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final SlotOffer slotOffer = new SlotOffer(
			slotRequest.getAllocationId(),
			0,
			DEFAULT_TESTING_PROFILE);

		final TaskManagerLocation invalidTaskManagerLocation = new LocalTaskManagerLocation();

		// slot from unregistered resource
		assertFalse(slotPool.offerSlot(invalidTaskManagerLocation, taskManagerGateway, slotOffer));

		final SlotOffer nonRequestedSlotOffer = new SlotOffer(
			new AllocationID(),
			0,
			DEFAULT_TESTING_PROFILE);

		// we'll also accept non requested slots
		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, nonRequestedSlotOffer));

		// accepted slot
		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));
		LogicalSlot slot = future.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);
		assertTrue(slot.isAlive());

		// duplicated offer with using slot
		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));
		assertTrue(slot.isAlive());

		final SlotOffer anotherSlotOfferWithSameAllocationId = new SlotOffer(
			slotRequest.getAllocationId(),
			1,
			DEFAULT_TESTING_PROFILE);
		assertFalse(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, anotherSlotOfferWithSameAllocationId));

		TaskManagerLocation anotherTaskManagerLocation = new LocalTaskManagerLocation();
		assertFalse(slotPool.offerSlot(anotherTaskManagerLocation, taskManagerGateway, slotOffer));

		// duplicated offer with free slot
		slot.releaseSlot();
		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));
		assertFalse(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, anotherSlotOfferWithSameAllocationId));
		assertFalse(slotPool.offerSlot(anotherTaskManagerLocation, taskManagerGateway, slotOffer));
	}
}
 
Example 28
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testReleaseResource() throws Exception {
	final CompletableFuture<SlotRequest> slotRequestFuture = new CompletableFuture<>();

	resourceManagerGateway.setRequestSlotConsumer(slotRequestFuture::complete);

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		CompletableFuture<LogicalSlot> future1 = scheduler.allocateSlot(
			new SlotRequestId(),
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			timeout);

		final SlotRequest slotRequest = slotRequestFuture.get(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		CompletableFuture<LogicalSlot> future2 = scheduler.allocateSlot(
			new SlotRequestId(),
			new DummyScheduledUnit(),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			timeout);

		final SlotOffer slotOffer = new SlotOffer(
			slotRequest.getAllocationId(),
			0,
			DEFAULT_TESTING_PROFILE);

		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));

		LogicalSlot slot1 = future1.get(1, TimeUnit.SECONDS);
		assertTrue(future1.isDone());
		assertFalse(future2.isDone());

		final CompletableFuture<?> releaseFuture = new CompletableFuture<>();
		final DummyPayload dummyPayload = new DummyPayload(releaseFuture);

		slot1.tryAssignPayload(dummyPayload);

		slotPool.releaseTaskManager(taskManagerLocation.getResourceID(), null);

		releaseFuture.get();
		assertFalse(slot1.isAlive());

		// slot released and not usable, second allocation still not fulfilled
		Thread.sleep(10);
		assertFalse(future2.isDone());
	}
}
 
Example 29
Source Project: flink   Source File: TaskExecutorTest.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Tests that accepted slots go into state assigned and the others are returned to the resource
 * manager.
 */
@Test
public void testSlotAcceptance() throws Exception {
	final InstanceID registrationId = new InstanceID();
	final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
	final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway resourceManagerGateway =
		createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);

	final AllocationID allocationId1 = new AllocationID();
	final AllocationID allocationId2 = new AllocationID();

	final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);

	final OneShotLatch offerSlotsLatch = new OneShotLatch();
	final OneShotLatch taskInTerminalState = new OneShotLatch();
	final CompletableFuture<Collection<SlotOffer>> offerResultFuture = new CompletableFuture<>();
	final TestingJobMasterGateway jobMasterGateway =
		createJobMasterWithSlotOfferAndTaskTerminationHooks(offerSlotsLatch, taskInTerminalState, offerResultFuture);

	rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
	rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);

	final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
	final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
	final TestingTaskExecutor taskManager = createTestingTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
		taskManager.waitUntilStarted();

		final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);

		// wait until registered at the RM
		taskExecutorIsRegistered.await();

		// request 2 slots for the given allocation ids
		requestSlots(
			tmGateway,
			Arrays.asList(allocationId1, allocationId2),
			resourceManagerGateway.getFencingToken(),
			jobMasterGateway.getAddress());

		// notify job leader to start slot offering
		jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());

		// wait until slots have been offered
		offerSlotsLatch.await();
		offerResultFuture.complete(Collections.singletonList(offer1));

		final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();

		final Tuple3<InstanceID, SlotID, AllocationID> expectedResult = Tuple3.of(registrationId, new SlotID(unresolvedTaskManagerLocation.getResourceID(), 1), allocationId2);

		assertThat(instanceIDSlotIDAllocationIDTuple3, equalTo(expectedResult));
		// the slot 1 can be activate for task submission
		submitNoOpInvokableTask(allocationId1, jobMasterGateway, tmGateway);
		// wait for the task completion
		taskInTerminalState.await();
		// the slot 2 can NOT be activate for task submission
		try {
			submitNoOpInvokableTask(allocationId2, jobMasterGateway, tmGateway);
			fail("It should not be possible to submit task to acquired by JM slot with index 1 (allocationId2)");
		} catch (CompletionException e) {
			assertThat(e.getCause(), instanceOf(TaskSubmissionException.class));
		}
		// the slot 2 is free to request
		tmGateway
			.requestSlot(
				new SlotID(unresolvedTaskManagerLocation.getResourceID(), 1),
				jobId,
				allocationId2,
				ResourceProfile.UNKNOWN,
				jobMasterGateway.getAddress(),
				resourceManagerGateway.getFencingToken(),
				timeout)
			.join();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 
Example 30
Source Project: Flink-CEPplus   Source File: SlotPoolImplTest.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testCheckIdleSlot() throws Exception {
	final ManualClock clock = new ManualClock();

	try (SlotPoolImpl slotPool = new SlotPoolImpl(
		jobId,
		clock,
		TestingUtils.infiniteTime(),
		timeout)) {
		final BlockingQueue<AllocationID> freedSlots = new ArrayBlockingQueue<>(1);
		taskManagerGateway.setFreeSlotFunction(
			(AllocationID allocationId, Throwable cause) -> {
				try {
					freedSlots.put(allocationId);
					return CompletableFuture.completedFuture(Acknowledge.get());
				} catch (InterruptedException e) {
					return FutureUtils.completedExceptionally(e);
				}
			});

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);

		final AllocationID expiredSlotID = new AllocationID();
		final AllocationID freshSlotID = new AllocationID();
		final SlotOffer slotToExpire = new SlotOffer(expiredSlotID, 0, ResourceProfile.UNKNOWN);
		final SlotOffer slotToNotExpire = new SlotOffer(freshSlotID, 1, ResourceProfile.UNKNOWN);

		assertThat(slotPool.registerTaskManager(taskManagerLocation.getResourceID()),
			Matchers.is(true));

		assertThat(
			slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire),
			Matchers.is(true));

		clock.advanceTime(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToNotExpire),
			Matchers.is(true));

		clock.advanceTime(1L, TimeUnit.MILLISECONDS);

		slotPool.triggerCheckIdleSlot();

		final AllocationID freedSlot = freedSlots.poll(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(freedSlot, Matchers.is(expiredSlotID));
		assertThat(freedSlots.isEmpty(), Matchers.is(true));
	}
}