Java Code Examples for org.apache.flink.runtime.clusterframework.types.ResourceProfile#ANY

private static Scheduler createSchedulerWithSlots(
	int numSlots, SlotPool slotPool, TaskManagerLocation taskManagerLocation) throws Exception {

	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 

public SimpleSlotProvider(int numSlots, TaskManagerGateway taskManagerGateway) {
	checkArgument(numSlots >= 0, "numSlots must be >= 0");

	this.slots = new ArrayDeque<>(numSlots);

	for (int i = 0; i < numSlots; i++) {
		SimpleSlotContext as = new SimpleSlotContext(
			new AllocationID(),
			new TaskManagerLocation(ResourceID.generate(), InetAddress.getLoopbackAddress(), 10000 + i),
			0,
			taskManagerGateway,
			ResourceProfile.ANY);
		slots.add(as);
	}

	allocatedSlots = new HashMap<>(slots.size());
}
 

private static Scheduler createSchedulerWithSlots(SlotPool slotPool, TaskManagerLocation taskManagerLocation, int numSlots) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	setupSlotPool(slotPool);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

	return scheduler;
}
 

private static Scheduler createSchedulerWithSlots(
		int numSlots,
		SlotPool slotPool,
		TaskManagerLocation taskManagerLocation) throws Exception {
	final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();
	final String jobManagerAddress = "foobar";
	final ResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
	slotPool.start(JobMasterId.generate(), jobManagerAddress, mainThreadExecutor);
	slotPool.connectToResourceManager(resourceManagerGateway);
	Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), slotPool);
	scheduler.start(mainThreadExecutor);

	CompletableFuture.runAsync(() -> slotPool.registerTaskManager(taskManagerLocation.getResourceID()), mainThreadExecutor).join();

	final List<SlotOffer> slotOffers = new ArrayList<>(NUM_TASKS);
	for (int i = 0; i < numSlots; i++) {
		final AllocationID allocationId = new AllocationID();
		final SlotOffer slotOffer = new SlotOffer(allocationId, 0, ResourceProfile.ANY);
		slotOffers.add(slotOffer);
	}

	CompletableFuture.runAsync(() -> slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers), mainThreadExecutor).join();

	return scheduler;
}
 

/**
 * Tests that only free slots can fulfill/complete a pending task manager slot.
 */
@Test
public void testOnlyFreeSlotsCanFulfillPendingTaskManagerSlot() throws Exception {
	final int numberSlots = 1;
	final TestingResourceActions resourceActions = new TestingResourceActionsBuilder().build();

	try (final SlotManagerImpl slotManager = createSlotManager(ResourceManagerId.generate(), resourceActions, numberSlots)) {
		final JobID jobId = new JobID();
		assertThat(slotManager.registerSlotRequest(createSlotRequest(jobId)), is(true));

		final TaskExecutorConnection taskExecutorConnection = createTaskExecutorConnection();
		final SlotID slotId = new SlotID(taskExecutorConnection.getResourceID(), 0);
		final SlotStatus slotStatus = new SlotStatus(slotId, ResourceProfile.ANY, jobId, new AllocationID());
		final SlotReport slotReport = new SlotReport(slotStatus);

		slotManager.registerTaskManager(taskExecutorConnection, slotReport);

		assertThat(slotManager.getNumberRegisteredSlots(), is(1));
		assertThat(slotManager.getNumberPendingTaskManagerSlots(), is(numberSlots));
		assertThat(slotManager.getNumberAssignedPendingTaskManagerSlots(), is(1));
	}
}
 

private static SlotContext createSlotContext() {
	return new SimpleSlotContext(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		new SimpleAckingTaskManagerGateway(),
		ResourceProfile.ANY);
}
 

public SimpleSlotContext(
		AllocationID allocationId,
		TaskManagerLocation taskManagerLocation,
		int physicalSlotNumber,
		TaskManagerGateway taskManagerGateway) {
	this(allocationId, taskManagerLocation, physicalSlotNumber, taskManagerGateway, ResourceProfile.ANY);
}
 

@Test
public void testFailingAllocationFailsRemappedPendingSlotRequests() throws Exception {
	final List<AllocationID> allocations = new ArrayList<>();
	resourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocations.add(slotRequest.getAllocationId()));

	try (SlotPoolImpl slotPool = setUpSlotPool()) {
		final CompletableFuture<PhysicalSlot> slotFuture1 = requestNewAllocatedSlot(slotPool, new SlotRequestId());
		final CompletableFuture<PhysicalSlot> slotFuture2 = requestNewAllocatedSlot(slotPool, new SlotRequestId());

		final AllocationID allocationId1 = allocations.get(0);
		final AllocationID allocationId2 = allocations.get(1);

		final TaskManagerLocation location = new LocalTaskManagerLocation();
		final SlotOffer slotOffer = new SlotOffer(allocationId2, 0, ResourceProfile.ANY);
		slotPool.registerTaskManager(location.getResourceID());
		slotPool.offerSlot(location, new SimpleAckingTaskManagerGateway(), slotOffer);

		assertThat(slotFuture1.isDone(), is(true));
		assertThat(slotFuture2.isDone(), is(false));

		final FlinkException cause = new FlinkException("Fail pending slot request failure.");
		final Optional<ResourceID> responseFuture = slotPool.failAllocation(allocationId1, cause);

		assertThat(responseFuture.isPresent(), is(false));

		try {
			slotFuture2.getNow(null);
			fail("Expected a slot allocation failure.");
		} catch (Throwable t) {
			assertThat(ExceptionUtils.stripCompletionException(t), equalTo(cause));
		}
	}
}
 

@Override
public void returnLogicalSlot(LogicalSlot logicalSlot) {
	synchronized (lock) {
		SimpleSlotContext as = new SimpleSlotContext(
			logicalSlot.getAllocationId(),
			logicalSlot.getTaskManagerLocation(),
			logicalSlot.getPhysicalSlotNumber(),
			logicalSlot.getTaskManagerGateway(),
			ResourceProfile.ANY);

		slots.add(as);
		allocatedSlots.remove(logicalSlot.getSlotRequestId());
	}
}
 

public static PhysicalSlot createPhysicalSlot() {
	return new AllocatedSlot(
		new AllocationID(),
		new LocalTaskManagerLocation(),
		0,
		ResourceProfile.ANY,
		new SimpleAckingTaskManagerGateway());
}
 

private AllocatedSlot createSlot(final AllocationID allocationId, final TaskManagerLocation taskManagerLocation) {
	return new AllocatedSlot(
		allocationId,
		taskManagerLocation,
		0,
		ResourceProfile.ANY,
		new SimpleAckingTaskManagerGateway());
}
 

/**
 * Tests that idle slots which cannot be released will be discarded. See FLINK-11059.
 */
@Test
public void testDiscardIdleSlotIfReleasingFailed() throws Exception {
	final ManualClock clock = new ManualClock();

	try (TestingSlotPoolImpl slotPool = createSlotPoolImpl(clock)) {

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final AllocationID expiredAllocationId = new AllocationID();
		final SlotOffer slotToExpire = new SlotOffer(expiredAllocationId, 0, ResourceProfile.ANY);

		OneShotLatch freeSlotLatch = new OneShotLatch();
		taskManagerGateway.setFreeSlotFunction((AllocationID allocationId, Throwable cause) -> {
			freeSlotLatch.trigger();
			return FutureUtils.completedExceptionally(new TimeoutException("Test failure"));
		});

		assertThat(slotPool.registerTaskManager(taskManagerLocation.getResourceID()), Matchers.is(true));

		assertThat(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire), Matchers.is(true));

		clock.advanceTime(timeout.toMilliseconds() + 1, TimeUnit.MILLISECONDS);

		slotPool.triggerCheckIdleSlot();

		freeSlotLatch.await();

		CompletableFuture<LogicalSlot> allocatedSlotFuture = allocateSlot(scheduler, new SlotRequestId());

		try {
			// since the slot must have been discarded, we cannot fulfill the slot request
			allocatedSlotFuture.get(10L, TimeUnit.MILLISECONDS);
			fail("Expected to fail with a timeout.");
		} catch (TimeoutException ignored) {
			// expected
			assertEquals(0, slotPool.getAvailableSlots().size());
		}
	}
}
 

/**
 * Tests that the {@link AllocatedSlotReport} contains up to date information and not
 * stale information about the allocated slots on the {@link JobMaster}.
 *
 * <p>This is a probabilistic test case which only fails if executed repeatedly without
 * the fix for FLINK-12863.
 */
@Test
public void testAllocatedSlotReportDoesNotContainStaleInformation() throws Exception {
	final CompletableFuture<Void> assertionFuture = new CompletableFuture<>();
	final UnresolvedTaskManagerLocation unresolvedTaskManagerLocation = new LocalUnresolvedTaskManagerLocation();
	final AtomicBoolean terminateHeartbeatVerification = new AtomicBoolean(false);
	final OneShotLatch hasReceivedSlotOffers = new OneShotLatch();
	final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder()
		.setHeartbeatJobManagerConsumer((taskManagerId, allocatedSlotReport) -> {
			try {
				if (hasReceivedSlotOffers.isTriggered()) {
					assertThat(allocatedSlotReport.getAllocatedSlotInfos(), hasSize(1));
				} else {
					assertThat(allocatedSlotReport.getAllocatedSlotInfos(), empty());
				}
			} catch (AssertionError e) {
				assertionFuture.completeExceptionally(e);
			}

			if (terminateHeartbeatVerification.get()) {
				assertionFuture.complete(null);
			}
		})
		.createTestingTaskExecutorGateway();

	rpcService.registerGateway(taskExecutorGateway.getAddress(), taskExecutorGateway);

	final JobManagerSharedServices jobManagerSharedServices = new TestingJobManagerSharedServicesBuilder().build();

	final JobMaster jobMaster = new JobMasterBuilder(JobGraphTestUtils.createSingleVertexJobGraph(), rpcService)
		.withHeartbeatServices(new HeartbeatServices(5L, 1000L))
		.withSlotPoolFactory(new TestingSlotPoolFactory(hasReceivedSlotOffers))
		.createJobMaster();

	CompletableFuture<Acknowledge> startFuture = jobMaster.start(jobMasterId);

	try {
		// wait for the start to complete
		startFuture.get(testingTimeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		final JobMasterGateway jobMasterGateway = jobMaster.getSelfGateway(JobMasterGateway.class);

		// register task manager will trigger monitor heartbeat target, schedule heartbeat request at interval time
		CompletableFuture<RegistrationResponse> registrationResponse = jobMasterGateway.registerTaskManager(
			taskExecutorGateway.getAddress(),
			unresolvedTaskManagerLocation,
			testingTimeout);

		// wait for the completion of the registration
		registrationResponse.get();

		final SlotOffer slotOffer = new SlotOffer(new AllocationID(), 0, ResourceProfile.ANY);

		final CompletableFuture<Collection<SlotOffer>> slotOfferFuture = jobMasterGateway.offerSlots(unresolvedTaskManagerLocation.getResourceID(), Collections.singleton(slotOffer), testingTimeout);

		assertThat(slotOfferFuture.get(), containsInAnyOrder(slotOffer));

		terminateHeartbeatVerification.set(true);

		// make sure that no assertion has been violated
		assertionFuture.get();
	} finally {
		RpcUtils.terminateRpcEndpoint(jobMaster, testingTimeout);
		jobManagerSharedServices.shutdown();
	}
}
 

@Test
public void testCheckIdleSlot() throws Exception {
	final ManualClock clock = new ManualClock();

	try (TestingSlotPoolImpl slotPool = createSlotPoolImpl(clock)) {
		final BlockingQueue<AllocationID> freedSlots = new ArrayBlockingQueue<>(1);
		taskManagerGateway.setFreeSlotFunction(
			(AllocationID allocationId, Throwable cause) -> {
				try {
					freedSlots.put(allocationId);
					return CompletableFuture.completedFuture(Acknowledge.get());
				} catch (InterruptedException e) {
					return FutureUtils.completedExceptionally(e);
				}
			});

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);

		final AllocationID expiredSlotID = new AllocationID();
		final AllocationID freshSlotID = new AllocationID();
		final SlotOffer slotToExpire = new SlotOffer(expiredSlotID, 0, ResourceProfile.ANY);
		final SlotOffer slotToNotExpire = new SlotOffer(freshSlotID, 1, ResourceProfile.ANY);

		assertThat(slotPool.registerTaskManager(taskManagerLocation.getResourceID()),
			Matchers.is(true));

		assertThat(
			slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToExpire),
			Matchers.is(true));

		clock.advanceTime(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotToNotExpire),
			Matchers.is(true));

		clock.advanceTime(1L, TimeUnit.MILLISECONDS);

		slotPool.triggerCheckIdleSlot();

		final AllocationID freedSlot = freedSlots.poll(timeout.toMilliseconds(), TimeUnit.MILLISECONDS);

		assertThat(freedSlot, Matchers.is(expiredSlotID));
		assertThat(freedSlots.isEmpty(), Matchers.is(true));
	}
}
 

/**
 * Tests that accepted slots go into state assigned and the others are returned to the resource
 * manager.
 */
@Test
public void testSlotAcceptance() throws Exception {
	final InstanceID registrationId = new InstanceID();
	final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
	final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway resourceManagerGateway =
		createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);

	final AllocationID allocationId1 = new AllocationID();
	final AllocationID allocationId2 = new AllocationID();

	final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);

	final OneShotLatch offerSlotsLatch = new OneShotLatch();
	final OneShotLatch taskInTerminalState = new OneShotLatch();
	final CompletableFuture<Collection<SlotOffer>> offerResultFuture = new CompletableFuture<>();
	final TestingJobMasterGateway jobMasterGateway =
		createJobMasterWithSlotOfferAndTaskTerminationHooks(offerSlotsLatch, taskInTerminalState, offerResultFuture);

	rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
	rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);

	final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
	final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
	final TestingTaskExecutor taskManager = createTestingTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
		taskManager.waitUntilStarted();

		final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);

		// wait until registered at the RM
		taskExecutorIsRegistered.await();

		// request 2 slots for the given allocation ids
		requestSlots(
			tmGateway,
			Arrays.asList(allocationId1, allocationId2),
			resourceManagerGateway.getFencingToken(),
			jobMasterGateway.getAddress());

		// notify job leader to start slot offering
		jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());

		// wait until slots have been offered
		offerSlotsLatch.await();
		offerResultFuture.complete(Collections.singletonList(offer1));

		final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();

		final Tuple3<InstanceID, SlotID, AllocationID> expectedResult = Tuple3.of(registrationId, new SlotID(unresolvedTaskManagerLocation.getResourceID(), 1), allocationId2);

		assertThat(instanceIDSlotIDAllocationIDTuple3, equalTo(expectedResult));
		// the slot 1 can be activate for task submission
		submitNoOpInvokableTask(allocationId1, jobMasterGateway, tmGateway);
		// wait for the task completion
		taskInTerminalState.await();
		// the slot 2 can NOT be activate for task submission
		try {
			submitNoOpInvokableTask(allocationId2, jobMasterGateway, tmGateway);
			fail("It should not be possible to submit task to acquired by JM slot with index 1 (allocationId2)");
		} catch (CompletionException e) {
			assertThat(e.getCause(), instanceOf(TaskSubmissionException.class));
		}
		// the slot 2 is free to request
		tmGateway
			.requestSlot(
				new SlotID(unresolvedTaskManagerLocation.getResourceID(), 1),
				jobId,
				allocationId2,
				ResourceProfile.UNKNOWN,
				jobMasterGateway.getAddress(),
				resourceManagerGateway.getFencingToken(),
				timeout)
			.join();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 

/**
 * This tests task executor receive SubmitTask before OfferSlot response.
 */
@Test
public void testSubmitTaskBeforeAcceptSlot() throws Exception {
	final InstanceID registrationId = new InstanceID();
	final OneShotLatch taskExecutorIsRegistered = new OneShotLatch();
	final CompletableFuture<Tuple3<InstanceID, SlotID, AllocationID>> availableSlotFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway resourceManagerGateway =
		createRmWithTmRegisterAndNotifySlotHooks(registrationId, taskExecutorIsRegistered, availableSlotFuture);

	final AllocationID allocationId1 = new AllocationID();
	final AllocationID allocationId2 = new AllocationID();

	final SlotOffer offer1 = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);

	final OneShotLatch offerSlotsLatch = new OneShotLatch();
	final OneShotLatch taskInTerminalState = new OneShotLatch();
	final CompletableFuture<Collection<SlotOffer>> offerResultFuture = new CompletableFuture<>();
	final TestingJobMasterGateway jobMasterGateway =
		createJobMasterWithSlotOfferAndTaskTerminationHooks(offerSlotsLatch, taskInTerminalState, offerResultFuture);

	rpc.registerGateway(resourceManagerGateway.getAddress(), resourceManagerGateway);
	rpc.registerGateway(jobMasterGateway.getAddress(), jobMasterGateway);

	final TaskSlotTable<Task> taskSlotTable = TaskSlotUtils.createTaskSlotTable(2);
	final TaskManagerServices taskManagerServices = createTaskManagerServicesWithTaskSlotTable(taskSlotTable);
	final TestingTaskExecutor taskManager = createTestingTaskExecutor(taskManagerServices);

	try {
		taskManager.start();
		taskManager.waitUntilStarted();

		final TaskExecutorGateway tmGateway = taskManager.getSelfGateway(TaskExecutorGateway.class);

		// wait until registered at the RM
		taskExecutorIsRegistered.await();

		// request 2 slots for the given allocation ids
		requestSlots(
			tmGateway,
			Arrays.asList(allocationId1, allocationId2),
			resourceManagerGateway.getFencingToken(),
			jobMasterGateway.getAddress());

		// notify job leader to start slot offering
		jobManagerLeaderRetriever.notifyListener(jobMasterGateway.getAddress(), jobMasterGateway.getFencingToken().toUUID());

		// wait until slots have been offered
		offerSlotsLatch.await();

		submitNoOpInvokableTask(allocationId1, jobMasterGateway, tmGateway);

		// acknowledge the offered slots
		offerResultFuture.complete(Collections.singleton(offer1));

		// check that the rejected slot will be made available again
		final Tuple3<InstanceID, SlotID, AllocationID> instanceIDSlotIDAllocationIDTuple3 = availableSlotFuture.get();
		assertThat(instanceIDSlotIDAllocationIDTuple3.f2, equalTo(allocationId2));

		// wait for the task completion
		taskInTerminalState.await();
	} finally {
		RpcUtils.terminateRpcEndpoint(taskManager, timeout);
	}
}
 

/**
 * Test delayed registration of task executor where the delay is introduced during connection from resource manager
 * to the registering task executor.
 */
@Test
public void testDelayedRegisterTaskExecutor() throws Exception {
	final Time fastTimeout = Time.milliseconds(1L);
	try {
		final OneShotLatch startConnection = new OneShotLatch();
		final OneShotLatch finishConnection = new OneShotLatch();

		// first registration is with blocking connection
		rpcService.setRpcGatewayFutureFunction(rpcGateway ->
			CompletableFuture.supplyAsync(
				() -> {
					startConnection.trigger();
					try {
						finishConnection.await();
					} catch (InterruptedException ignored) {}
					return rpcGateway;
				},
				TestingUtils.defaultExecutor()));

		TaskExecutorRegistration taskExecutorRegistration = new TaskExecutorRegistration(
			taskExecutorGateway.getAddress(),
			taskExecutorResourceID,
			dataPort,
			hardwareDescription,
			ResourceProfile.ZERO,
			ResourceProfile.ZERO);

		CompletableFuture<RegistrationResponse> firstFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, fastTimeout);
		try {
			firstFuture.get();
			fail("Should have failed because connection to taskmanager is delayed beyond timeout");
		} catch (Exception e) {
			final Throwable cause = ExceptionUtils.stripExecutionException(e);
			assertThat(cause, instanceOf(TimeoutException.class));
			assertThat(cause.getMessage(), containsString("ResourceManagerGateway.registerTaskExecutor"));
		}

		startConnection.await();

		// second registration after timeout is with no delay, expecting it to be succeeded
		rpcService.resetRpcGatewayFutureFunction();
		CompletableFuture<RegistrationResponse> secondFuture =
			rmGateway.registerTaskExecutor(taskExecutorRegistration, TIMEOUT);
		RegistrationResponse response = secondFuture.get();
		assertTrue(response instanceof TaskExecutorRegistrationSuccess);

		// on success, send slot report for taskmanager registration
		final SlotReport slotReport = new SlotReport(new SlotStatus(new SlotID(taskExecutorResourceID, 0), ResourceProfile.ANY));
		rmGateway.sendSlotReport(taskExecutorResourceID,
			((TaskExecutorRegistrationSuccess) response).getRegistrationId(), slotReport, TIMEOUT).get();

		// let the remaining part of the first registration proceed
		finishConnection.trigger();
		Thread.sleep(1L);

		// verify that the latest registration is valid not being unregistered by the delayed one
		final TaskManagerInfo taskManagerInfo = rmGateway.requestTaskManagerInfo(
			taskExecutorResourceID,
			TIMEOUT).get();
		assertThat(taskManagerInfo.getResourceId(), equalTo(taskExecutorResourceID));
		assertThat(taskManagerInfo.getNumberSlots(), equalTo(1));
	} finally {
		rpcService.resetRpcGatewayFutureFunction();
	}
}
 

/**
 * Tests that free slots which are reported as allocated won't be considered for fulfilling
 * other pending slot requests.
 *
 * <p>See: FLINK-8505
 */
@Test
public void testReportAllocatedSlot() throws Exception {
	final ResourceID taskManagerId = ResourceID.generate();
	final ResourceActions resourceActions = new TestingResourceActionsBuilder().build();
	final TestingTaskExecutorGateway taskExecutorGateway = new TestingTaskExecutorGatewayBuilder().createTestingTaskExecutorGateway();
	final TaskExecutorConnection taskExecutorConnection = new TaskExecutorConnection(taskManagerId, taskExecutorGateway);

	try (final SlotManagerImpl slotManager = createSlotManager(ResourceManagerId.generate(), resourceActions)) {

		// initially report a single slot as free
		final SlotID slotId = new SlotID(taskManagerId, 0);
		final SlotStatus initialSlotStatus = new SlotStatus(
			slotId,
			ResourceProfile.ANY);
		final SlotReport initialSlotReport = new SlotReport(initialSlotStatus);

		slotManager.registerTaskManager(taskExecutorConnection, initialSlotReport);

		assertThat(slotManager.getNumberRegisteredSlots(), is(equalTo(1)));

		// Now report this slot as allocated
		final SlotStatus slotStatus = new SlotStatus(
			slotId,
			ResourceProfile.ANY,
			new JobID(),
			new AllocationID());
		final SlotReport slotReport = new SlotReport(
			slotStatus);

		slotManager.reportSlotStatus(
			taskExecutorConnection.getInstanceID(),
			slotReport);

		// this slot request should not be fulfilled
		final AllocationID allocationId = new AllocationID();
		final SlotRequest slotRequest = new SlotRequest(
			new JobID(),
			allocationId,
			ResourceProfile.UNKNOWN,
			"foobar");

		// This triggered an IllegalStateException before
		slotManager.registerSlotRequest(slotRequest);

		assertThat(slotManager.getSlotRequest(allocationId).isAssigned(), is(false));
	}
}
 

/**
 * Tests that unused offered slots are directly used to fulfill pending slot
 * requests.
 *
 * <p>Moreover it tests that the old slot request is canceled
 *
 * <p>See FLINK-8089, FLINK-8934
 */
@Test
public void testFulfillingSlotRequestsWithUnusedOfferedSlots() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {
		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));
		final ArrayBlockingQueue<AllocationID> canceledAllocations = new ArrayBlockingQueue<>(2);
		resourceManagerGateway.setCancelSlotConsumer(canceledAllocations::offer);
		final SlotRequestId slotRequestId1 = new SlotRequestId();
		final SlotRequestId slotRequestId2 = new SlotRequestId();
		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		final Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final ScheduledUnit scheduledUnit = new ScheduledUnit(
			new JobVertexID(),
			null,
			null);

		CompletableFuture<LogicalSlot> slotFuture1 = scheduler.allocateSlot(
			slotRequestId1,
			scheduledUnit,
			SlotProfile.noRequirements(),
			timeout);

		// wait for the first slot request
		final AllocationID allocationId1 = allocationIds.take();

		CompletableFuture<LogicalSlot> slotFuture2 = scheduler.allocateSlot(
			slotRequestId2,
			scheduledUnit,
			SlotProfile.noRequirements(),
			timeout);

		// wait for the second slot request
		final AllocationID allocationId2 = allocationIds.take();

		slotPool.releaseSlot(slotRequestId1, null);

		try {
			// this should fail with a CancellationException
			slotFuture1.get();
			fail("The first slot future should have failed because it was cancelled.");
		} catch (ExecutionException ee) {
			// expected
			assertTrue(ExceptionUtils.stripExecutionException(ee) instanceof FlinkException);
		}

		assertEquals(allocationId1, canceledAllocations.take());

		final SlotOffer slotOffer = new SlotOffer(allocationId1, 0, ResourceProfile.ANY);

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		assertTrue(slotPool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer));

		// the slot offer should fulfill the second slot request
		assertEquals(allocationId1, slotFuture2.get().getAllocationId());

		// check that the second slot allocation has been canceled
		assertEquals(allocationId2, canceledAllocations.take());
	}
}
 

private void runSlotRequestCompletionTest(
	Supplier<SlotPoolImpl> slotPoolSupplier,
	Consumer<SlotPoolImpl> actionAfterSlotRequest) {
	try (final SlotPoolImpl slotPool = slotPoolSupplier.get()) {

		final int requestNum = 10;

		final List<SlotRequestId> slotRequestIds = IntStream.range(0, requestNum)
			.mapToObj(ignored -> new SlotRequestId())
			.collect(Collectors.toList());

		final List<SlotRequest> rmReceivedSlotRequests = new ArrayList<>(requestNum);
		resourceManagerGateway.setRequestSlotConsumer(request -> rmReceivedSlotRequests.add(request));

		final List<CompletableFuture<PhysicalSlot>> slotRequests = slotRequestIds
			.stream()
			.map(slotRequestId -> slotPool.requestNewAllocatedSlot(slotRequestId, ResourceProfile.UNKNOWN, TIMEOUT))
			.collect(Collectors.toList());

		actionAfterSlotRequest.accept(slotPool);

		final LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		slotPool.registerTaskManager(taskManagerLocation.getResourceID());

		// create a slot offer that is initiated by the last request
		final AllocationID lastAllocationId = rmReceivedSlotRequests.get(requestNum - 1).getAllocationId();
		final SlotOffer slotOffer = new SlotOffer(lastAllocationId, 0, ResourceProfile.ANY);
		final Collection<SlotOffer> acceptedSlots = slotPool.offerSlots(taskManagerLocation, new SimpleAckingTaskManagerGateway(), Collections.singleton(slotOffer));

		assertThat(acceptedSlots, containsInAnyOrder(slotOffer));

		final FlinkException testingReleaseException = new FlinkException("Testing release exception");

		// check that the slot requests get completed in sequential order
		for (int i = 0; i < slotRequestIds.size(); i++) {
			final CompletableFuture<PhysicalSlot> slotRequestFuture = slotRequests.get(i);
			assertThat(slotRequestFuture.getNow(null), is(not(nullValue())));
			slotPool.releaseSlot(slotRequestIds.get(i), testingReleaseException);
		}
	}
}