Java Code Examples for org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway#setRequestSlotConsumer()

The following examples show how to use org.apache.flink.runtime.resourcemanager.utils.TestingResourceManagerGateway#setRequestSlotConsumer() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a batch slot request does react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfAllocationFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(),
			new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN));

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
 
Example 2
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a batch slot request does not react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is not {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfAllocationFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(), new FlinkException("Failed request"));

		assertThat(slotFuture.isDone(), is(false));
	}
}
 
Example 3
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a batch slot request does react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestFailsIfAllocationFailsUnfulfillably() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(),
			new UnfulfillableSlotRequestException(new AllocationID(), ResourceProfile.UNKNOWN));

		assertThat(slotFuture.isCompletedExceptionally(), is(true));
	}
}
 
Example 4
Source File: SlotPoolBatchSlotRequestTest.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Tests that a batch slot request does not react to {@link SlotPool#failAllocation(AllocationID, Exception)}
 * signals whose exception is not {@link UnfulfillableSlotRequestException}.
 */
@Test
public void testPendingBatchSlotRequestDoesNotFailIfAllocationFails() throws Exception {
	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	testingResourceManagerGateway.setRequestSlotConsumer(slotRequest -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final ComponentMainThreadExecutor directMainThreadExecutor = ComponentMainThreadExecutorServiceAdapter.forMainThread();

	final Time batchSlotTimeout = Time.milliseconds(1000L);
	try (final SlotPoolImpl slotPool = new SlotPoolBuilder(directMainThreadExecutor)
		.setBatchSlotTimeout(batchSlotTimeout)
		.setResourceManagerGateway(testingResourceManagerGateway)
		.build()) {

		final CompletableFuture<PhysicalSlot> slotFuture = SlotPoolUtils.requestNewAllocatedBatchSlot(slotPool, directMainThreadExecutor, resourceProfile);

		SlotPoolUtils.failAllocation(slotPool, directMainThreadExecutor, allocationIdFuture.get(), new FlinkException("Failed request"));

		assertThat(slotFuture.isDone(), is(false));
	}
}
 
Example 5
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
 
Example 6
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.ANY));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example 7
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example 8
Source File: SlotPoolSlotSharingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
 
Example 9
Source File: SlotPoolSlotSharingTest.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
 
Example 10
Source File: SlotPoolCoLocationTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the scheduling of two tasks with a parallelism of 2 and a co-location constraint.
 */
@Test
public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, InterruptedException {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);

	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();

	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID());

	CoLocationGroup group = new CoLocationGroup();
	CoLocationConstraint coLocationConstraint1 = group.getLocationConstraint(0);
	CoLocationConstraint coLocationConstraint2 = group.getLocationConstraint(1);

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();

	JobVertexID jobVertexId1 = new JobVertexID();
	JobVertexID jobVertexId2 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture11 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint1),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture22 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint2),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture12 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint1),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture21 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint2),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId1 = allocationIds.take();
	final AllocationID allocationId2 = allocationIds.take();

	Collection<SlotOffer> slotOfferFuture1 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.UNKNOWN)));

	Collection<SlotOffer> slotOfferFuture2 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId2,
			0,
			ResourceProfile.UNKNOWN)));

	assertFalse(slotOfferFuture1.isEmpty());
	assertFalse(slotOfferFuture2.isEmpty());

	LogicalSlot logicalSlot11 = logicalSlotFuture11.get();
	LogicalSlot logicalSlot12 = logicalSlotFuture12.get();
	LogicalSlot logicalSlot21 = logicalSlotFuture21.get();
	LogicalSlot logicalSlot22 = logicalSlotFuture22.get();

	assertEquals(logicalSlot11.getAllocationId(), logicalSlot12.getAllocationId());
	assertEquals(logicalSlot21.getAllocationId(), logicalSlot22.getAllocationId());
	assertNotEquals(logicalSlot11.getAllocationId(), logicalSlot21.getAllocationId());
}
 
Example 11
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that extra slots are kept by the {@link SlotPoolImpl}.
 */
@Test
public void testExtraSlotsAreKept() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();

		TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		pool.connectToResourceManager(resourceManagerGateway);

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());

		AllocationID allocationId = allocationIdFuture.get();
		final SlotOffer slotOffer = new SlotOffer(
			allocationId,
			0,
			DEFAULT_TESTING_PROFILE);
		final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

		testMainThreadExecutor.execute(() -> pool.registerTaskManager(taskManagerLocation.getResourceID()));

		assertTrue(testMainThreadExecutor.execute(() -> pool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer)));

		assertTrue(pool.containsAvailableSlot(allocationId));
	}
}
 
Example 12
Source File: SlotPoolSlotSharingTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests queued slot scheduling with multiple slot sharing groups.
 */
@Test
public void testQueuedMultipleSlotSharingGroups() throws Exception {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(4);

	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotSharingGroupId slotSharingGroupId1 = new SlotSharingGroupId();
	final SlotSharingGroupId slotSharingGroupId2 = new SlotSharingGroupId();
	final JobVertexID jobVertexId1 = new JobVertexID();
	final JobVertexID jobVertexId2 = new JobVertexID();
	final JobVertexID jobVertexId3 = new JobVertexID();
	final JobVertexID jobVertexId4 = new JobVertexID();

	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture1 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId1,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId1,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId3,
			slotSharingGroupId2,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId4,
			slotSharingGroupId2,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());
	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	// we expect two slot requests
	final AllocationID allocationId1 = allocationIds.take();
	final AllocationID allocationId2 = allocationIds.take();

	boolean offerFuture1 = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.UNKNOWN));

	boolean offerFuture2 = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId2,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(offerFuture1);
	assertTrue(offerFuture2);

	LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	LogicalSlot logicalSlot2 = logicalSlotFuture2.get();
	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();
	LogicalSlot logicalSlot4 = logicalSlotFuture4.get();

	assertEquals(logicalSlot1.getTaskManagerLocation(), logicalSlot2.getTaskManagerLocation());
	assertEquals(logicalSlot3.getTaskManagerLocation(), logicalSlot4.getTaskManagerLocation());

	assertEquals(allocationId1, logicalSlot1.getAllocationId());
	assertEquals(allocationId2, logicalSlot3.getAllocationId());
}
 
Example 13
Source File: SlotPoolInteractionsTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that extra slots are kept by the {@link SlotPoolImpl}.
 */
@Test
public void testExtraSlotsAreKept() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = new TestingSlotPool(
		jid,
		SystemClock.getInstance(),
		TestingUtils.infiniteTime(),
		TestingUtils.infiniteTime())) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.INSTANCE, pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();

		TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		pool.connectToResourceManager(resourceManagerGateway);

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(SchedulerTestUtils.getDummyTask()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			true,
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());

		AllocationID allocationId = allocationIdFuture.get();
		final SlotOffer slotOffer = new SlotOffer(
			allocationId,
			0,
			DEFAULT_TESTING_PROFILE);
		final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

		testMainThreadExecutor.execute(() -> pool.registerTaskManager(taskManagerLocation.getResourceID()));

		assertTrue(testMainThreadExecutor.execute(() -> pool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer)));

		assertTrue(pool.containsAvailableSlot(allocationId));
	}
}
 
Example 14
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests queued slot scheduling with a single slot sharing group
 */
@Test
public void testQueuedSharedSlotScheduling() throws Exception {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final JobVertexID jobVertexId1 = new JobVertexID();
	final JobVertexID jobVertexId2 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture1 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());

	final AllocationID allocationId1 = allocationIds.take();

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	allocationIds.take();

	// this should fulfill the first two slot futures
	boolean offerFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(offerFuture);

	LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	LogicalSlot logicalSlot2 = logicalSlotFuture2.get();

	assertEquals(logicalSlot1.getTaskManagerLocation(), logicalSlot2.getTaskManagerLocation());
	assertEquals(allocationId1, logicalSlot1.getAllocationId());
	assertEquals(allocationId1, logicalSlot2.getAllocationId());

	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	// release the shared slot by releasing the individual tasks
	logicalSlot1.releaseSlot(null);
	logicalSlot2.releaseSlot(null);

	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();
	LogicalSlot logicalSlot4 = logicalSlotFuture4.get();

	assertEquals(logicalSlot3.getTaskManagerLocation(), logicalSlot4.getTaskManagerLocation());
	assertEquals(allocationId1, logicalSlot3.getAllocationId());
	assertEquals(allocationId1, logicalSlot4.getAllocationId());
}
 
Example 15
Source File: SlotPoolInteractionsTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that extra slots are kept by the {@link SlotPoolImpl}.
 */
@Test
public void testExtraSlotsAreKept() throws Exception {
	final JobID jid = new JobID();

	try (TestingSlotPool pool = createTestingSlotPool(jid)) {

		pool.start(JobMasterId.generate(), "foobar", testMainThreadExecutor.getMainThreadExecutor());

		Scheduler scheduler = new SchedulerImpl(LocationPreferenceSlotSelectionStrategy.createDefault(), pool);
		scheduler.start(testMainThreadExecutor.getMainThreadExecutor());

		final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();

		TestingResourceManagerGateway resourceManagerGateway = new TestingResourceManagerGateway();
		resourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

		final CompletableFuture<SlotRequestId> slotRequestTimeoutFuture = new CompletableFuture<>();
		pool.setTimeoutPendingSlotRequestConsumer(slotRequestTimeoutFuture::complete);

		pool.connectToResourceManager(resourceManagerGateway);

		SlotRequestId requestId = new SlotRequestId();
		CompletableFuture<LogicalSlot> future = testMainThreadExecutor.execute(() -> scheduler.allocateSlot(
			requestId,
			new ScheduledUnit(getExecution()),
			SlotProfile.noLocality(DEFAULT_TESTING_PROFILE),
			fastTimeout));

		try {
			future.get();
			fail("We expected a TimeoutException.");
		} catch (ExecutionException e) {
			assertTrue(ExceptionUtils.stripExecutionException(e) instanceof TimeoutException);
		}

		// wait until we have timed out the slot request
		slotRequestTimeoutFuture.get();

		assertEquals(0L, pool.getNumberOfPendingRequests());

		AllocationID allocationId = allocationIdFuture.get();
		final SlotOffer slotOffer = new SlotOffer(
			allocationId,
			0,
			DEFAULT_TESTING_PROFILE);
		final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
		final TaskManagerGateway taskManagerGateway = new SimpleAckingTaskManagerGateway();

		testMainThreadExecutor.execute(() -> pool.registerTaskManager(taskManagerLocation.getResourceID()));

		assertTrue(testMainThreadExecutor.execute(() -> pool.offerSlot(taskManagerLocation, taskManagerGateway, slotOffer)));

		assertTrue(pool.containsAvailableSlot(allocationId));
	}
}
 
Example 16
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests queued slot scheduling with multiple slot sharing groups.
 */
@Test
public void testQueuedMultipleSlotSharingGroups() throws Exception {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(4);

	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotSharingGroupId slotSharingGroupId1 = new SlotSharingGroupId();
	final SlotSharingGroupId slotSharingGroupId2 = new SlotSharingGroupId();
	final JobVertexID jobVertexId1 = new JobVertexID();
	final JobVertexID jobVertexId2 = new JobVertexID();
	final JobVertexID jobVertexId3 = new JobVertexID();
	final JobVertexID jobVertexId4 = new JobVertexID();

	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture1 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId1,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId1,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId3,
			slotSharingGroupId2,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId4,
			slotSharingGroupId2,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());
	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	// we expect two slot requests
	final AllocationID allocationId1 = allocationIds.take();
	final AllocationID allocationId2 = allocationIds.take();

	boolean offerFuture1 = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.ANY));

	boolean offerFuture2 = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId2,
			0,
			ResourceProfile.ANY));

	assertTrue(offerFuture1);
	assertTrue(offerFuture2);

	LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	LogicalSlot logicalSlot2 = logicalSlotFuture2.get();
	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();
	LogicalSlot logicalSlot4 = logicalSlotFuture4.get();

	assertEquals(logicalSlot1.getTaskManagerLocation(), logicalSlot2.getTaskManagerLocation());
	assertEquals(logicalSlot3.getTaskManagerLocation(), logicalSlot4.getTaskManagerLocation());

	assertEquals(allocationId1, logicalSlot1.getAllocationId());
	assertEquals(allocationId2, logicalSlot3.getAllocationId());
}
 
Example 17
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests that when matching from the allocated slot, the remaining resources of the slot
 * will be used instead of the total resource.
 */
@Test
public void testSlotSharingRespectsRemainingResource() throws Exception {
	final ResourceProfile allocatedSlotRp = ResourceProfile.fromResources(3.0, 300);
	final ResourceProfile largeRequestResource = ResourceProfile.fromResources(2.0, 200);
	final ResourceProfile smallRequestResource = ResourceProfile.fromResources(1.0, 100);

	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
			(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final JobVertexID jobVertexId1 = new JobVertexID();
	final JobVertexID jobVertexId2 = new JobVertexID();
	final JobVertexID jobVertexId3 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture1 = slotProvider.allocateSlot(
			new ScheduledUnit(
					jobVertexId1,
					slotSharingGroupId,
					null),
			SlotProfile.noLocality(largeRequestResource),
			TestingUtils.infiniteTime());

	final AllocationID allocationId1 = allocationIds.take();

	// This should fulfill the first request.
	boolean offerFuture = slotPool.offerSlot(
			taskManagerLocation,
			new SimpleAckingTaskManagerGateway(),
			new SlotOffer(
					allocationId1,
					0,
					allocatedSlotRp));

	assertTrue(offerFuture);
	assertTrue(logicalSlotFuture1.isDone());
	assertEquals(allocationId1, logicalSlotFuture1.get().getAllocationId());

	// The second request should not share the same slot with the first request since it is large.
	CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
			new ScheduledUnit(
					jobVertexId2,
					slotSharingGroupId,
					null),
			SlotProfile.noLocality(largeRequestResource),
			TestingUtils.infiniteTime());
	assertFalse(logicalSlotFuture2.isDone());

	// The third request should be able to share the same slot with the first request since it is small.
	CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
			new ScheduledUnit(
					jobVertexId3,
					slotSharingGroupId,
					null),
			SlotProfile.noLocality(smallRequestResource),
			TestingUtils.infiniteTime());
	assertTrue(logicalSlotFuture3.isDone());
	assertEquals(allocationId1, logicalSlotFuture1.get().getAllocationId());

	// The second request should be finally fulfilled by a new slot.
	final AllocationID allocationId2 = allocationIds.take();
	// This should fulfill the first two requests.
	offerFuture = slotPool.offerSlot(
			taskManagerLocation,
			new SimpleAckingTaskManagerGateway(),
			new SlotOffer(
					allocationId2,
					0,
					allocatedSlotRp));

	assertTrue(offerFuture);
	assertTrue(logicalSlotFuture2.isDone());
	assertEquals(allocationId2, logicalSlotFuture2.get().getAllocationId());
}
 
Example 18
Source File: SlotPoolCoLocationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the scheduling of two tasks with a parallelism of 2 and a co-location constraint.
 */
@Test
public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, InterruptedException {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);

	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();

	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID());

	CoLocationGroup group = new CoLocationGroup();
	CoLocationConstraint coLocationConstraint1 = group.getLocationConstraint(0);
	CoLocationConstraint coLocationConstraint2 = group.getLocationConstraint(1);

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();

	JobVertexID jobVertexId1 = new JobVertexID();
	JobVertexID jobVertexId2 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture11 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint1),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture22 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint2),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture12 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint1),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture21 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint2),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId1 = allocationIds.take();
	final AllocationID allocationId2 = allocationIds.take();

	Collection<SlotOffer> slotOfferFuture1 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.UNKNOWN)));

	Collection<SlotOffer> slotOfferFuture2 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId2,
			0,
			ResourceProfile.UNKNOWN)));

	assertFalse(slotOfferFuture1.isEmpty());
	assertFalse(slotOfferFuture2.isEmpty());

	LogicalSlot logicalSlot11 = logicalSlotFuture11.get();
	LogicalSlot logicalSlot12 = logicalSlotFuture12.get();
	LogicalSlot logicalSlot21 = logicalSlotFuture21.get();
	LogicalSlot logicalSlot22 = logicalSlotFuture22.get();

	assertEquals(logicalSlot11.getAllocationId(), logicalSlot12.getAllocationId());
	assertEquals(logicalSlot21.getAllocationId(), logicalSlot22.getAllocationId());
	assertNotEquals(logicalSlot11.getAllocationId(), logicalSlot21.getAllocationId());
}
 
Example 19
Source File: SlotPoolSlotSharingTest.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Tests queued slot scheduling with a single slot sharing group
 */
@Test
public void testQueuedSharedSlotScheduling() throws Exception {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final JobVertexID jobVertexId1 = new JobVertexID();
	final JobVertexID jobVertexId2 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture1 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture2 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture1.isDone());
	assertFalse(logicalSlotFuture2.isDone());

	final AllocationID allocationId1 = allocationIds.take();

	CompletableFuture<LogicalSlot> logicalSlotFuture3 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture4 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	allocationIds.take();

	// this should fulfill the first two slot futures
	boolean offerFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.UNKNOWN));

	assertTrue(offerFuture);

	LogicalSlot logicalSlot1 = logicalSlotFuture1.get();
	LogicalSlot logicalSlot2 = logicalSlotFuture2.get();

	assertEquals(logicalSlot1.getTaskManagerLocation(), logicalSlot2.getTaskManagerLocation());
	assertEquals(allocationId1, logicalSlot1.getAllocationId());
	assertEquals(allocationId1, logicalSlot2.getAllocationId());

	assertFalse(logicalSlotFuture3.isDone());
	assertFalse(logicalSlotFuture4.isDone());

	// release the shared slot by releasing the individual tasks
	logicalSlot1.releaseSlot(null);
	logicalSlot2.releaseSlot(null);

	LogicalSlot logicalSlot3 = logicalSlotFuture3.get();
	LogicalSlot logicalSlot4 = logicalSlotFuture4.get();

	assertEquals(logicalSlot3.getTaskManagerLocation(), logicalSlot4.getTaskManagerLocation());
	assertEquals(allocationId1, logicalSlot3.getAllocationId());
	assertEquals(allocationId1, logicalSlot4.getAllocationId());
}
 
Example 20
Source File: SlotPoolCoLocationTest.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Tests the scheduling of two tasks with a parallelism of 2 and a co-location constraint.
 */
@Test
public void testSimpleCoLocatedSlotScheduling() throws ExecutionException, InterruptedException {
	final BlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(2);

	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();

	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIds.offer(slotRequest.getAllocationId()));

	final TaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();

	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.registerTaskManager(taskManagerLocation.getResourceID());

	CoLocationGroup group = new CoLocationGroup();
	CoLocationConstraint coLocationConstraint1 = group.getLocationConstraint(0);
	CoLocationConstraint coLocationConstraint2 = group.getLocationConstraint(1);

	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();

	JobVertexID jobVertexId1 = new JobVertexID();
	JobVertexID jobVertexId2 = new JobVertexID();

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture11 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint1),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture22 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint2),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture12 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId2,
			slotSharingGroupId,
			coLocationConstraint1),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	CompletableFuture<LogicalSlot> logicalSlotFuture21 = slotProvider.allocateSlot(
		new ScheduledUnit(
			jobVertexId1,
			slotSharingGroupId,
			coLocationConstraint2),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId1 = allocationIds.take();
	final AllocationID allocationId2 = allocationIds.take();

	Collection<SlotOffer> slotOfferFuture1 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId1,
			0,
			ResourceProfile.ANY)));

	Collection<SlotOffer> slotOfferFuture2 = slotPoolGateway.offerSlots(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		Collections.singletonList(new SlotOffer(
			allocationId2,
			0,
			ResourceProfile.ANY)));

	assertFalse(slotOfferFuture1.isEmpty());
	assertFalse(slotOfferFuture2.isEmpty());

	LogicalSlot logicalSlot11 = logicalSlotFuture11.get();
	LogicalSlot logicalSlot12 = logicalSlotFuture12.get();
	LogicalSlot logicalSlot21 = logicalSlotFuture21.get();
	LogicalSlot logicalSlot22 = logicalSlotFuture22.get();

	assertEquals(logicalSlot11.getAllocationId(), logicalSlot12.getAllocationId());
	assertEquals(logicalSlot21.getAllocationId(), logicalSlot22.getAllocationId());
	assertNotEquals(logicalSlot11.getAllocationId(), logicalSlot21.getAllocationId());
}