org.apache.flink.runtime.jobmaster.AllocatedSlotInfo Java Examples

The following examples show how to use org.apache.flink.runtime.jobmaster.AllocatedSlotInfo. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example #1

Source File: TaskExecutor.java From Flink-CEPplus with Apache License 2.0

6 votes

private void failNoLongerAllocatedSlots(AllocatedSlotReport allocatedSlotReport, JobMasterGateway jobMasterGateway) {
	for (AllocatedSlotInfo allocatedSlotInfo : allocatedSlotReport.getAllocatedSlotInfos()) {
		final AllocationID allocationId = allocatedSlotInfo.getAllocationId();
		if (!taskSlotTable.isAllocated(
				allocatedSlotInfo.getSlotIndex(),
				allocatedSlotReport.getJobId(),
				allocationId)) {
			jobMasterGateway.failSlot(
					getResourceID(),
					allocationId,
					new FlinkException(
						String.format(
							"Slot %s on TaskExecutor %s is not allocated by job %s.",
							allocatedSlotInfo.getSlotIndex(),
							getResourceID(),
							allocatedSlotReport.getJobId())));
		}
	}
}

Example #2

Source File: TaskExecutor.java From flink with Apache License 2.0

6 votes

private void failNoLongerAllocatedSlots(AllocatedSlotReport allocatedSlotReport, JobMasterGateway jobMasterGateway) {
	for (AllocatedSlotInfo allocatedSlotInfo : allocatedSlotReport.getAllocatedSlotInfos()) {
		final AllocationID allocationId = allocatedSlotInfo.getAllocationId();
		if (!taskSlotTable.isAllocated(
				allocatedSlotInfo.getSlotIndex(),
				allocatedSlotReport.getJobId(),
				allocationId)) {
			jobMasterGateway.failSlot(
					getResourceID(),
					allocationId,
					new FlinkException(
						String.format(
							"Slot %s on TaskExecutor %s is not allocated by job %s.",
							allocatedSlotInfo.getSlotIndex(),
							getResourceID(),
							allocatedSlotReport.getJobId())));
		}
	}
}

Example #3

Source File: TaskExecutor.java From flink with Apache License 2.0

6 votes

private void failNoLongerAllocatedSlots(AllocatedSlotReport allocatedSlotReport, JobMasterGateway jobMasterGateway) {
	for (AllocatedSlotInfo allocatedSlotInfo : allocatedSlotReport.getAllocatedSlotInfos()) {
		final AllocationID allocationId = allocatedSlotInfo.getAllocationId();
		if (!taskSlotTable.isAllocated(
				allocatedSlotInfo.getSlotIndex(),
				allocatedSlotReport.getJobId(),
				allocationId)) {
			jobMasterGateway.failSlot(
					getResourceID(),
					allocationId,
					new FlinkException(
						String.format(
							"Slot %s on TaskExecutor %s is not allocated by job %s.",
							allocatedSlotInfo.getSlotIndex(),
							getResourceID(),
							allocatedSlotReport.getJobId())));
		}
	}
}

Example #4

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}

Example #5

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

5 votes

private static Matcher<AllocatedSlotInfo> isEqualAllocatedSlotInfo(AllocatedSlotInfo expectedAllocatedSlotInfo) {
	return new TypeSafeDiagnosingMatcher<AllocatedSlotInfo>() {
		@Override
		public void describeTo(Description description) {
			description.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
		}

		private String describeAllocatedSlotInformation(AllocatedSlotInfo expectedAllocatedSlotInformation) {
			return expectedAllocatedSlotInformation.toString();
		}

		@Override
		protected boolean matchesSafely(AllocatedSlotInfo item, Description mismatchDescription) {
			final boolean matches = item.getAllocationId().equals(expectedAllocatedSlotInfo.getAllocationId()) &&
				item.getSlotIndex() == expectedAllocatedSlotInfo.getSlotIndex();

			if (!matches) {
				mismatchDescription
					.appendText("Actual value ")
					.appendText(describeAllocatedSlotInformation(item))
					.appendText(" differs from expected value ")
					.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
			}

			return matches;
		}
	};
}

Example #6

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

5 votes

/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = createSlotPoolImpl()) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.ANY));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.ANY));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}

Example #7

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

private void freeNoLongerUsedSlots(AllocatedSlotReport allocatedSlotReport) {
	final Iterator<AllocationID> slotsTaskManagerSide = taskSlotTable.getActiveSlots(allocatedSlotReport.getJobId());
	final Set<AllocationID> activeSlots = Sets.newHashSet(slotsTaskManagerSide);
	final Set<AllocationID> reportedSlots = allocatedSlotReport.getAllocatedSlotInfos().stream()
			.map(AllocatedSlotInfo::getAllocationId).collect(Collectors.toSet());

	final Sets.SetView<AllocationID> difference = Sets.difference(activeSlots, reportedSlots);

	for (AllocationID allocationID : difference) {
		freeSlotInternal(
			allocationID,
			new FlinkException(
				String.format("%s is no longer allocated by job %s.", allocationID, allocatedSlotReport.getJobId())));
	}
}

Example #8

Source File: SlotPoolImpl.java From flink with Apache License 2.0

5 votes

@Override
public AllocatedSlotReport createAllocatedSlotReport(ResourceID taskManagerId) {
	final Set<AllocatedSlot> availableSlotsForTaskManager = availableSlots.getSlotsForTaskManager(taskManagerId);
	final Set<AllocatedSlot> allocatedSlotsForTaskManager = allocatedSlots.getSlotsForTaskManager(taskManagerId);

	List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(
			availableSlotsForTaskManager.size() + allocatedSlotsForTaskManager.size());
	for (AllocatedSlot allocatedSlot : Iterables.concat(availableSlotsForTaskManager, allocatedSlotsForTaskManager)) {
		allocatedSlotInfos.add(
				new AllocatedSlotInfo(allocatedSlot.getPhysicalSlotNumber(), allocatedSlot.getAllocationId()));
	}
	return new AllocatedSlotReport(jobId, allocatedSlotInfos);
}

Example #9

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

5 votes

private static Matcher<AllocatedSlotInfo> isEqualAllocatedSlotInfo(AllocatedSlotInfo expectedAllocatedSlotInfo) {
	return new TypeSafeDiagnosingMatcher<AllocatedSlotInfo>() {
		@Override
		public void describeTo(Description description) {
			description.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
		}

		private String describeAllocatedSlotInformation(AllocatedSlotInfo expectedAllocatedSlotInformation) {
			return expectedAllocatedSlotInformation.toString();
		}

		@Override
		protected boolean matchesSafely(AllocatedSlotInfo item, Description mismatchDescription) {
			final boolean matches = item.getAllocationId().equals(expectedAllocatedSlotInfo.getAllocationId()) &&
				item.getSlotIndex() == expectedAllocatedSlotInfo.getSlotIndex();

			if (!matches) {
				mismatchDescription
					.appendText("Actual value ")
					.appendText(describeAllocatedSlotInformation(item))
					.appendText(" differs from expected value ")
					.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
			}

			return matches;
		}
	};
}

Example #10

Source File: SlotPoolImpl.java From Flink-CEPplus with Apache License 2.0

5 votes

@Override
public AllocatedSlotReport createAllocatedSlotReport(ResourceID taskManagerId) {
	final Set<AllocatedSlot> availableSlotsForTaskManager = availableSlots.getSlotsForTaskManager(taskManagerId);
	final Set<AllocatedSlot> allocatedSlotsForTaskManager = allocatedSlots.getSlotsForTaskManager(taskManagerId);

	List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(
			availableSlotsForTaskManager.size() + allocatedSlotsForTaskManager.size());
	for (AllocatedSlot allocatedSlot : Iterables.concat(availableSlotsForTaskManager, allocatedSlotsForTaskManager)) {
		allocatedSlotInfos.add(
				new AllocatedSlotInfo(allocatedSlot.getPhysicalSlotNumber(), allocatedSlot.getAllocationId()));
	}
	return new AllocatedSlotReport(jobId, allocatedSlotInfos);
}

Example #11

Source File: TaskExecutor.java From flink with Apache License 2.0

5 votes

private void freeNoLongerUsedSlots(AllocatedSlotReport allocatedSlotReport) {
	final Iterator<AllocationID> slotsTaskManagerSide = taskSlotTable.getActiveSlots(allocatedSlotReport.getJobId());
	final Set<AllocationID> activeSlots = Sets.newHashSet(slotsTaskManagerSide);
	final Set<AllocationID> reportedSlots = allocatedSlotReport.getAllocatedSlotInfos().stream()
			.map(AllocatedSlotInfo::getAllocationId).collect(Collectors.toSet());

	final Sets.SetView<AllocationID> difference = Sets.difference(activeSlots, reportedSlots);

	for (AllocationID allocationID : difference) {
		freeSlotInternal(
			allocationID,
			new FlinkException(
				String.format("%s is no longer allocated by job %s.", allocationID, allocatedSlotReport.getJobId())));
	}
}

Example #12

Source File: SlotPoolImpl.java From flink with Apache License 2.0

5 votes

@Override
public AllocatedSlotReport createAllocatedSlotReport(ResourceID taskManagerId) {
	final Set<AllocatedSlot> availableSlotsForTaskManager = availableSlots.getSlotsForTaskManager(taskManagerId);
	final Set<AllocatedSlot> allocatedSlotsForTaskManager = allocatedSlots.getSlotsForTaskManager(taskManagerId);

	List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(
			availableSlotsForTaskManager.size() + allocatedSlotsForTaskManager.size());
	for (AllocatedSlot allocatedSlot : Iterables.concat(availableSlotsForTaskManager, allocatedSlotsForTaskManager)) {
		allocatedSlotInfos.add(
				new AllocatedSlotInfo(allocatedSlot.getPhysicalSlotNumber(), allocatedSlot.getAllocationId()));
	}
	return new AllocatedSlotReport(jobId, allocatedSlotInfos);
}

Example #13

Source File: SlotPoolImplTest.java From Flink-CEPplus with Apache License 2.0

5 votes

private static Matcher<AllocatedSlotInfo> isEqualAllocatedSlotInfo(AllocatedSlotInfo expectedAllocatedSlotInfo) {
	return new TypeSafeDiagnosingMatcher<AllocatedSlotInfo>() {
		@Override
		public void describeTo(Description description) {
			description.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
		}

		private String describeAllocatedSlotInformation(AllocatedSlotInfo expectedAllocatedSlotInformation) {
			return expectedAllocatedSlotInformation.toString();
		}

		@Override
		protected boolean matchesSafely(AllocatedSlotInfo item, Description mismatchDescription) {
			final boolean matches = item.getAllocationId().equals(expectedAllocatedSlotInfo.getAllocationId()) &&
				item.getSlotIndex() == expectedAllocatedSlotInfo.getSlotIndex();

			if (!matches) {
				mismatchDescription
					.appendText("Actual value ")
					.appendText(describeAllocatedSlotInformation(item))
					.appendText(" differs from expected value ")
					.appendText(describeAllocatedSlotInformation(expectedAllocatedSlotInfo));
			}

			return matches;
		}
	};
}

Example #14

Source File: SlotPoolImplTest.java From Flink-CEPplus with Apache License 2.0

5 votes

/**
 * Tests that create report of allocated slots on a {@link TaskExecutor}.
 */
@Test
public void testCreateAllocatedSlotReport() throws Exception {

	try (SlotPoolImpl slotPool = new SlotPoolImpl(jobId)) {

		final ArrayBlockingQueue<AllocationID> allocationIds = new ArrayBlockingQueue<>(1);
		resourceManagerGateway.setRequestSlotConsumer(
				slotRequest -> allocationIds.offer(slotRequest.getAllocationId()));

		setupSlotPool(slotPool, resourceManagerGateway, mainThreadExecutor);
		Scheduler scheduler = setupScheduler(slotPool, mainThreadExecutor);

		final SlotRequestId slotRequestId = new SlotRequestId();
		final CompletableFuture<LogicalSlot> slotRequestFuture = allocateSlot(scheduler, slotRequestId);

		final List<AllocatedSlotInfo> allocatedSlotInfos = new ArrayList<>(2);
		final List<SlotOffer> slotOffers = new ArrayList<>(2);

		final AllocationID allocatedId = allocationIds.take();
		slotOffers.add(new SlotOffer(allocatedId, 0, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(0, allocatedId));

		final AllocationID availableId = new AllocationID();
		slotOffers.add(new SlotOffer(availableId, 1, ResourceProfile.UNKNOWN));
		allocatedSlotInfos.add(new AllocatedSlotInfo(1, availableId));

		slotPool.registerTaskManager(taskManagerLocation.getResourceID());
		slotPool.offerSlots(taskManagerLocation, taskManagerGateway, slotOffers);

		// wait for the completion of slot future
		slotRequestFuture.get();

		final AllocatedSlotReport slotReport = slotPool.createAllocatedSlotReport(taskManagerLocation.getResourceID());
		assertThat(jobId, is(slotReport.getJobId()));
		assertThat(slotReport.getAllocatedSlotInfos(), containsInAnyOrder(isEachEqual(allocatedSlotInfos)));
	}
}

Example #15

Source File: TaskExecutor.java From Flink-CEPplus with Apache License 2.0

5 votes

private void freeNoLongerUsedSlots(AllocatedSlotReport allocatedSlotReport) {
	final Iterator<AllocationID> slotsTaskManagerSide = taskSlotTable.getActiveSlots(allocatedSlotReport.getJobId());
	final Set<AllocationID> activeSlots = Sets.newHashSet(slotsTaskManagerSide);
	final Set<AllocationID> reportedSlots = allocatedSlotReport.getAllocatedSlotInfos().stream()
			.map(AllocatedSlotInfo::getAllocationId).collect(Collectors.toSet());

	final Sets.SetView<AllocationID> difference = Sets.difference(activeSlots, reportedSlots);

	for (AllocationID allocationID : difference) {
		freeSlotInternal(
			allocationID,
			new FlinkException(
				String.format("%s is no longer allocated by job %s.", allocationID, allocatedSlotReport.getJobId())));
	}
}

Example #16

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

4 votes

private static Collection<Matcher<? super AllocatedSlotInfo>> isEachEqual(Collection<AllocatedSlotInfo> allocatedSlotInfos) {
	return allocatedSlotInfos
		.stream()
		.map(SlotPoolImplTest::isEqualAllocatedSlotInfo)
		.collect(Collectors.toList());
}

Example #17

Source File: TaskExecutorTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests that the TaskExecutor syncs its slots view with the JobMaster's view
 * via the AllocatedSlotReport reported by the heartbeat (See FLINK-11059).
 */
@Test
public void testSyncSlotsWithJobMasterByHeartbeat() throws Exception {
	final CountDownLatch activeSlots = new CountDownLatch(2);
	final TaskSlotTable taskSlotTable = new ActivateSlotNotifyingTaskSlotTable(
			Arrays.asList(ResourceProfile.UNKNOWN, ResourceProfile.UNKNOWN),
			timerService,
			activeSlots);
	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();

	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);

	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();

	final BlockingQueue<AllocationID> allocationsNotifiedFree = new ArrayBlockingQueue<>(2);

	OneShotLatch initialSlotReporting = new OneShotLatch();
	testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
		initialSlotReporting.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());

	});

	testingResourceManagerGateway.setNotifySlotAvailableConsumer(instanceIDSlotIDAllocationIDTuple3 ->
			allocationsNotifiedFree.offer(instanceIDSlotIDAllocationIDTuple3.f2));

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final BlockingQueue<AllocationID> failedSlotFutures = new ArrayBlockingQueue<>(2);
	final ResourceID jobManagerResourceId = ResourceID.generate();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
			.setFailSlotConsumer((resourceID, allocationID, throwable) ->
				failedSlotFutures.offer(allocationID))
			.setOfferSlotsFunction((resourceID, slotOffers) -> CompletableFuture.completedFuture(new ArrayList<>(slotOffers)))
			.setRegisterTaskManagerFunction((ignoredA, ignoredB) -> CompletableFuture.completedFuture(new JMTMRegistrationSuccess(jobManagerResourceId)))
			.build();
	final String jobManagerAddress = jobMasterGateway.getAddress();
	rpc.registerGateway(jobManagerAddress, jobMasterGateway);
	jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobMasterGateway.getFencingToken().toUUID());

	taskExecutor.start();

	try {
		final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

		initialSlotReporting.await();

		final SlotID slotId1 = new SlotID(taskExecutor.getResourceID(), 0);
		final SlotID slotId2 = new SlotID(taskExecutor.getResourceID(), 1);
		final AllocationID allocationIdInBoth = new AllocationID();
		final AllocationID allocationIdOnlyInJM = new AllocationID();
		final AllocationID allocationIdOnlyInTM = new AllocationID();

		taskExecutorGateway.requestSlot(slotId1, jobId, allocationIdInBoth, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);
		taskExecutorGateway.requestSlot(slotId2, jobId, allocationIdOnlyInTM, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);

		activeSlots.await();

		List<AllocatedSlotInfo> allocatedSlotInfos = Arrays.asList(
				new AllocatedSlotInfo(0, allocationIdInBoth),
				new AllocatedSlotInfo(1, allocationIdOnlyInJM)
		);
		AllocatedSlotReport allocatedSlotReport = new AllocatedSlotReport(jobId, allocatedSlotInfos);
		taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, allocatedSlotReport);

		assertThat(failedSlotFutures.take(), is(allocationIdOnlyInJM));
		assertThat(allocationsNotifiedFree.take(), is(allocationIdOnlyInTM));
		assertThat(failedSlotFutures.poll(5L, TimeUnit.MILLISECONDS), nullValue());
		assertThat(allocationsNotifiedFree.poll(5L, TimeUnit.MILLISECONDS), nullValue());
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}

Example #18

Source File: TaskExecutorTest.java From Flink-CEPplus with Apache License 2.0

4 votes

/**
 * Tests that the TaskExecutor syncs its slots view with the JobMaster's view
 * via the AllocatedSlotReport reported by the heartbeat (See FLINK-11059).
 */
@Test
public void testSyncSlotsWithJobMasterByHeartbeat() throws Exception {
	final CountDownLatch activeSlots = new CountDownLatch(2);
	final TaskSlotTable taskSlotTable = new ActivateSlotNotifyingTaskSlotTable(
			Arrays.asList(ResourceProfile.UNKNOWN, ResourceProfile.UNKNOWN),
			timerService,
			activeSlots);
	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();

	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);

	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();

	final BlockingQueue<AllocationID> allocationsNotifiedFree = new ArrayBlockingQueue<>(2);

	OneShotLatch initialSlotReporting = new OneShotLatch();
	testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
		initialSlotReporting.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());

	});

	testingResourceManagerGateway.setNotifySlotAvailableConsumer(instanceIDSlotIDAllocationIDTuple3 ->
			allocationsNotifiedFree.offer(instanceIDSlotIDAllocationIDTuple3.f2));

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final BlockingQueue<AllocationID> failedSlotFutures = new ArrayBlockingQueue<>(2);
	final ResourceID jobManagerResourceId = ResourceID.generate();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
			.setFailSlotConsumer((resourceID, allocationID, throwable) ->
				failedSlotFutures.offer(allocationID))
			.setOfferSlotsFunction((resourceID, slotOffers) -> CompletableFuture.completedFuture(new ArrayList<>(slotOffers)))
			.setRegisterTaskManagerFunction((ignoredA, ignoredB) -> CompletableFuture.completedFuture(new JMTMRegistrationSuccess(jobManagerResourceId)))
			.build();
	final String jobManagerAddress = jobMasterGateway.getAddress();
	rpc.registerGateway(jobManagerAddress, jobMasterGateway);
	jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobMasterGateway.getFencingToken().toUUID());

	taskExecutor.start();

	try {
		final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

		initialSlotReporting.await();

		final SlotID slotId1 = new SlotID(taskExecutor.getResourceID(), 0);
		final SlotID slotId2 = new SlotID(taskExecutor.getResourceID(), 1);
		final AllocationID allocationIdInBoth = new AllocationID();
		final AllocationID allocationIdOnlyInJM = new AllocationID();
		final AllocationID allocationIdOnlyInTM = new AllocationID();

		taskExecutorGateway.requestSlot(slotId1, jobId, allocationIdInBoth, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);
		taskExecutorGateway.requestSlot(slotId2, jobId, allocationIdOnlyInTM, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);

		activeSlots.await();

		List<AllocatedSlotInfo> allocatedSlotInfos = Arrays.asList(
				new AllocatedSlotInfo(0, allocationIdInBoth),
				new AllocatedSlotInfo(1, allocationIdOnlyInJM)
		);
		AllocatedSlotReport allocatedSlotReport = new AllocatedSlotReport(jobId, allocatedSlotInfos);
		taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, allocatedSlotReport);

		assertThat(failedSlotFutures.take(), is(allocationIdOnlyInJM));
		assertThat(allocationsNotifiedFree.take(), is(allocationIdOnlyInTM));
		assertThat(failedSlotFutures.poll(5L, TimeUnit.MILLISECONDS), nullValue());
		assertThat(allocationsNotifiedFree.poll(5L, TimeUnit.MILLISECONDS), nullValue());
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}

Example #19

Source File: SlotPoolImplTest.java From Flink-CEPplus with Apache License 2.0

4 votes

private static Collection<Matcher<? super AllocatedSlotInfo>> isEachEqual(Collection<AllocatedSlotInfo> allocatedSlotInfos) {
	return allocatedSlotInfos
		.stream()
		.map(SlotPoolImplTest::isEqualAllocatedSlotInfo)
		.collect(Collectors.toList());
}

Example #20

Source File: SlotPoolImplTest.java From flink with Apache License 2.0

4 votes

private static Collection<Matcher<? super AllocatedSlotInfo>> isEachEqual(Collection<AllocatedSlotInfo> allocatedSlotInfos) {
	return allocatedSlotInfos
		.stream()
		.map(SlotPoolImplTest::isEqualAllocatedSlotInfo)
		.collect(Collectors.toList());
}

Example #21

Source File: TaskExecutorTest.java From flink with Apache License 2.0

4 votes

/**
 * Tests that the TaskExecutor syncs its slots view with the JobMaster's view
 * via the AllocatedSlotReport reported by the heartbeat (See FLINK-11059).
 */
@Test
public void testSyncSlotsWithJobMasterByHeartbeat() throws Exception {
	final CountDownLatch activeSlots = new CountDownLatch(2);
	final TaskSlotTable<Task> taskSlotTable = new ActivateSlotNotifyingTaskSlotTable(
			2,
			activeSlots);
	final TaskManagerServices taskManagerServices = new TaskManagerServicesBuilder().setTaskSlotTable(taskSlotTable).build();

	final TaskExecutor taskExecutor = createTaskExecutor(taskManagerServices);

	final TestingResourceManagerGateway testingResourceManagerGateway = new TestingResourceManagerGateway();

	final BlockingQueue<AllocationID> allocationsNotifiedFree = new ArrayBlockingQueue<>(2);

	OneShotLatch initialSlotReporting = new OneShotLatch();
	testingResourceManagerGateway.setSendSlotReportFunction(resourceIDInstanceIDSlotReportTuple3 -> {
		initialSlotReporting.trigger();
		return CompletableFuture.completedFuture(Acknowledge.get());

	});

	testingResourceManagerGateway.setNotifySlotAvailableConsumer(instanceIDSlotIDAllocationIDTuple3 ->
			allocationsNotifiedFree.offer(instanceIDSlotIDAllocationIDTuple3.f2));

	rpc.registerGateway(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway);
	resourceManagerLeaderRetriever.notifyListener(testingResourceManagerGateway.getAddress(), testingResourceManagerGateway.getFencingToken().toUUID());

	final BlockingQueue<AllocationID> failedSlotFutures = new ArrayBlockingQueue<>(2);
	final ResourceID jobManagerResourceId = ResourceID.generate();
	final TestingJobMasterGateway jobMasterGateway = new TestingJobMasterGatewayBuilder()
			.setFailSlotConsumer((resourceID, allocationID, throwable) ->
				failedSlotFutures.offer(allocationID))
			.setOfferSlotsFunction((resourceID, slotOffers) -> CompletableFuture.completedFuture(new ArrayList<>(slotOffers)))
			.setRegisterTaskManagerFunction((ignoredA, ignoredB) -> CompletableFuture.completedFuture(new JMTMRegistrationSuccess(jobManagerResourceId)))
			.build();
	final String jobManagerAddress = jobMasterGateway.getAddress();
	rpc.registerGateway(jobManagerAddress, jobMasterGateway);
	jobManagerLeaderRetriever.notifyListener(jobManagerAddress, jobMasterGateway.getFencingToken().toUUID());

	taskExecutor.start();

	try {
		final TaskExecutorGateway taskExecutorGateway = taskExecutor.getSelfGateway(TaskExecutorGateway.class);

		initialSlotReporting.await();

		final SlotID slotId1 = new SlotID(taskExecutor.getResourceID(), 0);
		final SlotID slotId2 = new SlotID(taskExecutor.getResourceID(), 1);
		final AllocationID allocationIdInBoth = new AllocationID();
		final AllocationID allocationIdOnlyInJM = new AllocationID();
		final AllocationID allocationIdOnlyInTM = new AllocationID();

		taskExecutorGateway.requestSlot(slotId1, jobId, allocationIdInBoth, ResourceProfile.ZERO, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);
		taskExecutorGateway.requestSlot(slotId2, jobId, allocationIdOnlyInTM, ResourceProfile.ZERO, "foobar", testingResourceManagerGateway.getFencingToken(), timeout);

		activeSlots.await();

		List<AllocatedSlotInfo> allocatedSlotInfos = Arrays.asList(
				new AllocatedSlotInfo(0, allocationIdInBoth),
				new AllocatedSlotInfo(1, allocationIdOnlyInJM)
		);
		AllocatedSlotReport allocatedSlotReport = new AllocatedSlotReport(jobId, allocatedSlotInfos);
		taskExecutorGateway.heartbeatFromJobManager(jobManagerResourceId, allocatedSlotReport);

		assertThat(failedSlotFutures.take(), is(allocationIdOnlyInJM));
		assertThat(allocationsNotifiedFree.take(), is(allocationIdOnlyInTM));
		assertThat(failedSlotFutures.poll(5L, TimeUnit.MILLISECONDS), nullValue());
		assertThat(allocationsNotifiedFree.poll(5L, TimeUnit.MILLISECONDS), nullValue());
	} finally {
		RpcUtils.terminateRpcEndpoint(taskExecutor, timeout);
	}
}