org.apache.flink.runtime.instance.SlotSharingGroupId Java Examples

The following examples show how to use org.apache.flink.runtime.instance.SlotSharingGroupId. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestingLogicalSlot.java    From flink with Apache License 2.0 6 votes vote down vote up 
TestingLogicalSlot(
		TaskManagerLocation taskManagerLocation,
		TaskManagerGateway taskManagerGateway,
		int slotNumber,
		AllocationID allocationId,
		SlotRequestId slotRequestId,
		SlotSharingGroupId slotSharingGroupId,
		boolean automaticallyCompleteReleaseFuture,
		SlotOwner slotOwner) {

	this.taskManagerLocation = Preconditions.checkNotNull(taskManagerLocation);
	this.taskManagerGateway = Preconditions.checkNotNull(taskManagerGateway);
	this.payloadReference = new AtomicReference<>();
	this.slotNumber = slotNumber;
	this.allocationId = Preconditions.checkNotNull(allocationId);
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotSharingGroupId = Preconditions.checkNotNull(slotSharingGroupId);
	this.releaseFuture = new CompletableFuture<>();
	this.automaticallyCompleteReleaseFuture = automaticallyCompleteReleaseFuture;
	this.slotOwner = slotOwner;
}
Example #2
Source File: SingleLogicalSlot.java    From flink with Apache License 2.0 6 votes vote down vote up 
public SingleLogicalSlot(
		SlotRequestId slotRequestId,
		SlotContext slotContext,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Locality locality,
		SlotOwner slotOwner) {
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotContext = Preconditions.checkNotNull(slotContext);
	this.slotSharingGroupId = slotSharingGroupId;
	this.locality = Preconditions.checkNotNull(locality);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
	this.releaseFuture = new CompletableFuture<>();

	this.state = State.ALIVE;
	this.payload = null;
}
Example #3
Source File: SchedulerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
Example #4
Source File: SchedulerImpl.java    From flink with Apache License 2.0 6 votes vote down vote up 
private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
Example #5
Source File: SingleLogicalSlot.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
public SingleLogicalSlot(
		SlotRequestId slotRequestId,
		SlotContext slotContext,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Locality locality,
		SlotOwner slotOwner) {
	this.slotRequestId = Preconditions.checkNotNull(slotRequestId);
	this.slotContext = Preconditions.checkNotNull(slotContext);
	this.slotSharingGroupId = slotSharingGroupId;
	this.locality = Preconditions.checkNotNull(locality);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);
	this.releaseFuture = new CompletableFuture<>();

	this.state = State.ALIVE;
	this.payload = null;
}
Example #6
Source File: SchedulerTestBase.java    From flink with Apache License 2.0 6 votes vote down vote up 
public int getNumberOfAvailableSlotsForGroup(SlotSharingGroupId slotSharingGroupId, JobVertexID jobVertexId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		int availableSlots = 0;

		for (SlotSharingManager.MultiTaskSlot multiTaskSlot : multiTaskSlotManager.getResolvedRootSlots()) {
			if (!multiTaskSlot.contains(jobVertexId)) {
				availableSlots++;
			}
		}

		return availableSlots;
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotmanager registered under " + slotSharingGroupId + '.');
	}
}
Example #7
Source File: SchedulerImpl.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up 
private void releaseSharedSlot(
	@Nonnull SlotRequestId slotRequestId,
	@Nonnull SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	final SlotSharingManager multiTaskSlotManager = slotSharingManagers.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		final SlotSharingManager.TaskSlot taskSlot = multiTaskSlotManager.getTaskSlot(slotRequestId);

		if (taskSlot != null) {
			taskSlot.release(cause);
		} else {
			log.debug("Could not find slot [{}] in slot sharing group {}. Ignoring release slot request.", slotRequestId, slotSharingGroupId);
		}
	} else {
		log.debug("Could not find slot sharing group {}. Ignoring release slot request.", slotSharingGroupId);
	}
}
Example #8
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
/**
 * Tests that returned slot futures are failed if the allocation request is failed.
 */
@Test
public void testFailingQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			new SlotSharingGroupId(),
			null),
		true,
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	final AllocationID allocationId = allocationIdFuture.get();

	// this should fail the returned logical slot future
	final SlotPool slotPoolGateway = slotPoolResource.getSlotPool();
	slotPoolGateway.failAllocation(allocationId, new FlinkException("Testing Exception"));

	try {
		logicalSlotFuture.get();
		fail("The slot future should have failed.");
	} catch (ExecutionException ee) {
		assertTrue(ExceptionUtils.findThrowable(ee, FlinkException.class).isPresent());
	}
}
Example #9
Source File: ProgrammedSlotProvider.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
		SlotRequestId slotRequestId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		Throwable cause) {
	canceledSlotRequests.add(slotRequestId);
}
Example #10
Source File: SchedulerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
Example #11
Source File: SchedulerTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public int getNumberOfSharedSlots(SlotSharingGroupId slotSharingGroupId) {
	final SlotSharingManager multiTaskSlotManager = slotSharingManagersMap.get(slotSharingGroupId);

	if (multiTaskSlotManager != null) {
		return multiTaskSlotManager.getResolvedRootSlots().size();
	} else {
		throw new FlinkRuntimeException("No MultiTaskSlotManager registered under " + slotSharingGroupId + '.');
	}
}
Example #12
Source File: SchedulerTestBase.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}
Example #13
Source File: SimpleSlotProvider.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(SlotRequestId slotRequestId, @Nullable SlotSharingGroupId slotSharingGroupId, Throwable cause) {
	synchronized (lock) {
		final SlotContext slotContext = allocatedSlots.remove(slotRequestId);

		if (slotContext != null) {
			slots.add(slotContext);
		} else {
			throw new FlinkRuntimeException("Unknown slot request id " + slotRequestId + '.');
		}
	}
}
Example #14
Source File: TestingSlotProvider.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(SlotRequestId slotRequestId, @Nullable SlotSharingGroupId slotSharingGroupId, Throwable cause) {
	final CompletableFuture<LogicalSlot> slotFuture = slotFutures.remove(slotRequestId);
	slotFuture.cancel(false);

	slotCanceller.accept(slotRequestId);
}
Example #15
Source File: SlotPoolSlotSharingTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void testSingleQueuedSharedSlotScheduling() throws Exception {
	final CompletableFuture<AllocationID> allocationIdFuture = new CompletableFuture<>();
	final TestingResourceManagerGateway testingResourceManagerGateway = slotPoolResource.getTestingResourceManagerGateway();
	testingResourceManagerGateway.setRequestSlotConsumer(
		(SlotRequest slotRequest) -> allocationIdFuture.complete(slotRequest.getAllocationId()));

	LocalTaskManagerLocation taskManagerLocation = new LocalTaskManagerLocation();
	final SlotPoolImpl slotPool = slotPoolResource.getSlotPool();
	slotPool.registerTaskManager(taskManagerLocation.getResourceID());

	SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();
	final SlotProvider slotProvider = slotPoolResource.getSlotProvider();
	CompletableFuture<LogicalSlot> logicalSlotFuture = slotProvider.allocateSlot(
		new ScheduledUnit(
			new JobVertexID(),
			slotSharingGroupId,
			null),
		SlotProfile.noRequirements(),
		TestingUtils.infiniteTime());

	assertFalse(logicalSlotFuture.isDone());

	final AllocationID allocationId = allocationIdFuture.get();

	boolean booleanCompletableFuture = slotPool.offerSlot(
		taskManagerLocation,
		new SimpleAckingTaskManagerGateway(),
		new SlotOffer(
			allocationId,
			0,
			ResourceProfile.ANY));

	assertTrue(booleanCompletableFuture);

	final LogicalSlot logicalSlot = logicalSlotFuture.get();

	assertEquals(slotSharingGroupId, logicalSlot.getSlotSharingGroupId());
}
Example #16
Source File: ScheduledUnit.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
public ScheduledUnit(
		JobVertexID jobVertexId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		null,
		jobVertexId,
		slotSharingGroupId,
		coLocationConstraint);
}
Example #17
Source File: ExecutionVertexSchedulingRequirements.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionVertexSchedulingRequirements(
		ExecutionVertexID executionVertexId,
		@Nullable AllocationID previousAllocationId,
		ResourceProfile taskResourceProfile,
		ResourceProfile physicalSlotResourceProfile,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this.executionVertexId = checkNotNull(executionVertexId);
	this.previousAllocationId = previousAllocationId;
	this.taskResourceProfile = checkNotNull(taskResourceProfile);
	this.physicalSlotResourceProfile = checkNotNull(physicalSlotResourceProfile);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;
}
Example #18
Source File: DefaultExecutionSlotAllocatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
		final SlotRequestId slotRequestId,
		@Nullable final SlotSharingGroupId slotSharingGroupId,
		final Throwable cause) {
	cancelledSlotRequestIds.add(slotRequestId);
}
Example #19
Source File: SchedulerImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void returnLogicalSlot(LogicalSlot logicalSlot) {
	SlotRequestId slotRequestId = logicalSlot.getSlotRequestId();
	SlotSharingGroupId slotSharingGroupId = logicalSlot.getSlotSharingGroupId();
	FlinkException cause = new FlinkException("Slot is being returned to the SlotPool.");
	cancelSlotRequest(slotRequestId, slotSharingGroupId, cause);
}
Example #20
Source File: SlotPoolSlotSpreadOutTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Test
public void allocateSharedSlot_withNoRequirements_selectsSlotsSoThatWorkloadIsSpreadOut() {
	final int numberSlotsPerTaskExecutor = 2;
	final int numberTaskExecutors = 2;
	final int numberSlots = numberTaskExecutors * numberSlotsPerTaskExecutor;

	registerTaskExecutors(numberTaskExecutors, numberSlotsPerTaskExecutor);

	final JobVertexID sourceJobVertexId = new JobVertexID();
	final JobVertexID sinkJobVertexId = new JobVertexID();
	final SlotSharingGroupId slotSharingGroupId = new SlotSharingGroupId();

	final List<ScheduledUnit> sourceScheduledUnits = IntStream.range(0, numberSlots)
		.mapToObj(ignored -> createSharedSlotRequest(sourceJobVertexId, slotSharingGroupId))
		.collect(Collectors.toList());

	final List<ScheduledUnit> sinkScheduledUnits = IntStream.range(0, numberTaskExecutors)
		.mapToObj(ignored -> createSharedSlotRequest(sinkJobVertexId, slotSharingGroupId))
		.collect(Collectors.toList());

	sourceScheduledUnits.forEach(this::allocateSlot);
	final Set<TaskManagerLocation> sinkLocations = sinkScheduledUnits.stream()
		.map(this::allocateSlot)
		.map(this::getTaskManagerLocation)
		.collect(Collectors.toSet());

	// verify that the sinks have been evenly spread across the available TaskExecutors
	assertThat(sinkLocations, hasSize(numberTaskExecutors));
}
Example #21
Source File: SchedulerImpl.java    From Flink-CEPplus with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
Example #22
Source File: DefaultExecutionSlotAllocatorTest.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
		final SlotRequestId slotRequestId,
		@Nullable final SlotSharingGroupId slotSharingGroupId,
		final Throwable cause) {
	cancelledSlotRequestIds.add(slotRequestId);
}
Example #23
Source File: SlotSharingManager.java    From flink with Apache License 2.0 5 votes vote down vote up 
SlotSharingManager(
		SlotSharingGroupId slotSharingGroupId,
		AllocatedSlotActions allocatedSlotActions,
		SlotOwner slotOwner) {
	this.slotSharingGroupId = Preconditions.checkNotNull(slotSharingGroupId);
	this.allocatedSlotActions = Preconditions.checkNotNull(allocatedSlotActions);
	this.slotOwner = Preconditions.checkNotNull(slotOwner);

	allTaskSlots = new HashMap<>(16);
	unresolvedRootSlots = new HashMap<>(16);
	resolvedRootSlots = new HashMap<>(16);
}
Example #24
Source File: ExecutionVertexSchedulingRequirements.java    From flink with Apache License 2.0 5 votes vote down vote up 
private ExecutionVertexSchedulingRequirements(
		ExecutionVertexID executionVertexId,
		@Nullable AllocationID previousAllocationId,
		ResourceProfile resourceProfile,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint,
		Collection<TaskManagerLocation> preferredLocations) {
	this.executionVertexId = checkNotNull(executionVertexId);
	this.previousAllocationId = previousAllocationId;
	this.resourceProfile = checkNotNull(resourceProfile);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;
	this.preferredLocations = checkNotNull(preferredLocations);
}
Example #25
Source File: SchedulerImpl.java    From flink with Apache License 2.0 5 votes vote down vote up 
@Override
public void cancelSlotRequest(
	SlotRequestId slotRequestId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	Throwable cause) {

	componentMainThreadExecutor.assertRunningInMainThread();

	if (slotSharingGroupId != null) {
		releaseSharedSlot(slotRequestId, slotSharingGroupId, cause);
	} else {
		slotPool.releaseSlot(slotRequestId, cause);
	}
}
Example #26
Source File: ScheduledUnit.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ScheduledUnit(Execution task, @Nullable SlotSharingGroupId slotSharingGroupId) {
	this(
		Preconditions.checkNotNull(task),
		task.getVertex().getJobvertexId(),
		slotSharingGroupId,
		null);
}
Example #27
Source File: ScheduledUnit.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ScheduledUnit(
		Execution task,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		Preconditions.checkNotNull(task),
		task.getVertex().getJobvertexId(),
		slotSharingGroupId,
		coLocationConstraint);
}
Example #28
Source File: ScheduledUnit.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ScheduledUnit(
		JobVertexID jobVertexId,
		@Nullable SlotSharingGroupId slotSharingGroupId,
		@Nullable CoLocationConstraint coLocationConstraint) {
	this(
		null,
		jobVertexId,
		slotSharingGroupId,
		coLocationConstraint);
}
Example #29
Source File: ScheduledUnit.java    From flink with Apache License 2.0 5 votes vote down vote up 
public ScheduledUnit(
	@Nullable Execution task,
	JobVertexID jobVertexId,
	@Nullable SlotSharingGroupId slotSharingGroupId,
	@Nullable CoLocationConstraint coLocationConstraint) {

	this.vertexExecution = task;
	this.jobVertexId = Preconditions.checkNotNull(jobVertexId);
	this.slotSharingGroupId = slotSharingGroupId;
	this.coLocationConstraint = coLocationConstraint;

}
Example #30
Source File: SchedulerTestBase.java    From flink with Apache License 2.0 5 votes vote down vote up 
public TestingScheduler(
	@Nonnull Map<SlotSharingGroupId, SlotSharingManager> slotSharingManagersMap,
	@Nonnull SlotSelectionStrategy slotSelectionStrategy,
	@Nonnull SlotPool slotPoolGateway) {

	super(slotSelectionStrategy, slotPoolGateway, slotSharingManagersMap);
	this.slotSharingManagersMap = slotSharingManagersMap;
}