Java Code Examples for org.apache.flink.runtime.taskexecutor.slot.SlotOffer#getAllocationId()

The following examples show how to use org.apache.flink.runtime.taskexecutor.slot.SlotOffer#getAllocationId() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SlotPoolImpl.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// check whether we have request waiting for this slot
	PendingRequest pendingRequest = pendingRequests.removeKeyB(allocationID);
	if (pendingRequest != null) {
		// we were waiting for this!
		allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot);

		if (!pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot)) {
			// we could not complete the pending slot future --> try to fulfill another pending request
			allocatedSlots.remove(pendingRequest.getSlotRequestId());
			tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
		} else {
			log.debug("Fulfilled slot request [{}] with allocated slot [{}].", pendingRequest.getSlotRequestId(), allocationID);
		}
	}
	else {
		// we were actually not waiting for this:
		//   - could be that this request had been fulfilled
		//   - we are receiving the slots from TaskManagers after becoming leaders
		tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
	}

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 
Example 2
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// check whether we have request waiting for this slot
	PendingRequest pendingRequest = pendingRequests.removeKeyB(allocationID);
	if (pendingRequest != null) {
		// we were waiting for this!
		allocatedSlots.add(pendingRequest.getSlotRequestId(), allocatedSlot);

		if (!pendingRequest.getAllocatedSlotFuture().complete(allocatedSlot)) {
			// we could not complete the pending slot future --> try to fulfill another pending request
			allocatedSlots.remove(pendingRequest.getSlotRequestId());
			tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
		} else {
			log.debug("Fulfilled slot request [{}] with allocated slot [{}].", pendingRequest.getSlotRequestId(), allocationID);
		}
	}
	else {
		// we were actually not waiting for this:
		//   - could be that this request had been fulfilled
		//   - we are receiving the slots from TaskManagers after becoming leaders
		tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);
	}

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}
 
Example 3
Source File: SlotPoolImpl.java    From flink with Apache License 2.0 4 votes vote down vote up
/**
 * Slot offering by TaskExecutor with AllocationID. The AllocationID is originally generated by this pool and
 * transfer through the ResourceManager to TaskManager. We use it to distinguish the different allocation
 * we issued. Slot offering may be rejected if we find something mismatching or there is actually no pending
 * request waiting for this slot (maybe fulfilled by some other returned slot).
 *
 * @param taskManagerLocation location from where the offer comes from
 * @param taskManagerGateway TaskManager gateway
 * @param slotOffer the offered slot
 * @return True if we accept the offering
 */
boolean offerSlot(
		final TaskManagerLocation taskManagerLocation,
		final TaskManagerGateway taskManagerGateway,
		final SlotOffer slotOffer) {

	componentMainThreadExecutor.assertRunningInMainThread();

	// check if this TaskManager is valid
	final ResourceID resourceID = taskManagerLocation.getResourceID();
	final AllocationID allocationID = slotOffer.getAllocationId();

	if (!registeredTaskManagers.contains(resourceID)) {
		log.debug("Received outdated slot offering [{}] from unregistered TaskManager: {}",
				slotOffer.getAllocationId(), taskManagerLocation);
		return false;
	}

	// check whether we have already using this slot
	AllocatedSlot existingSlot;
	if ((existingSlot = allocatedSlots.get(allocationID)) != null ||
		(existingSlot = availableSlots.get(allocationID)) != null) {

		// we need to figure out if this is a repeated offer for the exact same slot,
		// or another offer that comes from a different TaskManager after the ResourceManager
		// re-tried the request

		// we write this in terms of comparing slot IDs, because the Slot IDs are the identifiers of
		// the actual slots on the TaskManagers
		// Note: The slotOffer should have the SlotID
		final SlotID existingSlotId = existingSlot.getSlotId();
		final SlotID newSlotId = new SlotID(taskManagerLocation.getResourceID(), slotOffer.getSlotIndex());

		if (existingSlotId.equals(newSlotId)) {
			log.info("Received repeated offer for slot [{}]. Ignoring.", allocationID);

			// return true here so that the sender will get a positive acknowledgement to the retry
			// and mark the offering as a success
			return true;
		} else {
			// the allocation has been fulfilled by another slot, reject the offer so the task executor
			// will offer the slot to the resource manager
			return false;
		}
	}

	final AllocatedSlot allocatedSlot = new AllocatedSlot(
		allocationID,
		taskManagerLocation,
		slotOffer.getSlotIndex(),
		slotOffer.getResourceProfile(),
		taskManagerGateway);

	// use the slot to fulfill pending request, in requested order
	tryFulfillSlotRequestOrMakeAvailable(allocatedSlot);

	// we accepted the request in any case. slot will be released after it idled for
	// too long and timed out
	return true;
}