Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#fitsIn()

/**
 * Called when this application already has an existing reservation on the
 * given node.  Sees whether we can turn the reservation into an allocation.
 * Also checks whether the application needs the reservation anymore, and
 * releases it if not.
 *
 * @param node
 *     Node that the application has an existing reservation on
 */
public Resource assignReservedContainer(FSSchedulerNode node) {
  RMContainer rmContainer = node.getReservedContainer();
  Priority priority = rmContainer.getReservedPriority();

  // Make sure the application still needs requests at this priority
  if (getTotalRequiredResources(priority) == 0) {
    unreserve(priority, node);
    return Resources.none();
  }

  // Fail early if the reserved container won't fit.
  // Note that we have an assumption here that there's only one container size
  // per priority.
  if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(),
      node.getAvailableResource())) {
    return Resources.none();
  }

  return assignContainer(node, true);
}
 

/**
 * Called when this application already has an existing reservation on the
 * given node.  Sees whether we can turn the reservation into an allocation.
 * Also checks whether the application needs the reservation anymore, and
 * releases it if not.
 *
 * @param node
 *     Node that the application has an existing reservation on
 */
public Resource assignReservedContainer(FSSchedulerNode node) {
  RMContainer rmContainer = node.getReservedContainer();
  Priority priority = rmContainer.getReservedPriority();

  // Make sure the application still needs requests at this priority
  if (getTotalRequiredResources(priority) == 0) {
    unreserve(priority, node);
    return Resources.none();
  }

  // Fail early if the reserved container won't fit.
  // Note that we have an assumption here that there's only one container size
  // per priority.
  if (!Resources.fitsIn(node.getReservedContainer().getReservedResource(),
      node.getAvailableResource())) {
    return Resources.none();
  }

  return assignContainer(node, true);
}
 

/**
 * Helper method to check if the queue should attempt assigning resources
 * 
 * @return true if check passes (can assign) or false otherwise
 */
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
  if (!Resources.fitsIn(getResourceUsage(),
      scheduler.getAllocationConfiguration().getMaxResources(getName()))
      || node.getReservedContainer() != null) {
    return false;
  }
  return true;
}
 

void continuousSchedulingAttempt() throws InterruptedException {
  long start = getClock().getTime();
  List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
  // Sort the nodes by space available on them, so that we offer
  // containers on emptier nodes first, facilitating an even spread. This
  // requires holding the scheduler lock, so that the space available on a
  // node doesn't change during the sort.
  synchronized (this) {
    Collections.sort(nodeIdList, nodeAvailableResourceComparator);
  }

  // iterate all nodes
  for (NodeId nodeId : nodeIdList) {
    FSSchedulerNode node = getFSSchedulerNode(nodeId);
    try {
      if (node != null && Resources.fitsIn(minimumAllocation,
          node.getAvailableResource())) {
        attemptScheduling(node);
      }
    } catch (Throwable ex) {
      LOG.error("Error while attempting scheduling for node " + node +
          ": " + ex.toString(), ex);
    }
  }

  long duration = getClock().getTime() - start;
  fsOpDurations.addContinuousSchedulingRunDuration(duration);
}
 

static boolean fitsInMaxShare(FSQueue queue, Resource
    additionalResource) {
  Resource usagePlusAddition =
      Resources.add(queue.getResourceUsage(), additionalResource);

  if (!Resources.fitsIn(usagePlusAddition, queue.getMaxShare())) {
    return false;
  }
  
  FSQueue parentQueue = queue.getParent();
  if (parentQueue != null) {
    return fitsInMaxShare(parentQueue, additionalResource);
  }
  return true;
}
 

private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app,
    FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException {
  String queueName = targetQueue.getQueueName();
  ApplicationAttemptId appAttId = app.getApplicationAttemptId();
  // When checking maxResources and maxRunningApps, only need to consider
  // queues before the lowest common ancestor of the two queues because the
  // total running apps in queues above will not be changed.
  FSQueue lowestCommonAncestor = findLowestCommonAncestorQueue(oldQueue,
      targetQueue);
  Resource consumption = app.getCurrentConsumption();
  
  // Check whether the move would go over maxRunningApps or maxShare
  FSQueue cur = targetQueue;
  while (cur != lowestCommonAncestor) {
    // maxRunningApps
    if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) {
      throw new YarnException("Moving app attempt " + appAttId + " to queue "
          + queueName + " would violate queue maxRunningApps constraints on"
          + " queue " + cur.getQueueName());
    }
    
    // maxShare
    if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption),
        cur.getMaxShare())) {
      throw new YarnException("Moving app attempt " + appAttId + " to queue "
          + queueName + " would violate queue maxShare constraints on"
          + " queue " + cur.getQueueName());
    }
    
    cur = cur.getParent();
  }
}
 

/**
 * Helper method to check if the queue should attempt assigning resources
 * 
 * @return true if check passes (can assign) or false otherwise
 */
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
  if (!Resources.fitsIn(getResourceUsage(),
      scheduler.getAllocationConfiguration().getMaxResources(getName()))
      || node.getReservedContainer() != null) {
    return false;
  }
  return true;
}
 

void continuousSchedulingAttempt() throws InterruptedException {
  long start = getClock().getTime();
  List<NodeId> nodeIdList = new ArrayList<NodeId>(nodes.keySet());
  // Sort the nodes by space available on them, so that we offer
  // containers on emptier nodes first, facilitating an even spread. This
  // requires holding the scheduler lock, so that the space available on a
  // node doesn't change during the sort.
  synchronized (this) {
    Collections.sort(nodeIdList, nodeAvailableResourceComparator);
  }

  // iterate all nodes
  for (NodeId nodeId : nodeIdList) {
    FSSchedulerNode node = getFSSchedulerNode(nodeId);
    try {
      if (node != null && Resources.fitsIn(minimumAllocation,
          node.getAvailableResource())) {
        attemptScheduling(node);
      }
    } catch (Throwable ex) {
      LOG.error("Error while attempting scheduling for node " + node +
          ": " + ex.toString(), ex);
    }
  }

  long duration = getClock().getTime() - start;
  fsOpDurations.addContinuousSchedulingRunDuration(duration);
}
 

static boolean fitsInMaxShare(FSQueue queue, Resource
    additionalResource) {
  Resource usagePlusAddition =
      Resources.add(queue.getResourceUsage(), additionalResource);

  if (!Resources.fitsIn(usagePlusAddition, queue.getMaxShare())) {
    return false;
  }
  
  FSQueue parentQueue = queue.getParent();
  if (parentQueue != null) {
    return fitsInMaxShare(parentQueue, additionalResource);
  }
  return true;
}
 

private void verifyMoveDoesNotViolateConstraints(FSAppAttempt app,
    FSLeafQueue oldQueue, FSLeafQueue targetQueue) throws YarnException {
  String queueName = targetQueue.getQueueName();
  ApplicationAttemptId appAttId = app.getApplicationAttemptId();
  // When checking maxResources and maxRunningApps, only need to consider
  // queues before the lowest common ancestor of the two queues because the
  // total running apps in queues above will not be changed.
  FSQueue lowestCommonAncestor = findLowestCommonAncestorQueue(oldQueue,
      targetQueue);
  Resource consumption = app.getCurrentConsumption();
  
  // Check whether the move would go over maxRunningApps or maxShare
  FSQueue cur = targetQueue;
  while (cur != lowestCommonAncestor) {
    // maxRunningApps
    if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) {
      throw new YarnException("Moving app attempt " + appAttId + " to queue "
          + queueName + " would violate queue maxRunningApps constraints on"
          + " queue " + cur.getQueueName());
    }
    
    // maxShare
    if (!Resources.fitsIn(Resources.add(cur.getResourceUsage(), consumption),
        cur.getMaxShare())) {
      throw new YarnException("Moving app attempt " + appAttId + " to queue "
          + queueName + " would violate queue maxShare constraints on"
          + " queue " + cur.getQueueName());
    }
    
    cur = cur.getParent();
  }
}
 

@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
  return !Resources.fitsIn(usage, fairShare);
}
 

@Override
public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) {
  return !Resources.fitsIn(usage, maxAMResource);
}
 

/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}
 

@Override
public boolean checkIfUsageOverFairShare(Resource usage, Resource fairShare) {
  return !Resources.fitsIn(usage, fairShare);
}
 

@Override
public boolean checkIfAMResourceUsageOverLimit(Resource usage, Resource maxAMResource) {
  return !Resources.fitsIn(usage, maxAMResource);
}
 

/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}