Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#min()

The following examples show how to use org.apache.hadoop.yarn.util.resource.Resources#min() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: AbstractCSQueue.java From hadoop with Apache License 2.0

6 votes

private Resource getCurrentLimitResource(String nodeLabel,
    Resource clusterResource, ResourceLimits currentResourceLimits) {
  /*
   * Current limit resource: For labeled resource: limit = queue-max-resource
   * (TODO, this part need update when we support labeled-limit) For
   * non-labeled resource: limit = min(queue-max-resource,
   * limit-set-by-parent)
   */
  Resource queueMaxResource =
      Resources.multiplyAndNormalizeDown(resourceCalculator,
          labelManager.getResourceByLabel(nodeLabel, clusterResource),
          queueCapacities.getAbsoluteMaximumCapacity(nodeLabel), minimumAllocation);
  if (nodeLabel.equals(RMNodeLabelsManager.NO_LABEL)) {
    return Resources.min(resourceCalculator, clusterResource,
        queueMaxResource, currentResourceLimits.getLimit());
  }
  return queueMaxResource;
}

Example 2

Source File: AbstractCSQueue.java From big-c with Apache License 2.0

6 votes

private Resource getCurrentLimitResource(String nodeLabel,
    Resource clusterResource, ResourceLimits currentResourceLimits) {
  /*
   * Current limit resource: For labeled resource: limit = queue-max-resource
   * (TODO, this part need update when we support labeled-limit) For
   * non-labeled resource: limit = min(queue-max-resource,
   * limit-set-by-parent)
   */
  Resource queueMaxResource =
      Resources.multiplyAndNormalizeDown(resourceCalculator,
          labelManager.getResourceByLabel(nodeLabel, clusterResource),
          queueCapacities.getAbsoluteMaximumCapacity(nodeLabel), minimumAllocation);
  if (nodeLabel.equals(RMNodeLabelsManager.NO_LABEL)) {
    return Resources.min(resourceCalculator, clusterResource,
        queueMaxResource, currentResourceLimits.getLimit());
  }
  return queueMaxResource;
}

Example 3

Source File: FairSharePolicy.java From hadoop with Apache License 2.0

5 votes

@Override
public int compare(Schedulable s1, Schedulable s2) {
  double minShareRatio1, minShareRatio2;
  double useToWeightRatio1, useToWeightRatio2;
  Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
      s1.getMinShare(), s1.getDemand());
  Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
      s2.getMinShare(), s2.getDemand());
  boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
      s1.getResourceUsage(), minShare1);
  boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
      s2.getResourceUsage(), minShare2);
  minShareRatio1 = (double) s1.getResourceUsage().getMemory()
      / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemory();
  minShareRatio2 = (double) s2.getResourceUsage().getMemory()
      / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemory();
  useToWeightRatio1 = s1.getResourceUsage().getMemory() /
      s1.getWeights().getWeight(ResourceType.MEMORY);
  useToWeightRatio2 = s2.getResourceUsage().getMemory() /
      s2.getWeights().getWeight(ResourceType.MEMORY);
  int res = 0;
  if (s1Needy && !s2Needy)
    res = -1;
  else if (s2Needy && !s1Needy)
    res = 1;
  else if (s1Needy && s2Needy)
    res = (int) Math.signum(minShareRatio1 - minShareRatio2);
  else
    // Neither schedulable is needy
    res = (int) Math.signum(useToWeightRatio1 - useToWeightRatio2);
  if (res == 0) {
    // Apps are tied in fairness ratio. Break the tie by submit time and job
    // name to get a deterministic ordering, which is useful for unit tests.
    res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
    if (res == 0)
      res = s1.getName().compareTo(s2.getName());
  }
  return res;
}

Example 4

Source File: ParentQueue.java From hadoop with Apache License 2.0

5 votes

private ResourceLimits getResourceLimitsOfChild(CSQueue child,
    Resource clusterResource, ResourceLimits parentLimits) {
  // Set resource-limit of a given child, child.limit =
  // min(my.limit - my.used + child.used, child.max)

  // Parent available resource = parent-limit - parent-used-resource
  Resource parentMaxAvailableResource =
      Resources.subtract(parentLimits.getLimit(), getUsedResources());

  // Child's limit = parent-available-resource + child-used
  Resource childLimit =
      Resources.add(parentMaxAvailableResource, child.getUsedResources());

  // Get child's max resource
  Resource childConfiguredMaxResource =
      Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
          .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
          child.getAbsoluteMaximumCapacity(), minimumAllocation);

  // Child's limit should be capped by child configured max resource
  childLimit =
      Resources.min(resourceCalculator, clusterResource, childLimit,
          childConfiguredMaxResource);

  // Normalize before return
  childLimit =
      Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

  return new ResourceLimits(childLimit);
}

Example 5

Source File: LeafQueue.java From hadoop with Apache License 2.0

5 votes

private Resource getHeadroom(User user, Resource currentResourceLimit,
    Resource clusterResource, Resource userLimit) {
  /** 
   * Headroom is:
   *    min(
   *        min(userLimit, queueMaxCap) - userConsumed,
   *        queueMaxCap - queueUsedResources
   *       )
   * 
   * ( which can be expressed as, 
   *  min (userLimit - userConsumed, queuMaxCap - userConsumed, 
   *    queueMaxCap - queueUsedResources)
   *  )
   *
   * given that queueUsedResources >= userConsumed, this simplifies to
   *
   * >> min (userlimit - userConsumed,   queueMaxCap - queueUsedResources) << 
   *
   */
  Resource headroom = 
    Resources.min(resourceCalculator, clusterResource,
      Resources.subtract(userLimit, user.getUsed()),
      Resources.subtract(currentResourceLimit, queueUsage.getUsed())
      );
  // Normalize it before return
  headroom =
      Resources.roundDown(resourceCalculator, headroom, minimumAllocation);
  return headroom;
}

Example 6

Source File: ProportionalCapacityPreemptionPolicy.java From hadoop with Apache License 2.0

5 votes

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0, 0));
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  Resource accepted = 
      Resources.min(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.min(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  Resource remain = Resources.subtract(avail, accepted);
  Resources.addTo(idealAssigned, accepted);
  return remain;
}

Example 7

Source File: FairSharePolicy.java From big-c with Apache License 2.0

5 votes

@Override
public int compare(Schedulable s1, Schedulable s2) {
  double minShareRatio1, minShareRatio2;
  double useToWeightRatio1, useToWeightRatio2;
  Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null,
      s1.getMinShare(), s1.getDemand());
  Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null,
      s2.getMinShare(), s2.getDemand());
  boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
      s1.getResourceUsage(), minShare1);
  boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null,
      s2.getResourceUsage(), minShare2);
  minShareRatio1 = (double) s1.getResourceUsage().getMemory()
      / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemory();
  minShareRatio2 = (double) s2.getResourceUsage().getMemory()
      / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemory();
  useToWeightRatio1 = s1.getResourceUsage().getMemory() /
      s1.getWeights().getWeight(ResourceType.MEMORY);
  useToWeightRatio2 = s2.getResourceUsage().getMemory() /
      s2.getWeights().getWeight(ResourceType.MEMORY);
  int res = 0;
  if (s1Needy && !s2Needy)
    res = -1;
  else if (s2Needy && !s1Needy)
    res = 1;
  else if (s1Needy && s2Needy)
    res = (int) Math.signum(minShareRatio1 - minShareRatio2);
  else
    // Neither schedulable is needy
    res = (int) Math.signum(useToWeightRatio1 - useToWeightRatio2);
  if (res == 0) {
    // Apps are tied in fairness ratio. Break the tie by submit time and job
    // name to get a deterministic ordering, which is useful for unit tests.
    res = (int) Math.signum(s1.getStartTime() - s2.getStartTime());
    if (res == 0)
      res = s1.getName().compareTo(s2.getName());
  }
  return res;
}

Example 8

Source File: ParentQueue.java From big-c with Apache License 2.0

5 votes

private ResourceLimits getResourceLimitsOfChild(CSQueue child,
     Resource clusterResource, ResourceLimits parentLimits) {
   // Set resource-limit of a given child, child.limit =
   // min(my.limit - my.used + child.used, child.max)//why ?
// my.limit-my-used+child.used 等于在父队列中除去别的子对列所用资源后剩下的资源，有可能别的子对列所用资源超过了他们自身的限制

   // Parent available resource = parent-limit - parent-used-resource
   Resource parentMaxAvailableResource =
       Resources.subtract(parentLimits.getLimit(), getUsedResources());
   
   // Child's limit = parent-available-resource + child-used
   Resource childLimit =
       Resources.add(parentMaxAvailableResource, child.getUsedResources());

   // Get child's max resource
   Resource childConfiguredMaxResource =
       Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
           .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
           child.getAbsoluteMaximumCapacity(), minimumAllocation);

   // Child's limit should be capped by child configured max resource
   childLimit =
       Resources.min(resourceCalculator, clusterResource, childLimit,
           childConfiguredMaxResource);

   // Normalize before return
   childLimit =
       Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

   return new ResourceLimits(childLimit);
 }

Example 9

Source File: LeafQueue.java From big-c with Apache License 2.0

5 votes

private Resource getHeadroom(User user, Resource currentResourceLimit, //queueMaxCap 是不是hard limit
    Resource clusterResource, Resource userLimit) {
  /** 
   * Headroom is:
   *    min(
   *        min(userLimit, queueMaxCap) - userConsumed,
   *        queueMaxCap - queueUsedResources
   *       )
   * 
   * ( which can be expressed as, 
   *  min (userLimit - userConsumed, queuMaxCap - userConsumed, 
   *    queueMaxCap - queueUsedResources)
   *  )
   *
   * given that queueUsedResources >= userConsumed, this simplifies to
   *
   * >> min (userlimit - userConsumed,   queueMaxCap - queueUsedResources) << 
   *
   */
  Resource headroom = 
    Resources.min(resourceCalculator, clusterResource,
      Resources.subtract(userLimit, user.getUsed()),
      Resources.subtract(currentResourceLimit, queueUsage.getUsed())
      );
  // Normalize it before return
  headroom =
      Resources.roundDown(resourceCalculator, headroom, minimumAllocation);
  return headroom;
}

Example 10

Source File: FSLeafQueue.java From hadoop with Apache License 2.0

4 votes

private boolean isStarved(Resource share) {
  Resource desiredShare = Resources.min(scheduler.getResourceCalculator(),
      scheduler.getClusterResource(), share, getDemand());
  return Resources.lessThan(scheduler.getResourceCalculator(),
      scheduler.getClusterResource(), getResourceUsage(), desiredShare);
}

Example 11

Source File: FSLeafQueue.java From big-c with Apache License 2.0

4 votes

private boolean isStarved(Resource share) {
  Resource desiredShare = Resources.min(scheduler.getResourceCalculator(),
      scheduler.getClusterResource(), share, getDemand());
  return Resources.lessThan(scheduler.getResourceCalculator(),
      scheduler.getClusterResource(), getResourceUsage(), desiredShare);
}