Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#min()
The following examples show how to use
org.apache.hadoop.yarn.util.resource.Resources#min() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractCSQueue.java From hadoop with Apache License 2.0 | 6 votes |
private Resource getCurrentLimitResource(String nodeLabel, Resource clusterResource, ResourceLimits currentResourceLimits) { /* * Current limit resource: For labeled resource: limit = queue-max-resource * (TODO, this part need update when we support labeled-limit) For * non-labeled resource: limit = min(queue-max-resource, * limit-set-by-parent) */ Resource queueMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, clusterResource), queueCapacities.getAbsoluteMaximumCapacity(nodeLabel), minimumAllocation); if (nodeLabel.equals(RMNodeLabelsManager.NO_LABEL)) { return Resources.min(resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit()); } return queueMaxResource; }
Example 2
Source File: AbstractCSQueue.java From big-c with Apache License 2.0 | 6 votes |
private Resource getCurrentLimitResource(String nodeLabel, Resource clusterResource, ResourceLimits currentResourceLimits) { /* * Current limit resource: For labeled resource: limit = queue-max-resource * (TODO, this part need update when we support labeled-limit) For * non-labeled resource: limit = min(queue-max-resource, * limit-set-by-parent) */ Resource queueMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, clusterResource), queueCapacities.getAbsoluteMaximumCapacity(nodeLabel), minimumAllocation); if (nodeLabel.equals(RMNodeLabelsManager.NO_LABEL)) { return Resources.min(resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit()); } return queueMaxResource; }
Example 3
Source File: FairSharePolicy.java From hadoop with Apache License 2.0 | 5 votes |
@Override public int compare(Schedulable s1, Schedulable s2) { double minShareRatio1, minShareRatio2; double useToWeightRatio1, useToWeightRatio2; Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null, s1.getMinShare(), s1.getDemand()); Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null, s2.getMinShare(), s2.getDemand()); boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null, s1.getResourceUsage(), minShare1); boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null, s2.getResourceUsage(), minShare2); minShareRatio1 = (double) s1.getResourceUsage().getMemory() / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemory(); minShareRatio2 = (double) s2.getResourceUsage().getMemory() / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemory(); useToWeightRatio1 = s1.getResourceUsage().getMemory() / s1.getWeights().getWeight(ResourceType.MEMORY); useToWeightRatio2 = s2.getResourceUsage().getMemory() / s2.getWeights().getWeight(ResourceType.MEMORY); int res = 0; if (s1Needy && !s2Needy) res = -1; else if (s2Needy && !s1Needy) res = 1; else if (s1Needy && s2Needy) res = (int) Math.signum(minShareRatio1 - minShareRatio2); else // Neither schedulable is needy res = (int) Math.signum(useToWeightRatio1 - useToWeightRatio2); if (res == 0) { // Apps are tied in fairness ratio. Break the tie by submit time and job // name to get a deterministic ordering, which is useful for unit tests. res = (int) Math.signum(s1.getStartTime() - s2.getStartTime()); if (res == 0) res = s1.getName().compareTo(s2.getName()); } return res; }
Example 4
Source File: ParentQueue.java From hadoop with Apache License 2.0 | 5 votes |
private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource clusterResource, ResourceLimits parentLimits) { // Set resource-limit of a given child, child.limit = // min(my.limit - my.used + child.used, child.max) // Parent available resource = parent-limit - parent-used-resource Resource parentMaxAvailableResource = Resources.subtract(parentLimits.getLimit(), getUsedResources()); // Child's limit = parent-available-resource + child-used Resource childLimit = Resources.add(parentMaxAvailableResource, child.getUsedResources()); // Get child's max resource Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), child.getAbsoluteMaximumCapacity(), minimumAllocation); // Child's limit should be capped by child configured max resource childLimit = Resources.min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource); // Normalize before return childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); return new ResourceLimits(childLimit); }
Example 5
Source File: LeafQueue.java From hadoop with Apache License 2.0 | 5 votes |
private Resource getHeadroom(User user, Resource currentResourceLimit, Resource clusterResource, Resource userLimit) { /** * Headroom is: * min( * min(userLimit, queueMaxCap) - userConsumed, * queueMaxCap - queueUsedResources * ) * * ( which can be expressed as, * min (userLimit - userConsumed, queuMaxCap - userConsumed, * queueMaxCap - queueUsedResources) * ) * * given that queueUsedResources >= userConsumed, this simplifies to * * >> min (userlimit - userConsumed, queueMaxCap - queueUsedResources) << * */ Resource headroom = Resources.min(resourceCalculator, clusterResource, Resources.subtract(userLimit, user.getUsed()), Resources.subtract(currentResourceLimit, queueUsage.getUsed()) ); // Normalize it before return headroom = Resources.roundDown(resourceCalculator, headroom, minimumAllocation); return headroom; }
Example 6
Source File: ProportionalCapacityPreemptionPolicy.java From hadoop with Apache License 2.0 | 5 votes |
Resource offer(Resource avail, ResourceCalculator rc, Resource clusterResource) { Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax( Resources.subtract(maxCapacity, idealAssigned), Resource.newInstance(0, 0, 0)); // remain = avail - min(avail, (max - assigned), (current + pending - assigned)) Resource accepted = Resources.min(rc, clusterResource, absMaxCapIdealAssignedDelta, Resources.min(rc, clusterResource, avail, Resources.subtract( Resources.add(current, pending), idealAssigned))); Resource remain = Resources.subtract(avail, accepted); Resources.addTo(idealAssigned, accepted); return remain; }
Example 7
Source File: FairSharePolicy.java From big-c with Apache License 2.0 | 5 votes |
@Override public int compare(Schedulable s1, Schedulable s2) { double minShareRatio1, minShareRatio2; double useToWeightRatio1, useToWeightRatio2; Resource minShare1 = Resources.min(RESOURCE_CALCULATOR, null, s1.getMinShare(), s1.getDemand()); Resource minShare2 = Resources.min(RESOURCE_CALCULATOR, null, s2.getMinShare(), s2.getDemand()); boolean s1Needy = Resources.lessThan(RESOURCE_CALCULATOR, null, s1.getResourceUsage(), minShare1); boolean s2Needy = Resources.lessThan(RESOURCE_CALCULATOR, null, s2.getResourceUsage(), minShare2); minShareRatio1 = (double) s1.getResourceUsage().getMemory() / Resources.max(RESOURCE_CALCULATOR, null, minShare1, ONE).getMemory(); minShareRatio2 = (double) s2.getResourceUsage().getMemory() / Resources.max(RESOURCE_CALCULATOR, null, minShare2, ONE).getMemory(); useToWeightRatio1 = s1.getResourceUsage().getMemory() / s1.getWeights().getWeight(ResourceType.MEMORY); useToWeightRatio2 = s2.getResourceUsage().getMemory() / s2.getWeights().getWeight(ResourceType.MEMORY); int res = 0; if (s1Needy && !s2Needy) res = -1; else if (s2Needy && !s1Needy) res = 1; else if (s1Needy && s2Needy) res = (int) Math.signum(minShareRatio1 - minShareRatio2); else // Neither schedulable is needy res = (int) Math.signum(useToWeightRatio1 - useToWeightRatio2); if (res == 0) { // Apps are tied in fairness ratio. Break the tie by submit time and job // name to get a deterministic ordering, which is useful for unit tests. res = (int) Math.signum(s1.getStartTime() - s2.getStartTime()); if (res == 0) res = s1.getName().compareTo(s2.getName()); } return res; }
Example 8
Source File: ParentQueue.java From big-c with Apache License 2.0 | 5 votes |
private ResourceLimits getResourceLimitsOfChild(CSQueue child, Resource clusterResource, ResourceLimits parentLimits) { // Set resource-limit of a given child, child.limit = // min(my.limit - my.used + child.used, child.max)//why ? // my.limit-my-used+child.used 等于在父队列中除去别的子对列所用资源后剩下的资源,有可能别的子对列所用资源超过了他们自身的限制 // Parent available resource = parent-limit - parent-used-resource Resource parentMaxAvailableResource = Resources.subtract(parentLimits.getLimit(), getUsedResources()); // Child's limit = parent-available-resource + child-used Resource childLimit = Resources.add(parentMaxAvailableResource, child.getUsedResources()); // Get child's max resource Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), child.getAbsoluteMaximumCapacity(), minimumAllocation); // Child's limit should be capped by child configured max resource childLimit = Resources.min(resourceCalculator, clusterResource, childLimit, childConfiguredMaxResource); // Normalize before return childLimit = Resources.roundDown(resourceCalculator, childLimit, minimumAllocation); return new ResourceLimits(childLimit); }
Example 9
Source File: LeafQueue.java From big-c with Apache License 2.0 | 5 votes |
private Resource getHeadroom(User user, Resource currentResourceLimit, //queueMaxCap 是不是hard limit Resource clusterResource, Resource userLimit) { /** * Headroom is: * min( * min(userLimit, queueMaxCap) - userConsumed, * queueMaxCap - queueUsedResources * ) * * ( which can be expressed as, * min (userLimit - userConsumed, queuMaxCap - userConsumed, * queueMaxCap - queueUsedResources) * ) * * given that queueUsedResources >= userConsumed, this simplifies to * * >> min (userlimit - userConsumed, queueMaxCap - queueUsedResources) << * */ Resource headroom = Resources.min(resourceCalculator, clusterResource, Resources.subtract(userLimit, user.getUsed()), Resources.subtract(currentResourceLimit, queueUsage.getUsed()) ); // Normalize it before return headroom = Resources.roundDown(resourceCalculator, headroom, minimumAllocation); return headroom; }
Example 10
Source File: FSLeafQueue.java From hadoop with Apache License 2.0 | 4 votes |
private boolean isStarved(Resource share) { Resource desiredShare = Resources.min(scheduler.getResourceCalculator(), scheduler.getClusterResource(), share, getDemand()); return Resources.lessThan(scheduler.getResourceCalculator(), scheduler.getClusterResource(), getResourceUsage(), desiredShare); }
Example 11
Source File: FSLeafQueue.java From big-c with Apache License 2.0 | 4 votes |
private boolean isStarved(Resource share) { Resource desiredShare = Resources.min(scheduler.getResourceCalculator(), scheduler.getClusterResource(), share, getDemand()); return Resources.lessThan(scheduler.getResourceCalculator(), scheduler.getClusterResource(), getResourceUsage(), desiredShare); }