Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#subtract()

private Resource getUnallocatedReservedResources(
    ReservationAllocation reservation) {
  Resource resResource;
  Resource reservationResource = planFollower
      .getReservationQueueResourceIfExists
          (plan, reservation.getReservationId());
  if (reservationResource != null) {
    resResource =
        Resources.subtract(
            reservation.getResourcesAtTime(now),
            reservationResource);
  } else {
    resResource = reservation.getResourcesAtTime(now);
  }
  return resResource;
}
 

public synchronized void adjusctContianerResouce(ContainerId contianerId, Resource resource){
 
 RMContainer container = getRMContainer(contianerId);
 //get original resource
 Resource allocatedResource = container.getAllocatedResource();
 Resource marginalResource=Resources.subtract(allocatedResource, resource);
 
 if(marginalResource.getMemory() > 0 ){		  
  LOG.info("increase memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());		  
 }else{		  
  LOG.info("decrease memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());
 }
 
 if(marginalResource.getVirtualCores() > 0){
  LOG.info("increase cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }else{
  LOG.info("dncrease cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }
 
 SchedulerNode schedulerNode = getSchedulerNode(container.getAllocatedNode());
 //now inform the schedulerNode to adjust resurce assumption
 schedulerNode.addAvailableResource(marginalResource);
 //now inform RMContinaer to adjuct its allocatedResource
 container.handle(new RMContainerResourceUpdateEvent(contianerId, RMContainerEventType.RESOURCE_UPDATE, resource));
 	  
}
 

/**
 * Headroom depends on resources in the cluster, current usage of the
 * queue, queue's fair-share and queue's max-resources.
 */
@Override
public Resource getHeadroom() {
  final FSQueue queue = (FSQueue) this.queue;
  SchedulingPolicy policy = queue.getPolicy();

  Resource queueFairShare = queue.getFairShare();
  Resource queueUsage = queue.getResourceUsage();
  Resource clusterResource = this.scheduler.getClusterResource();
  Resource clusterUsage = this.scheduler.getRootQueueMetrics()
      .getAllocatedResources();

  Resource clusterAvailableResources =
      Resources.subtract(clusterResource, clusterUsage);
  Resource queueMaxAvailableResources =
      Resources.subtract(queue.getMaxShare(), queueUsage);
  Resource maxAvailableResource = Resources.componentwiseMin(
      clusterAvailableResources, queueMaxAvailableResources);

  Resource headroom = policy.getHeadroom(queueFairShare,
      queueUsage, maxAvailableResource);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for " + this.getName() + ":" +
        "Min(" +
        "(queueFairShare=" + queueFairShare +
        " - queueUsage=" + queueUsage + ")," +
        " maxAvailableResource=" + maxAvailableResource +
        "Headroom=" + headroom);
  }
  return headroom;
}
 

private ResourceLimits getResourceLimitsOfChild(CSQueue child,
    Resource clusterResource, ResourceLimits parentLimits) {
  // Set resource-limit of a given child, child.limit =
  // min(my.limit - my.used + child.used, child.max)

  // Parent available resource = parent-limit - parent-used-resource
  Resource parentMaxAvailableResource =
      Resources.subtract(parentLimits.getLimit(), getUsedResources());

  // Child's limit = parent-available-resource + child-used
  Resource childLimit =
      Resources.add(parentMaxAvailableResource, child.getUsedResources());

  // Get child's max resource
  Resource childConfiguredMaxResource =
      Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
          .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
          child.getAbsoluteMaximumCapacity(), minimumAllocation);

  // Child's limit should be capped by child configured max resource
  childLimit =
      Resources.min(resourceCalculator, clusterResource, childLimit,
          childConfiguredMaxResource);

  // Normalize before return
  childLimit =
      Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

  return new ResourceLimits(childLimit);
}
 

private ResourceLimits getResourceLimitsOfChild(CSQueue child,
     Resource clusterResource, ResourceLimits parentLimits) {
   // Set resource-limit of a given child, child.limit =
   // min(my.limit - my.used + child.used, child.max)//why ?
// my.limit-my-used+child.used 等于在父队列中除去别的子对列所用资源后剩下的资源，有可能别的子对列所用资源超过了他们自身的限制

   // Parent available resource = parent-limit - parent-used-resource
   Resource parentMaxAvailableResource =
       Resources.subtract(parentLimits.getLimit(), getUsedResources());
   
   // Child's limit = parent-available-resource + child-used
   Resource childLimit =
       Resources.add(parentMaxAvailableResource, child.getUsedResources());

   // Get child's max resource
   Resource childConfiguredMaxResource =
       Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager
           .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
           child.getAbsoluteMaximumCapacity(), minimumAllocation);

   // Child's limit should be capped by child configured max resource
   childLimit =
       Resources.min(resourceCalculator, clusterResource, childLimit,
           childConfiguredMaxResource);

   // Normalize before return
   childLimit =
       Resources.roundDown(resourceCalculator, childLimit, minimumAllocation);

   return new ResourceLimits(childLimit);
 }
 

/**
 * Headroom depends on resources in the cluster, current usage of the
 * queue, queue's fair-share and queue's max-resources.
 */
@Override
public Resource getHeadroom() {
  final FSQueue queue = (FSQueue) this.queue;
  SchedulingPolicy policy = queue.getPolicy();

  Resource queueFairShare = queue.getFairShare();
  Resource queueUsage = queue.getResourceUsage();
  Resource clusterResource = this.scheduler.getClusterResource();
  Resource clusterUsage = this.scheduler.getRootQueueMetrics()
      .getAllocatedResources();

  Resource clusterAvailableResources =
      Resources.subtract(clusterResource, clusterUsage);
  Resource queueMaxAvailableResources =
      Resources.subtract(queue.getMaxShare(), queueUsage);
  Resource maxAvailableResource = Resources.componentwiseMin(
      clusterAvailableResources, queueMaxAvailableResources);

  Resource headroom = policy.getHeadroom(queueFairShare,
      queueUsage, maxAvailableResource);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for " + this.getName() + ":" +
        "Min(" +
        "(queueFairShare=" + queueFairShare +
        " - queueUsage=" + queueUsage + ")," +
        " maxAvailableResource=" + maxAvailableResource +
        "Headroom=" + headroom);
  }
  return headroom;
}
 

public Resource getCurrentUsedResource(){
 if(isSuspending){
  return Resources.subtract(container.getResource(), preempted);
 }else{
  return container.getResource();
 }
}
 

private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 

@Test
public void testHeadroom() {
  final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class);
  Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock());

  final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class);

  final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3);
  final Resource queueFairShare = Resources.createResource(4096, 2);
  final Resource queueUsage = Resource.newInstance(2048, 2);

  final Resource queueStarvation =
      Resources.subtract(queueFairShare, queueUsage);
  final Resource queueMaxResourcesAvailable =
      Resources.subtract(queueMaxResources, queueUsage);

  final Resource clusterResource = Resources.createResource(8192, 8);
  final Resource clusterUsage = Resources.createResource(2048, 2);
  final Resource clusterAvailable =
      Resources.subtract(clusterResource, clusterUsage);

  final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class);

  Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
  Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
  Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
  Mockito.when(mockScheduler.getClusterResource()).thenReturn
      (clusterResource);
  Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
      (clusterUsage);
  Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn
      (fakeRootQueueMetrics);

  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  RMContext rmContext = resourceManager.getRMContext();
  FSAppAttempt schedulerApp =
      new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue ,
          null, rmContext);

  // Min of Memory and CPU across cluster and queue is used in
  // DominantResourceFairnessPolicy
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(DominantResourceFairnessPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      min(queueStarvation.getVirtualCores(),
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  // Fair and Fifo ignore CPU of queue, so use cluster available CPU
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FairSharePolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FifoPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );
}
 

@Private
@VisibleForTesting
void preemptReducesIfNeeded() {
  if (reduceResourceRequest.equals(Resources.none())) {
    return; // no reduces
  }
  //check if reduces have taken over the whole cluster and there are 
  //unassigned maps
  if (scheduledRequests.maps.size() > 0) {
    Resource resourceLimit = getResourceLimit();
    Resource availableResourceForMap =
        Resources.subtract(
          resourceLimit,
          Resources.multiply(reduceResourceRequest,
            assignedRequests.reduces.size()
                - assignedRequests.preemptionWaitingReduces.size()));
    // availableMemForMap must be sufficient to run at least 1 map
    if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap,
      mapResourceRequest, getSchedulerResourceTypes()) <= 0) {
      // to make sure new containers are given to maps and not reduces
      // ramp down all scheduled reduces if any
      // (since reduces are scheduled at higher priority than maps)
      LOG.info("Ramping down all scheduled reduces:"
          + scheduledRequests.reduces.size());
      for (ContainerRequest req : scheduledRequests.reduces.values()) {
        pendingReduces.add(req);
      }
      scheduledRequests.reduces.clear();
 
      //do further checking to find the number of map requests that were
      //hanging around for a while
      int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
      if (hangingMapRequests > 0) {
        // preempt for making space for at least one map
        int preemptionReduceNumForOneMap =
            ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest,
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForPreemptionLimit =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(resourceLimit, maxReducePreemptionLimit),
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForAllMaps =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(mapResourceRequest, hangingMapRequests),
              reduceResourceRequest, getSchedulerResourceTypes());
        int toPreempt =
            Math.min(Math.max(preemptionReduceNumForOneMap,
              preemptionReduceNumForPreemptionLimit),
              preemptionReduceNumForAllMaps);

        LOG.info("Going to preempt " + toPreempt
            + " due to lack of space for maps");
        assignedRequests.preemptReduce(toPreempt);
      }
    }
  }
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    
    //absoluteUsedCapacity = 
    //    Resources.divide(calculator, clusterResource, 
    //        usedResources, clusterResource);
    
    absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0));
  
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

/**
 * Removes a resource for the specified interval
 * 
 * @param reservationInterval the interval for which the resource is to be
 *          removed
 * @param capacity the resource to be removed
 * @return true if removal is successful, false otherwise
 */
public boolean removeInterval(ReservationInterval reservationInterval,
    ReservationRequest capacity) {
  Resource totCap =
      Resources.multiply(capacity.getCapability(),
          (float) capacity.getNumContainers());
  if (totCap.equals(ZERO_RESOURCE)) {
    return true;
  }
  writeLock.lock();
  try {
    long startKey = reservationInterval.getStartTime();
    long endKey = reservationInterval.getEndTime();
    // update the start key
    NavigableMap<Long, Resource> ticks =
        cumulativeCapacity.headMap(endKey, false);
    // Decrease all the capacities of overlapping intervals
    SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey);
    if (overlapSet != null && !overlapSet.isEmpty()) {
      Resource updatedCapacity = Resource.newInstance(0, 0);
      long currentKey = -1;
      for (Iterator<Entry<Long, Resource>> overlapEntries =
          overlapSet.entrySet().iterator(); overlapEntries.hasNext();) {
        Entry<Long, Resource> entry = overlapEntries.next();
        currentKey = entry.getKey();
        updatedCapacity = Resources.subtract(entry.getValue(), totCap);
        // update each entry between start and end key
        cumulativeCapacity.put(currentKey, updatedCapacity);
      }
      // Remove the first overlap entry if it is same as previous after
      // updation
      Long firstKey = overlapSet.firstKey();
      if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) {
        cumulativeCapacity.remove(firstKey);
      }
      // Remove the next entry if it is same as end entry after updation
      if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) {
        cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey));
      }
    }
    return true;
  } finally {
    writeLock.unlock();
  }
}
 

@Private
@VisibleForTesting
void preemptReducesIfNeeded() {
  if (reduceResourceRequest.equals(Resources.none())) {
    return; // no reduces
  }
  //check if reduces have taken over the whole cluster and there are 
  //unassigned maps
  if (scheduledRequests.maps.size() > 0) {
    Resource resourceLimit = getResourceLimit();
    Resource availableResourceForMap =
        Resources.subtract(
          resourceLimit,
          Resources.multiply(reduceResourceRequest,
            assignedRequests.reduces.size()
                - assignedRequests.preemptionWaitingReduces.size()));
    // availableMemForMap must be sufficient to run at least 1 map
    if (ResourceCalculatorUtils.computeAvailableContainers(availableResourceForMap,
      mapResourceRequest, getSchedulerResourceTypes()) <= 0) {
      // to make sure new containers are given to maps and not reduces
      // ramp down all scheduled reduces if any
      // (since reduces are scheduled at higher priority than maps)
      LOG.info("Ramping down all scheduled reduces:"
          + scheduledRequests.reduces.size());
      for (ContainerRequest req : scheduledRequests.reduces.values()) {
        pendingReduces.add(req);
      }
      scheduledRequests.reduces.clear();
 
      //do further checking to find the number of map requests that were
      //hanging around for a while
      int hangingMapRequests = getNumOfHangingRequests(scheduledRequests.maps);
      if (hangingMapRequests > 0) {
        // preempt for making space for at least one map
        int preemptionReduceNumForOneMap =
            ResourceCalculatorUtils.divideAndCeilContainers(mapResourceRequest,
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForPreemptionLimit =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(resourceLimit, maxReducePreemptionLimit),
              reduceResourceRequest, getSchedulerResourceTypes());
        int preemptionReduceNumForAllMaps =
            ResourceCalculatorUtils.divideAndCeilContainers(
              Resources.multiply(mapResourceRequest, hangingMapRequests),
              reduceResourceRequest, getSchedulerResourceTypes());
        int toPreempt =
            Math.min(Math.max(preemptionReduceNumForOneMap,
              preemptionReduceNumForPreemptionLimit),
              preemptionReduceNumForAllMaps);

        LOG.info("Going to preempt " + toPreempt
            + " due to lack of space for maps");
        assignedRequests.preemptReduce(toPreempt);
      }
    }
  }
}
 

@Test
public void testHeadroom() {
  final FairScheduler mockScheduler = Mockito.mock(FairScheduler.class);
  Mockito.when(mockScheduler.getClock()).thenReturn(scheduler.getClock());

  final FSLeafQueue mockQueue = Mockito.mock(FSLeafQueue.class);

  final Resource queueMaxResources = Resource.newInstance(5 * 1024, 3, 3);
  final Resource queueFairShare = Resources.createResource(4096, 2, 2);
  final Resource queueUsage = Resource.newInstance(2048, 2, 2);

  final Resource queueStarvation =
      Resources.subtract(queueFairShare, queueUsage);
  final Resource queueMaxResourcesAvailable =
      Resources.subtract(queueMaxResources, queueUsage);

  final Resource clusterResource = Resources.createResource(8192, 8, 8);
  final Resource clusterUsage = Resources.createResource(2048, 2, 2);
  final Resource clusterAvailable =
      Resources.subtract(clusterResource, clusterUsage);

  final QueueMetrics fakeRootQueueMetrics = Mockito.mock(QueueMetrics.class);

  Mockito.when(mockQueue.getMaxShare()).thenReturn(queueMaxResources);
  Mockito.when(mockQueue.getFairShare()).thenReturn(queueFairShare);
  Mockito.when(mockQueue.getResourceUsage()).thenReturn(queueUsage);
  Mockito.when(mockScheduler.getClusterResource()).thenReturn
      (clusterResource);
  Mockito.when(fakeRootQueueMetrics.getAllocatedResources()).thenReturn
      (clusterUsage);
  Mockito.when(mockScheduler.getRootQueueMetrics()).thenReturn
      (fakeRootQueueMetrics);

  ApplicationAttemptId applicationAttemptId = createAppAttemptId(1, 1);
  RMContext rmContext = resourceManager.getRMContext();
  FSAppAttempt schedulerApp =
      new FSAppAttempt(mockScheduler, applicationAttemptId, "user1", mockQueue ,
          null, rmContext);

  // Min of Memory and CPU across cluster and queue is used in
  // DominantResourceFairnessPolicy
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(DominantResourceFairnessPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      min(queueStarvation.getVirtualCores(),
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  // Fair and Fifo ignore CPU of queue, so use cluster available CPU
  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FairSharePolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );

  Mockito.when(mockQueue.getPolicy()).thenReturn(SchedulingPolicy
      .getInstance(FifoPolicy.class));
  verifyHeadroom(schedulerApp,
      min(queueStarvation.getMemory(),
          clusterAvailable.getMemory(),
          queueMaxResourcesAvailable.getMemory()),
      Math.min(
          clusterAvailable.getVirtualCores(),
          queueMaxResourcesAvailable.getVirtualCores())
  );
}
 

@Override
public void onContainersCompleted(List<ContainerStatus> statuses) {
  if (isStopped.get()) {
    return;
  }
  Map<Object, ContainerStatus> appContainerStatus =
                      new HashMap<Object, ContainerStatus>(statuses.size());
  synchronized (this) {
    for(ContainerStatus containerStatus : statuses) {
      ContainerId completedId = containerStatus.getContainerId();
      HeldContainer delayedContainer = heldContainers.get(completedId);

      Object task = releasedContainers.remove(completedId);
      if(task != null){
        if (delayedContainer != null) {
          LOG.warn("Held container should be null since releasedContainer is not");
        }
        // TODO later we may want to check if exit code matched expectation
        // e.g. successful container should not come back fail exit code after
        // being released
        // completion of a container we had released earlier
        // an allocated container completed. notify app
        LOG.info("Released container completed:" + completedId +
                 " last allocated to task: " + task);
        appContainerStatus.put(task, containerStatus);
        continue;
      }

      // not found in released containers. check currently allocated containers
      // no need to release this container as the RM has already completed it
      task = unAssignContainer(completedId, false);
      if (delayedContainer != null) {
        heldContainers.remove(completedId);
        Resources.subtract(allocatedResources, delayedContainer.getContainer().getResource());
      } else {
        LOG.warn("Held container expected to be not null for a non-AM-released container");
      }
      if(task != null) {
        // completion of a container we have allocated currently
        // an allocated container completed. notify app
        LOG.info("Allocated container completed:" + completedId +
                 " last allocated to task: " + task);
        appContainerStatus.put(task, containerStatus);
        continue;
      }

      // container neither allocated nor released
      LOG.info("Ignoring unknown container: " + containerStatus.getContainerId());
    }
  }

  // upcall to app must be outside locks
  for (Entry<Object, ContainerStatus> entry : appContainerStatus.entrySet()) {
    appClientDelegate.containerCompleted(entry.getKey(), entry.getValue());
  }
}
 

/**
 * Given a set of queues compute the fix-point distribution of unassigned
 * resources among them. As pending request of a queue are exhausted, the
 * queue is removed from the set and remaining capacity redistributed among
 * remaining queues. The distribution is weighted based on guaranteed
 * capacity, unless asked to ignoreGuarantee, in which case resources are
 * distributed uniformly.
 */
private void computeFixpointAllocation(ResourceCalculator rc,
    Resource tot_guarant, Collection<TempQueue> qAlloc, Resource unassigned, 
    boolean ignoreGuarantee) {
  // Prior to assigning the unused resources, process each queue as follows:
  // If current > guaranteed, idealAssigned = guaranteed + untouchable extra
  // Else idealAssigned = current;
  // Subtract idealAssigned resources from unassigned.
  // If the queue has all of its needs met (that is, if 
  // idealAssigned >= current + pending), remove it from consideration.
  // Sort queues from most under-guaranteed to most over-guaranteed.
  TQComparator tqComparator = new TQComparator(rc, tot_guarant);
  PriorityQueue<TempQueue> orderedByNeed =
                               new PriorityQueue<TempQueue>(10,tqComparator);
  for (Iterator<TempQueue> i = qAlloc.iterator(); i.hasNext();) {
    TempQueue q = i.next();
    if (Resources.greaterThan(rc, tot_guarant, q.current, q.guaranteed)) {
      q.idealAssigned = Resources.add(q.guaranteed, q.untouchableExtra);
    } else {
      q.idealAssigned = Resources.clone(q.current);
    }
    Resources.subtractFrom(unassigned, q.idealAssigned);
    // If idealAssigned < (current + pending), q needs more resources, so
    // add it to the list of underserved queues, ordered by need.
    Resource curPlusPend = Resources.add(q.current, q.pending);
    if (Resources.lessThan(rc, tot_guarant, q.idealAssigned, curPlusPend)) {
      orderedByNeed.add(q);
    }
  }

  //assign all cluster resources until no more demand, or no resources are left
  while (!orderedByNeed.isEmpty()
     && Resources.greaterThan(rc,tot_guarant, unassigned,Resources.none())) {
    Resource wQassigned = Resource.newInstance(0, 0, 0);
    // we compute normalizedGuarantees capacity based on currently active
    // queues
    resetCapacity(rc, unassigned, orderedByNeed, ignoreGuarantee);

    // For each underserved queue (or set of queues if multiple are equally
    // underserved), offer its share of the unassigned resources based on its
    // normalized guarantee. After the offer, if the queue is not satisfied,
    // place it back in the ordered list of queues, recalculating its place
    // in the order of most under-guaranteed to most over-guaranteed. In this
    // way, the most underserved queue(s) are always given resources first.
    Collection<TempQueue> underserved =
        getMostUnderservedQueues(orderedByNeed, tqComparator);
    for (Iterator<TempQueue> i = underserved.iterator(); i.hasNext();) {
      TempQueue sub = i.next();
      Resource wQavail = Resources.multiplyAndNormalizeUp(rc,
          unassigned, sub.normalizedGuarantee, Resource.newInstance(1, 1, 0));
      Resource wQidle = sub.offer(wQavail, rc, tot_guarant);
      Resource wQdone = Resources.subtract(wQavail, wQidle);

      if (Resources.greaterThan(rc, tot_guarant,
            wQdone, Resources.none())) {
        // The queue is still asking for more. Put it back in the priority
        // queue, recalculating its order based on need.
        orderedByNeed.add(sub);
      }
      Resources.addTo(wQassigned, wQdone);
    }
    Resources.subtractFrom(unassigned, wQassigned);
  }
}
 

@Private
protected synchronized boolean canAssignToUser(Resource clusterResource,
    String userName, Resource limit, FiCaSchedulerApp application,
    Set<String> requestLabels, ResourceLimits currentResoureLimits) {
  User user = getUser(userName);
  
  String label = CommonNodeLabelsManager.NO_LABEL;
  if (requestLabels != null && !requestLabels.isEmpty()) {
    label = requestLabels.iterator().next();
  }

  // Note: We aren't considering the current request since there is a fixed
  // overhead of the AM, but it's a > check, not a >= check, so...
  if (Resources
      .greaterThan(resourceCalculator, clusterResource,
          user.getUsed(label),
          limit)) {
    // if enabled, check to see if could we potentially use this node instead
    // of a reserved node if the application has reserved containers
    if (this.reservationsContinueLooking
        && label.equals(CommonNodeLabelsManager.NO_LABEL)) {
      if (Resources.lessThanOrEqual(
          resourceCalculator,
          clusterResource,
          Resources.subtract(user.getUsed(), application.getCurrentReservation()),
          limit)) {

        if (LOG.isDebugEnabled()) {
          LOG.debug("User " + userName + " in queue " + getQueueName()
              + " will exceed limit based on reservations - " + " consumed: "
              + user.getUsed() + " reserved: "
              + application.getCurrentReservation() + " limit: " + limit);
        }
        Resource amountNeededToUnreserve = Resources.subtract(user.getUsed(label), limit);
        // we can only acquire a new container if we unreserve first since we ignored the
        // user limit. Choose the max of user limit or what was previously set by max
        // capacity.
        currentResoureLimits.setAmountNeededUnreserve(Resources.max(resourceCalculator,
            clusterResource, currentResoureLimits.getAmountNeededUnreserve(),
            amountNeededToUnreserve));
        return true;
      }
    }
    if (LOG.isDebugEnabled()) {
      LOG.debug("User " + userName + " in queue " + getQueueName()
          + " will exceed limit - " + " consumed: "
          + user.getUsed() + " limit: " + limit);
    }
    return false;
  }
  return true;
}
 

synchronized boolean canAssignToThisQueue(Resource clusterResource,
    Set<String> nodeLabels, ResourceLimits currentResourceLimits,
    Resource nowRequired, Resource resourceCouldBeUnreserved) {
  // Get label of this queue can access, it's (nodeLabel AND queueLabel)
  Set<String> labelCanAccess;
  if (null == nodeLabels || nodeLabels.isEmpty()) {
    labelCanAccess = new HashSet<String>();
    // Any queue can always access any node without label
    labelCanAccess.add(RMNodeLabelsManager.NO_LABEL);
  } else {
    labelCanAccess = new HashSet<String>(
        accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels
            : Sets.intersection(accessibleLabels, nodeLabels));
  }
  
  for (String label : labelCanAccess) {
    // New total resource = used + required
    Resource newTotalResource =
        Resources.add(queueUsage.getUsed(label), nowRequired);

    Resource currentLimitResource =
        getCurrentLimitResource(label, clusterResource, currentResourceLimits);

    if (Resources.greaterThan(resourceCalculator, clusterResource,
        newTotalResource, currentLimitResource)) {

      // if reservation continous looking enabled, check to see if could we
      // potentially use this node instead of a reserved node if the application
      // has reserved containers.
      // TODO, now only consider reservation cases when the node has no label
      if (this.reservationsContinueLooking
          && label.equals(RMNodeLabelsManager.NO_LABEL)
          && Resources.greaterThan(resourceCalculator, clusterResource,
          resourceCouldBeUnreserved, Resources.none())) {
        // resource-without-reserved = used - reserved
        Resource newTotalWithoutReservedResource =
            Resources.subtract(newTotalResource, resourceCouldBeUnreserved);

        // when total-used-without-reserved-resource < currentLimit, we still
        // have chance to allocate on this node by unreserving some containers
        if (Resources.lessThan(resourceCalculator, clusterResource,
            newTotalWithoutReservedResource, currentLimitResource)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("try to use reserved: " + getQueueName()
                + " usedResources: " + queueUsage.getUsed()
                + ", clusterResources: " + clusterResource
                + ", reservedResources: " + resourceCouldBeUnreserved
                + ", capacity-without-reserved: "
                + newTotalWithoutReservedResource + ", maxLimitCapacity: "
                + currentLimitResource);
          }
          currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource,
              currentLimitResource));
          return true;
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(getQueueName()
            + "Check assign to queue, label=" + label
            + " usedResources: " + queueUsage.getUsed(label)
            + " clusterResources: " + clusterResource
            + " currentUsedCapacity "
            + Resources.divide(resourceCalculator, clusterResource,
            queueUsage.getUsed(label),
            labelManager.getResourceByLabel(label, clusterResource))
            + " max-capacity: "
            + queueCapacities.getAbsoluteMaximumCapacity(label)
            + ")");
      }
      return false;
    }
    return true;
  }
  
  // Actually, this will not happen, since labelCanAccess will be always
  // non-empty
  return false;
}
 

synchronized boolean canAssignToThisQueue(Resource clusterResource,
    Set<String> nodeLabels, ResourceLimits currentResourceLimits,
    Resource nowRequired, Resource resourceCouldBeUnreserved) {
  // Get label of this queue can access, it's (nodeLabel AND queueLabel)
  Set<String> labelCanAccess;
  if (null == nodeLabels || nodeLabels.isEmpty()) {
    labelCanAccess = new HashSet<String>();
    // Any queue can always access any node without label
    labelCanAccess.add(RMNodeLabelsManager.NO_LABEL);
  } else {
    labelCanAccess = new HashSet<String>(
        accessibleLabels.contains(CommonNodeLabelsManager.ANY) ? nodeLabels
            : Sets.intersection(accessibleLabels, nodeLabels));
  }
  
  for (String label : labelCanAccess) {
    // New total resource = used + required
    Resource newTotalResource =
        Resources.add(queueUsage.getUsed(label), nowRequired);

    Resource currentLimitResource =
        getCurrentLimitResource(label, clusterResource, currentResourceLimits);

    //current resource is much more than currentlimit
    if (Resources.greaterThan(resourceCalculator, clusterResource,
        newTotalResource, currentLimitResource)) {

      // if reservation continous looking enabled, check to see if could we
      // potentially use this node instead of a reserved node if the application
      // has reserved containers.
      // TODO, now only consider reservation cases when the node has no label
      if (this.reservationsContinueLooking
          && label.equals(RMNodeLabelsManager.NO_LABEL)
          && Resources.greaterThan(resourceCalculator, clusterResource,
          resourceCouldBeUnreserved, Resources.none())) {
        // resource-without-reserved = used - reserved
        Resource newTotalWithoutReservedResource =
            Resources.subtract(newTotalResource, resourceCouldBeUnreserved);

        // when total-used-without-reserved-resource < currentLimit, we still
        // have chance to allocate on this node by unreserving some containers
        if (Resources.lessThan(resourceCalculator, clusterResource,
            newTotalWithoutReservedResource, currentLimitResource)) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("try to use reserved: " + getQueueName()
                + " usedResources: " + queueUsage.getUsed()
                + ", clusterResources: " + clusterResource
                + ", reservedResources: " + resourceCouldBeUnreserved
                + ", capacity-without-reserved: "
                + newTotalWithoutReservedResource + ", maxLimitCapacity: "
                + currentLimitResource);
          }
          currentResourceLimits.setAmountNeededUnreserve(Resources.subtract(newTotalResource,
              currentLimitResource));
          return true;
        }
      }
      if (LOG.isDebugEnabled()) {
        LOG.debug(getQueueName()
            + "Check assign to queue, label=" + label
            + " usedResources: " + queueUsage.getUsed(label)
            + " clusterResources: " + clusterResource
            + " currentUsedCapacity "
            + Resources.divide(resourceCalculator, clusterResource,
            queueUsage.getUsed(label),
            labelManager.getResourceByLabel(label, clusterResource))
            + " max-capacity: "
            + queueCapacities.getAbsoluteMaximumCapacity(label)
            + ")");
      }
      return false;
    }
    return true;
  }
  
  // Actually, this will not happen, since labelCanAccess will be always
  // non-empty
  return false;
}