Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#multiply()

@Override
public void updateDemand() {
  demand = Resources.createResource(0);
  // Demand is current consumption plus outstanding requests
  Resources.addTo(demand, getCurrentConsumption());

  // Add up outstanding resource requests
  synchronized (this) {
    for (Priority p : getPriorities()) {
      for (ResourceRequest r : getResourceRequests(p).values()) {
        Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
        Resources.addTo(demand, total);
      }
    }
  }
}
 

@Override
public void updateDemand() {
  demand = Resources.createResource(0);
  // Demand is current consumption plus outstanding requests
  Resources.addTo(demand, getCurrentConsumption());

  // Add up outstanding resource requests
  synchronized (this) {
    for (Priority p : getPriorities()) {
      for (ResourceRequest r : getResourceRequests(p).values()) {
        Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
        Resources.addTo(demand, total);
      }
    }
  }
}
 

synchronized public void move(Queue newQueue) {
  QueueMetrics oldMetrics = queue.getMetrics();
  QueueMetrics newMetrics = newQueue.getMetrics();
  for (Map<String, ResourceRequest> asks : requests.values()) {
    ResourceRequest request = asks.get(ResourceRequest.ANY);
    if (request != null) {
      oldMetrics.decrPendingResources(user, request.getNumContainers(),
          request.getCapability());
      newMetrics.incrPendingResources(user, request.getNumContainers(),
          request.getCapability());
      
      Resource delta = Resources.multiply(request.getCapability(),
          request.getNumContainers()); 
      // Update Queue
      queue.decPendingResource(request.getNodeLabelExpression(), delta);
      newQueue.incPendingResource(request.getNodeLabelExpression(), delta);
    }
  }
  oldMetrics.moveAppFrom(this);
  newMetrics.moveAppTo(this);
  activeUsersManager.deactivateApplication(user, applicationId);
  activeUsersManager = newQueue.getActiveUsersManager();
  activeUsersManager.activateApplication(user, applicationId);
  this.queue = newQueue;
  this.queueName = newQueue.getQueueName();
}
 

@Override
protected Resource getReservationQueueResourceIfExists(Plan plan,
    ReservationId reservationId) {
  CSQueue resQueue = cs.getQueue(reservationId.toString());
  Resource reservationResource = null;
  if (resQueue != null) {
    reservationResource = Resources.multiply(cs.getClusterResource(),
        resQueue.getAbsoluteCapacity());
  }
  return reservationResource;
}
 

@Override
protected Resource getPlanResources(
    Plan plan, Queue queue, Resource clusterResources) {
  PlanQueue planQueue = (PlanQueue)queue;
  float planAbsCap = planQueue.getAbsoluteCapacity();
  Resource planResources = Resources.multiply(clusterResources, planAbsCap);
  plan.setTotalCapacity(planResources);
  return planResources;
}
 

@Private
public Resource getResourceLimit() {
  Resource headRoom = getAvailableResources();
  if (headRoom == null) {
    headRoom = Resources.none();
  }
  Resource assignedMapResource =
      Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
  Resource assignedReduceResource =
      Resources.multiply(reduceResourceRequest,
        assignedRequests.reduces.size());
  return Resources.add(headRoom,
    Resources.add(assignedMapResource, assignedReduceResource));
}
 

/**
 * Used capacity of label =
 *   (queue's used resources labeled by nodeLabel)
 *      / (   (all resources labeled by nodLabel)
 *          X (percent of labeled resources allocated to this queue)  )
 */
public synchronized float getUsedCapacity(String nodeLabel) {
  Resource availableToQueue =
      Resources.multiply(labelManager.getResourceByLabel(nodeLabel, cr),
          queueCapacities.getAbsoluteCapacity(nodeLabel));
  if (!Resources.greaterThan(resourceCalculator, cr,
       availableToQueue, Resources.none())) {
    return 0.0f;
  }
  return
      Resources.divide(resourceCalculator, cr,
          queueUsage.getUsed(nodeLabel), availableToQueue);
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    absoluteUsedCapacity = 
        Resources.divide(calculator, clusterResource, 
            usedResources, clusterResource);
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

/**
 * Check whether this queue can run this application master under the
 * maxAMShare limit
 *
 * @param amResource
 * @return true if this queue can run
 */
public boolean canRunAppAM(Resource amResource) {
  float maxAMShare =
      scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
  if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
    return true;
  }
  Resource maxAMResource = Resources.multiply(getFairShare(), maxAMShare);
  Resource ifRunAMResource = Resources.add(amResourceUsage, amResource);
  return !policy
      .checkIfAMResourceUsageOverLimit(ifRunAMResource, maxAMResource);
}
 

public void assignPreemption(float scalingFactor,
    ResourceCalculator rc, Resource clusterResource) {
  if (Resources.greaterThan(rc, clusterResource, current, idealAssigned)) {
      toBePreempted = Resources.multiply(
          Resources.subtract(current, idealAssigned), scalingFactor);
  } else {
    toBePreempted = Resource.newInstance(0, 0, 0);
  }
}
 

@Private
public Resource getResourceLimit() {
  Resource headRoom = getAvailableResources();
  if (headRoom == null) {
    headRoom = Resources.none();
  }
  Resource assignedMapResource =
      Resources.multiply(mapResourceRequest, assignedRequests.maps.size());
  Resource assignedReduceResource =
      Resources.multiply(reduceResourceRequest,
        assignedRequests.reduces.size());
  return Resources.add(headRoom,
    Resources.add(assignedMapResource, assignedReduceResource));
}
 

@Override
protected Resource getPlanResources(
    Plan plan, Queue queue, Resource clusterResources) {
  PlanQueue planQueue = (PlanQueue)queue;
  float planAbsCap = planQueue.getAbsoluteCapacity();
  Resource planResources = Resources.multiply(clusterResources, planAbsCap);
  plan.setTotalCapacity(planResources);
  return planResources;
}
 

@Override
protected Resource getReservationQueueResourceIfExists(Plan plan,
    ReservationId reservationId) {
  CSQueue resQueue = cs.getQueue(reservationId.toString());
  Resource reservationResource = null;
  if (resQueue != null) {
    reservationResource = Resources.multiply(cs.getClusterResource(),
        resQueue.getAbsoluteCapacity());
  }
  return reservationResource;
}
 

public void assignPreemption(float scalingFactor,
    ResourceCalculator rc, Resource clusterResource) {
  if (Resources.greaterThan(rc, clusterResource, current, idealAssigned)) {
	  //to avoid negative memory or cores
	  //we only prempte current.cores > idealAssigned.cores || current.memory > idealAssigned.memory
      toBePreempted = Resources.multiply(
          Resources.subtracts(current, idealAssigned), scalingFactor);
      //we still have resource left
      LOG.info("assignPreemption queue  "+queueName+" toBePreempted  "+toBePreempted);
  } else {
    toBePreempted = Resource.newInstance(0, 0);
  }
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    
    //absoluteUsedCapacity = 
    //    Resources.divide(calculator, clusterResource, 
    //        usedResources, clusterResource);
    
    absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0));
  
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

/**
 * Removes a resource for the specified interval
 * 
 * @param reservationInterval the interval for which the resource is to be
 *          removed
 * @param capacity the resource to be removed
 * @return true if removal is successful, false otherwise
 */
public boolean removeInterval(ReservationInterval reservationInterval,
    ReservationRequest capacity) {
  Resource totCap =
      Resources.multiply(capacity.getCapability(),
          (float) capacity.getNumContainers());
  if (totCap.equals(ZERO_RESOURCE)) {
    return true;
  }
  writeLock.lock();
  try {
    long startKey = reservationInterval.getStartTime();
    long endKey = reservationInterval.getEndTime();
    // update the start key
    NavigableMap<Long, Resource> ticks =
        cumulativeCapacity.headMap(endKey, false);
    // Decrease all the capacities of overlapping intervals
    SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey);
    if (overlapSet != null && !overlapSet.isEmpty()) {
      Resource updatedCapacity = Resource.newInstance(0, 0);
      long currentKey = -1;
      for (Iterator<Entry<Long, Resource>> overlapEntries =
          overlapSet.entrySet().iterator(); overlapEntries.hasNext();) {
        Entry<Long, Resource> entry = overlapEntries.next();
        currentKey = entry.getKey();
        updatedCapacity = Resources.subtract(entry.getValue(), totCap);
        // update each entry between start and end key
        cumulativeCapacity.put(currentKey, updatedCapacity);
      }
      // Remove the first overlap entry if it is same as previous after
      // updation
      Long firstKey = overlapSet.firstKey();
      if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) {
        cumulativeCapacity.remove(firstKey);
      }
      // Remove the next entry if it is same as end entry after updation
      if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) {
        cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey));
      }
    }
    return true;
  } finally {
    writeLock.unlock();
  }
}
 

/**
 * Based a resource preemption target drop reservations of containers and
 * if necessary select containers for preemption from applications in each
 * over-capacity queue. It uses {@link #NATURAL_TERMINATION_FACTOR} to
 * account for containers that will naturally complete.
 *
 * @param queues set of leaf queues to preempt from
 * @param clusterResource total amount of cluster resources
 * @return a map of applciationID to set of containers to preempt
 */
private Map<ApplicationAttemptId,Set<RMContainer>> getContainersToPreempt(
    List<TempQueue> queues, Resource clusterResource) {

  Map<ApplicationAttemptId,Set<RMContainer>> preemptMap =
      new HashMap<ApplicationAttemptId,Set<RMContainer>>();
  List<RMContainer> skippedAMContainerlist = new ArrayList<RMContainer>();

  for (TempQueue qT : queues) {
    if (qT.preemptionDisabled && qT.leafQueue != null) {
      if (LOG.isDebugEnabled()) {
        if (Resources.greaterThan(rc, clusterResource,
            qT.toBePreempted, Resource.newInstance(0, 0, 0))) {
          LOG.debug("Tried to preempt the following "
                    + "resources from non-preemptable queue: "
                    + qT.queueName + " - Resources: " + qT.toBePreempted);
        }
      }
      continue;
    }
    // we act only if we are violating balance by more than
    // maxIgnoredOverCapacity
    if (Resources.greaterThan(rc, clusterResource, qT.current,
        Resources.multiply(qT.guaranteed, 1.0 + maxIgnoredOverCapacity))) {
      // we introduce a dampening factor naturalTerminationFactor that
      // accounts for natural termination of containers
      Resource resToObtain =
        Resources.multiply(qT.toBePreempted, naturalTerminationFactor);
      Resource skippedAMSize = Resource.newInstance(0, 0, 0);

      // lock the leafqueue while we scan applications and unreserve
      synchronized (qT.leafQueue) {
        NavigableSet<FiCaSchedulerApp> ns = 
            (NavigableSet<FiCaSchedulerApp>) qT.leafQueue.getApplications();
        Iterator<FiCaSchedulerApp> desc = ns.descendingIterator();
        qT.actuallyPreempted = Resources.clone(resToObtain);
        while (desc.hasNext()) {
          FiCaSchedulerApp fc = desc.next();
          if (Resources.lessThanOrEqual(rc, clusterResource, resToObtain,
              Resources.none())) {
            break;
          }
          preemptMap.put(
              fc.getApplicationAttemptId(),
              preemptFrom(fc, clusterResource, resToObtain,
                  skippedAMContainerlist, skippedAMSize));
        }
        Resource maxAMCapacityForThisQueue = Resources.multiply(
            Resources.multiply(clusterResource,
                qT.leafQueue.getAbsoluteCapacity()),
            qT.leafQueue.getMaxAMResourcePerQueuePercent());

        // Can try preempting AMContainers (still saving atmost
        // maxAMCapacityForThisQueue AMResource's) if more resources are
        // required to be preempted from this Queue.
        preemptAMContainers(clusterResource, preemptMap,
            skippedAMContainerlist, resToObtain, skippedAMSize,
            maxAMCapacityForThisQueue);
      }
    }
  }
  return preemptMap;
}
 

/**
 * Based a resource preemption target drop reservations of containers and
 * if necessary select containers for preemption from applications in each
 * over-capacity queue. It uses {@link #NATURAL_TERMINATION_FACTOR} to
 * account for containers that will naturally complete.
 *
 * @param queues set of leaf queues to preempt from
 * @param clusterResource total amount of cluster resources
 * @return a map of applciationID to set of containers to preempt
 */
private Map<ApplicationAttemptId,Map<RMContainer,Resource>> getContainersToPreempt(
    List<TempQueue> queues, Resource clusterResource) {

  Map<ApplicationAttemptId, Map<RMContainer,Resource>> preemptMap =
      new HashMap<ApplicationAttemptId, Map<RMContainer,Resource>>();
  
  List<RMContainer> skippedAMContainerlist = new ArrayList<RMContainer>();
  
  //for test only
  if(isTest){
  	
  	getContainersToPreemptForTest(preemptMap, queues, clusterResource);
  }
  

  for (TempQueue qT : queues) {
    if (qT.preemptionDisabled && qT.leafQueue != null) {
      if (LOG.isDebugEnabled()) {
        if (Resources.greaterThan(rc, clusterResource,
            qT.toBePreempted, Resource.newInstance(0, 0))) {
          LOG.info("Tried to preempt the following "
                    + "resources from non-preemptable queue: "
                    + qT.queueName + " - Resources: " + qT.toBePreempted);
        }
      }
      continue;
    }
    // we act only if we are violating balance by more than
    // maxIgnoredOverCapacity
    if (Resources.greaterThan(rc, clusterResource, qT.current,
        Resources.multiply(qT.guaranteed, 1.0 + maxIgnoredOverCapacity))) {
      // we introduce a dampening factor naturalTerminationFactor that
      // accounts for natural termination of containers
      Resource resToObtain =
        Resources.multiply(qT.toBePreempted, naturalTerminationFactor);
      Resource skippedAMSize = Resource.newInstance(0, 0);
      
      LOG.info("try to preempt: "+resToObtain+" from queue: "+qT.queueName);
      if(resToObtain.getMemory() > 0){
      	LOG.info("resToObtain memory: "+resToObtain.getMemory());
      }
      // lock the leafqueue while we scan applications and unreserve
      synchronized (qT.leafQueue) {
        //what is the descending order
        NavigableSet<FiCaSchedulerApp> ns = 
            (NavigableSet<FiCaSchedulerApp>) qT.leafQueue.getApplications();
       Iterator<FiCaSchedulerApp> desc = ns.descendingIterator();
        qT.actuallyPreempted = Resources.clone(resToObtain);
        while (desc.hasNext()) {
          FiCaSchedulerApp fc = desc.next();
          if (Resources.lessThanOrEqual(rc, clusterResource, resToObtain,
              Resources.none())) {
            break;
          }
          LOG.info("now try to preempt applicatin:"+fc.getApplicationId());
          preemptMap.put(
              fc.getApplicationAttemptId(),
              preemptFrom(fc, clusterResource, resToObtain,
                  skippedAMContainerlist, skippedAMSize));
        }
        
        //we allow preempt AM for kill based approach
        if(false){
       //we will never preempt am resource 
        Resource maxAMCapacityForThisQueue = Resources.multiply(
           Resources.multiply(clusterResource,
                qT.leafQueue.getAbsoluteCapacity()),
            qT.leafQueue.getMaxAMResourcePerQueuePercent());

         //Can try preempting AMContainers (still saving atmost
        // maxAMCapacityForThisQueue AMResource's) if more resources are
        // required to be preempted from this Queue.
        preemptAMContainers(clusterResource, preemptMap,
           skippedAMContainerlist, resToObtain, skippedAMSize,
           maxAMCapacityForThisQueue);
        }
      }
    }
  }
  return preemptMap;
}
 

/**
 * Removes a resource for the specified interval
 * 
 * @param reservationInterval the interval for which the resource is to be
 *          removed
 * @param capacity the resource to be removed
 * @return true if removal is successful, false otherwise
 */
public boolean removeInterval(ReservationInterval reservationInterval,
    ReservationRequest capacity) {
  Resource totCap =
      Resources.multiply(capacity.getCapability(),
          (float) capacity.getNumContainers());
  if (totCap.equals(ZERO_RESOURCE)) {
    return true;
  }
  writeLock.lock();
  try {
    long startKey = reservationInterval.getStartTime();
    long endKey = reservationInterval.getEndTime();
    // update the start key
    NavigableMap<Long, Resource> ticks =
        cumulativeCapacity.headMap(endKey, false);
    // Decrease all the capacities of overlapping intervals
    SortedMap<Long, Resource> overlapSet = ticks.tailMap(startKey);
    if (overlapSet != null && !overlapSet.isEmpty()) {
      Resource updatedCapacity = Resource.newInstance(0, 0, 0);
      long currentKey = -1;
      for (Iterator<Entry<Long, Resource>> overlapEntries =
          overlapSet.entrySet().iterator(); overlapEntries.hasNext();) {
        Entry<Long, Resource> entry = overlapEntries.next();
        currentKey = entry.getKey();
        updatedCapacity = Resources.subtract(entry.getValue(), totCap);
        // update each entry between start and end key
        cumulativeCapacity.put(currentKey, updatedCapacity);
      }
      // Remove the first overlap entry if it is same as previous after
      // updation
      Long firstKey = overlapSet.firstKey();
      if (isSameAsPrevious(firstKey, overlapSet.get(firstKey))) {
        cumulativeCapacity.remove(firstKey);
      }
      // Remove the next entry if it is same as end entry after updation
      if ((currentKey != -1) && (isSameAsNext(currentKey, updatedCapacity))) {
        cumulativeCapacity.remove(cumulativeCapacity.higherKey(currentKey));
      }
    }
    return true;
  } finally {
    writeLock.unlock();
  }
}
 

/**
 * Add a resource for the specified interval
 * 
 * @param reservationInterval the interval for which the resource is to be
 *          added
 * @param capacity the resource to be added
 * @return true if addition is successful, false otherwise
 */
public boolean addInterval(ReservationInterval reservationInterval,
    ReservationRequest capacity) {
  Resource totCap =
      Resources.multiply(capacity.getCapability(),
          (float) capacity.getNumContainers());
  if (totCap.equals(ZERO_RESOURCE)) {
    return true;
  }
  writeLock.lock();
  try {
    long startKey = reservationInterval.getStartTime();
    long endKey = reservationInterval.getEndTime();
    NavigableMap<Long, Resource> ticks =
        cumulativeCapacity.headMap(endKey, false);
    if (ticks != null && !ticks.isEmpty()) {
      Resource updatedCapacity = Resource.newInstance(0, 0, 0);
      Entry<Long, Resource> lowEntry = ticks.floorEntry(startKey);
      if (lowEntry == null) {
        // This is the earliest starting interval
        cumulativeCapacity.put(startKey, totCap);
      } else {
        updatedCapacity = Resources.add(lowEntry.getValue(), totCap);
        // Add a new tick only if the updated value is different
        // from the previous tick
        if ((startKey == lowEntry.getKey())
            && (isSameAsPrevious(lowEntry.getKey(), updatedCapacity))) {
          cumulativeCapacity.remove(lowEntry.getKey());
        } else {
          cumulativeCapacity.put(startKey, updatedCapacity);
        }
      }
      // Increase all the capacities of overlapping intervals
      Set<Entry<Long, Resource>> overlapSet =
          ticks.tailMap(startKey, false).entrySet();
      for (Entry<Long, Resource> entry : overlapSet) {
        updatedCapacity = Resources.add(entry.getValue(), totCap);
        entry.setValue(updatedCapacity);
      }
    } else {
      // This is the first interval to be added
      cumulativeCapacity.put(startKey, totCap);
    }
    Resource nextTick = cumulativeCapacity.get(endKey);
    if (nextTick != null) {
      // If there is overlap, remove the duplicate entry
      if (isSameAsPrevious(endKey, nextTick)) {
        cumulativeCapacity.remove(endKey);
      }
    } else {
      // Decrease capacity as this is end of the interval
      cumulativeCapacity.put(endKey, Resources.subtract(cumulativeCapacity
          .floorEntry(endKey).getValue(), totCap));
    }
    return true;
  } finally {
    writeLock.unlock();
  }
}