Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#equals()

@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
 

@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();

  // If this queue is over its limit, reject
  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Collections.sort(childQueues, policy.getComparator());
  for (FSQueue child : childQueues) {
    assigned = child.assignContainer(node);
    if (!Resources.equals(assigned, Resources.none())) {
      break;
    }
  }
  return assigned;
}
 

@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
 

@Override
public Resource assignContainer(FSSchedulerNode node) {
  Resource assigned = Resources.none();

  // If this queue is over its limit, reject
  if (!assignContainerPreCheck(node)) {
    return assigned;
  }

  Collections.sort(childQueues, policy.getComparator());
  for (FSQueue child : childQueues) {
    assigned = child.assignContainer(node);
    if (!Resources.equals(assigned, Resources.none())) {
      break;
    }
  }
  return assigned;
}
 

@Override
public boolean equals(Object obj) {
  if (obj instanceof NodeLabel) {
    NodeLabel other = (NodeLabel) obj;
    return Resources.equals(resource, other.getResource())
        && StringUtils.equals(labelName, other.getLabelName())
        && (other.getNumActiveNMs() == numActiveNMs); 
  }
  return false;
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    absoluteUsedCapacity = 
        Resources.divide(calculator, clusterResource, 
            usedResources, clusterResource);
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

private boolean check(ReservationAllocation cs, long start, long end,
    int containers, int mem, int cores) {

  boolean res = true;
  for (long i = start; i < end; i++) {
    res = res
        && Resources.equals(cs.getResourcesAtTime(i),
            Resource.newInstance(mem * containers, cores * containers, cores * containers));
  }
  return res;
}
 

@Override
public boolean equals(Object obj) {
  if (obj instanceof NodeLabel) {
    NodeLabel other = (NodeLabel) obj;
    return Resources.equals(resource, other.getResource())
        && StringUtils.equals(labelName, other.getLabelName())
        && (other.getNumActiveNMs() == numActiveNMs); 
  }
  return false;
}
 

@Override
  public RMContainerState transition(RMContainerImpl container,
      RMContainerEvent event) {
   container.resumeTime.add(System.currentTimeMillis());
if(Resources.equals(container.getPreemptedResource(),Resources.none())){
//if all the preempted resource has been resumed  
   container.isSuspending = false;
   return RMContainerState.RUNNING;
   }else{
  	return RMContainerState.DEHYDRATED;
   }

}
 

@Override
  public synchronized Collection<RMContainer> getUnPreemtedContainers() {
	    Collection<RMContainer> returnList = new ArrayList<RMContainer>();
	    for(RMContainer rmContainer : liveContainers.values()){
	    	//skip if all resource has been preempted
	    	if(Resources.equals(rmContainer.getCurrentUsedResource(),Resources.none())){
	    		continue;
	    	}
	    	
	    	returnList.add(rmContainer);
	    }
	     
	    return returnList;
}
 

private boolean check(ReservationAllocation cs, long start, long end,
    int containers, int mem, int cores) {

  boolean res = true;
  for (long i = start; i < end; i++) {
    res = res
        && Resources.equals(cs.getResourcesAtTime(i),
            Resource.newInstance(mem * containers, cores * containers));
  }
  return res;
}
 

/**
 * 1. Updates {@link RMNode#getTotalCapability()} with newCapacity.
 * 2. Sends out a {@link NodeResourceUpdateSchedulerEvent} that's handled by YARN's scheduler.
 * The scheduler updates the corresponding {@link SchedulerNode} with the newCapacity.
 *
 * @param rmNode
 * @param newCapacity
 */
@SuppressWarnings("unchecked")
public void setNodeCapacity(RMNode rmNode, Resource newCapacity) {
  //NOOP prevent YARN warning changing to same size
  if ((Resources.equals(rmNode.getTotalCapability(), newCapacity))) {
    return;
  }
  if (yarnScheduler.getSchedulerNode(rmNode.getNodeID()) == null) {
    LOGGER.info("Yarn Scheduler doesn't have node {}, probably UNHEALTHY", rmNode.getNodeID());
    return;
  }
  yarnSchedulerLock.lock();
  try {
    if (newCapacity.getMemory() < 0 || newCapacity.getVirtualCores() < 0) {
      Resource zeroed = ResourceUtils.componentwiseMax(ZERO_RESOURCE, newCapacity);
      rmNode.getTotalCapability().setMemory(zeroed.getMemory());
      rmNode.getTotalCapability().setVirtualCores(zeroed.getVirtualCores());
      LOGGER.warn("Asked to set Node {} to a value less than zero!  Had {}, setting to {}.",
          rmNode.getHttpAddress(), rmNode.getTotalCapability().toString(), zeroed.toString());
    } else {
      rmNode.getTotalCapability().setMemory(newCapacity.getMemory());
      rmNode.getTotalCapability().setVirtualCores(newCapacity.getVirtualCores());
      if (LOGGER.isInfoEnabled()) {
        LOGGER.info("Setting capacity for node {} to {}", rmNode.getHostName(), newCapacity);
      }
    }
    // updates the scheduler with the new capacity for the NM.
    // the event is handled by the scheduler asynchronously
    rmContext.getDispatcher().getEventHandler().handle(new NodeResourceUpdateSchedulerEvent(rmNode, ResourceOption.newInstance(
        rmNode.getTotalCapability(), RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT)));
  } finally {
    yarnSchedulerLock.unlock();
  }
}
 

private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}
 

@Lock(CSQueue.class)
public static void updateQueueStatistics(
    final ResourceCalculator calculator,
    final CSQueue childQueue, final CSQueue parentQueue, 
    final Resource clusterResource, final Resource minimumAllocation) {
  Resource queueLimit = Resources.none();
  Resource usedResources = childQueue.getUsedResources();
  
  float absoluteUsedCapacity = 0.0f;
  float usedCapacity = 0.0f;

  if (Resources.greaterThan(
      calculator, clusterResource, clusterResource, Resources.none())) {
    queueLimit = 
        Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
    
    //absoluteUsedCapacity = 
    //    Resources.divide(calculator, clusterResource, 
    //        usedResources, clusterResource);
    
    absoluteUsedCapacity = (float) ((usedResources.getVirtualCores()*1.0)/(clusterResource.getVirtualCores()*1.0));
  
    usedCapacity = 
        Resources.equals(queueLimit, Resources.none()) ? 0 :
        Resources.divide(calculator, clusterResource, 
            usedResources, queueLimit);
  }

  childQueue.setUsedCapacity(usedCapacity);
  childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
  
  Resource available = Resources.subtract(queueLimit, usedResources);
  childQueue.getMetrics().setAvailableResourcesToQueue(
      Resources.max(
          calculator, 
          clusterResource, 
          available, 
          Resources.none()
          )
      );
 }
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}
 

private synchronized void assign(Priority priority, NodeType type, 
    List<Container> containers) throws IOException, YarnException {
  for (Iterator<Container> i=containers.iterator(); i.hasNext();) {
    Container container = i.next();
    String host = container.getNodeId().toString();
    
    if (Resources.equals(requestSpec.get(priority), container.getResource())) { 
      // See which task can use this container
      for (Iterator<Task> t=tasks.get(priority).iterator(); t.hasNext();) {
        Task task = t.next();
        if (task.getState() == State.PENDING && task.canSchedule(type, host)) {
          NodeManager nodeManager = getNodeManager(host);
          
          task.start(nodeManager, container.getId());
          i.remove();
          
          // Track application resource usage
          Resources.addTo(used, container.getResource());
          
          LOG.info("Assigned container (" + container + ") of type " + type +
              " to task " + task.getTaskId() + " at priority " + priority + 
              " on node " + nodeManager.getHostName() +
              ", currently using " + used + " resources");

          // Update resource requests
          updateResourceRequests(requests.get(priority), type, task);

          // Launch the container
          StartContainerRequest scRequest =
              StartContainerRequest.newInstance(createCLC(),
                container.getContainerToken());
          List<StartContainerRequest> list =
              new ArrayList<StartContainerRequest>();
          list.add(scRequest);
          StartContainersRequest allRequests =
              StartContainersRequest.newInstance(list);
          nodeManager.startContainers(allRequests);
          break;
        }
      }
    }
  }
}