Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#addTo()

private Resource getTotalResource(List<ResourceRequest> requests) {
  Resource totalResource = Resource.newInstance(0, 0, 0);
  if (requests == null) {
    return totalResource;
  }
  for (ResourceRequest request : requests) {
    if (request.getNumContainers() == 0) {
      continue;
    }
    if (request.getResourceName().equals(ResourceRequest.ANY)) {
      Resources.addTo(
        totalResource,
        Resources.multiply(request.getCapability(),
          request.getNumContainers()));
    }
  }
  return totalResource;
}
 

private void pushNewContainerToDelayed(List<Container> containers){
  long expireTime = getHeldContainerExpireTime(System.currentTimeMillis());

  synchronized (delayedContainerManager) {
    for (Container container : containers) {
      if (heldContainers.put(container.getId(), new HeldContainer(container,
          -1, expireTime, null, this.containerSignatureMatcher)) != null) {
        throw new TezUncheckedException("New container " + container.getId()
            + " is already held.");
      }
      long nextScheduleTime = delayedContainerManager.maxScheduleTimeSeen;
      if (delayedContainerManager.maxScheduleTimeSeen == -1) {
        nextScheduleTime = System.currentTimeMillis();
      }
      Resources.addTo(allocatedResources, container.getResource());
      delayedContainerManager.addDelayedContainer(container,
        nextScheduleTime + 1);
    }
  }
  delayedContainerManager.triggerScheduling(false);      
}
 

@Override
public void updateDemand() {
  demand = Resources.createResource(0);
  // Demand is current consumption plus outstanding requests
  Resources.addTo(demand, getCurrentConsumption());

  // Add up outstanding resource requests
  synchronized (this) {
    for (Priority p : getPriorities()) {
      for (ResourceRequest r : getResourceRequests(p).values()) {
        Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
        Resources.addTo(demand, total);
      }
    }
  }
}
 

/**
 * Check for queues that need tasks preempted, either because they have been
 * below their guaranteed share for minSharePreemptionTimeout or they have
 * been below their fair share threshold for the fairSharePreemptionTimeout. If
 * such queues exist, compute how many tasks of each type need to be preempted
 * and then select the right ones using preemptTasks.
 */
protected synchronized void preemptTasksIfNecessary() {
  if (!shouldAttemptPreemption()) {
    return;
  }

  long curTime = getClock().getTime();
  if (curTime - lastPreemptCheckTime < preemptionInterval) {
    return;
  }
  lastPreemptCheckTime = curTime;

  Resource resToPreempt = Resources.clone(Resources.none());
  for (FSLeafQueue sched : queueMgr.getLeafQueues()) {
    Resources.addTo(resToPreempt, resToPreempt(sched, curTime));
  }
  if (Resources.greaterThan(RESOURCE_CALCULATOR, clusterResource, resToPreempt,
      Resources.none())) {
    preemptResources(resToPreempt);
  }
}
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0, 0));
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  Resource accepted = 
      Resources.min(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.min(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  Resource remain = Resources.subtract(avail, accepted);
  Resources.addTo(idealAssigned, accepted);
  return remain;
}
 

private synchronized void deductAvailableResource(Resource resource) {
  if (resource == null) {
    LOG.error("Invalid deduction of null resource for "
        + rmNode.getNodeAddress());
    return;
  }
  Resources.subtractFrom(availableResource, resource);
  Resources.addTo(usedResource, resource);
}
 

public synchronized Resource getTotalResourcePending() {
  Resource ret = BuilderUtils.newResource(0, 0);
  for (FiCaSchedulerApp f : activeApplications) {
    Resources.addTo(ret, f.getTotalPendingRequests());
  }
  return ret;
}
 

/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Notify NodeLabelsManager about this change
    rmContext.getNodeLabelManager().updateNodeResource(nm.getNodeID(),
        newResource);

    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 

private synchronized void deductAvailableResource(Resource resource) {
  if (resource == null) {
    LOG.error("Invalid deduction of null resource for "
        + rmNode.getNodeAddress());
    return;
  }
  Resources.subtractFrom(availableResource, resource);
  Resources.addTo(usedResource, resource);
}
 

/**
 * Process resource update on a node.
 */
public synchronized void updateNodeResource(RMNode nm, 
    ResourceOption resourceOption) {
  SchedulerNode node = getSchedulerNode(nm.getNodeID());
  Resource newResource = resourceOption.getResource();
  Resource oldResource = node.getTotalResource();
  if(!oldResource.equals(newResource)) {
    // Log resource change
    LOG.info("Update resource on node: " + node.getNodeName()
        + " from: " + oldResource + ", to: "
        + newResource);

    nodes.remove(nm.getNodeID());
    updateMaximumAllocation(node, false);

    // update resource to node
    node.setTotalResource(newResource);

    nodes.put(nm.getNodeID(), (N)node);
    updateMaximumAllocation(node, true);

    // update resource to clusterResource
    Resources.subtractFrom(clusterResource, oldResource);
    Resources.addTo(clusterResource, newResource);
  } else {
    // Log resource change
    LOG.warn("Update resource on node: " + node.getNodeName() 
        + " with the same resource: " + newResource);
  }
}
 

@VisibleForTesting
protected Resource getResourcesUnderUse(RMNodeStatusEvent statusEvent) {
  Resource usedResources = Resource.newInstance(0, 0);
  for (ContainerStatus status : statusEvent.getContainers()) {
    if (containerInUse(status)) {
      RMContainer rmContainer = yarnScheduler.getRMContainer(status.getContainerId());
      // (sdaingade) This check is needed as RMContainer information may not be populated
      // immediately after a RM restart.
      if (rmContainer != null) {
        Resources.addTo(usedResources, rmContainer.getAllocatedResource());
      }
    }
  }
  return usedResources;
}
 

public synchronized Resource getTotalPendingRequests() {
  Resource ret = Resource.newInstance(0, 0);
  for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) {
    // to avoid double counting we count only "ANY" resource requests
    if (ResourceRequest.isAnyLocation(rr.getResourceName())){
      Resources.addTo(ret,
          Resources.multiply(rr.getCapability(), rr.getNumContainers()));
    }
  }
  return ret;
}
 

private void _inc(String label, ResourceType type, Resource res) {
  try {
    writeLock.lock();
    UsageByLabel usage = getAndAddIfMissing(label);
    Resources.addTo(usage.resArr[type.idx], res);
  } finally {
    writeLock.unlock();
  }
}
 

public void reinitializeQueueLabels(Map<String, Set<String>> queueToLabels) {
  try {
    writeLock.lock();
    // clear before set
    this.queueCollections.clear();

    for (Entry<String, Set<String>> entry : queueToLabels.entrySet()) {
      String queue = entry.getKey();
      Queue q = new Queue();
      this.queueCollections.put(queue, q);

      Set<String> labels = entry.getValue();
      if (labels.contains(ANY)) {
        continue;
      }

      q.acccessibleNodeLabels.addAll(labels);
      for (Host host : nodeCollections.values()) {
        for (Entry<NodeId, Node> nentry : host.nms.entrySet()) {
          NodeId nodeId = nentry.getKey();
          Node nm = nentry.getValue();
          if (nm.running && isNodeUsableByQueue(getLabelsByNode(nodeId), q)) {
            Resources.addTo(q.resource, nm.resource);
          }
        }
      }
    }
  } finally {
    writeLock.unlock();
  }
}
 

public void addNode(Resource nodeRes) {
  Resources.addTo(resource, nodeRes);
  numActiveNMs++;
}
 

synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
    Priority priority, ResourceRequest request,
    Container container) {
  // Update allowed locality level
  NodeType allowed = allowedLocalityLevel.get(priority);
  if (allowed != null) {
    if (allowed.equals(NodeType.OFF_SWITCH) &&
        (type.equals(NodeType.NODE_LOCAL) ||
            type.equals(NodeType.RACK_LOCAL))) {
      this.resetAllowedLocalityLevel(priority, type);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL) &&
        type.equals(NodeType.NODE_LOCAL)) {
      this.resetAllowedLocalityLevel(priority, type);
    }
  }

  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, 
      getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  Resources.addTo(currentConsumption, container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 

@Override
synchronized public StopContainersResponse stopContainers(StopContainersRequest request) 
throws YarnException {
  for (ContainerId containerID : request.getContainerIds()) {
    String applicationId =
        String.valueOf(containerID.getApplicationAttemptId()
          .getApplicationId().getId());
    // Mark the container as COMPLETE
    List<Container> applicationContainers = containers.get(containerID.getApplicationAttemptId()
            .getApplicationId());
    for (Container c : applicationContainers) {
      if (c.getId().compareTo(containerID) == 0) {
        ContainerStatus containerStatus = containerStatusMap.get(c);
        containerStatus.setState(ContainerState.COMPLETE);
        containerStatusMap.put(c, containerStatus);
      }
    }

    // Send a heartbeat
    try {
      heartbeat();
    } catch (IOException ioe) {
      throw RPCUtil.getRemoteException(ioe);
    }

    // Remove container and update status
    int ctr = 0;
    Container container = null;
    for (Iterator<Container> i = applicationContainers.iterator(); i
      .hasNext();) {
      container = i.next();
      if (container.getId().compareTo(containerID) == 0) {
        i.remove();
        ++ctr;
      }
    }

    if (ctr != 1) {
      throw new IllegalStateException("Container " + containerID
          + " stopped " + ctr + " times!");
    }

    Resources.addTo(available, container.getResource());
    Resources.subtractFrom(used, container.getResource());

    if (LOG.isDebugEnabled()) {
      LOG.debug("stopContainer:" + " node=" + containerManagerAddress
          + " application=" + applicationId + " container=" + containerID
          + " available=" + available + " used=" + used);
    }
  }
  return StopContainersResponse.newInstance(null,null);
}
 

public void addAMResourceUsage(Resource amResource) {
  if (amResource != null) {
    Resources.addTo(amResourceUsage, amResource);
  }
}
 

@Override
synchronized public StopContainersResponse stopContainers(StopContainersRequest request) 
throws YarnException {
  for (ContainerId containerID : request.getContainerIds()) {
    String applicationId =
        String.valueOf(containerID.getApplicationAttemptId()
          .getApplicationId().getId());
    // Mark the container as COMPLETE
    List<Container> applicationContainers = containers.get(containerID.getApplicationAttemptId()
            .getApplicationId());
    for (Container c : applicationContainers) {
      if (c.getId().compareTo(containerID) == 0) {
        ContainerStatus containerStatus = containerStatusMap.get(c);
        containerStatus.setState(ContainerState.COMPLETE);
        containerStatusMap.put(c, containerStatus);
      }
    }

    // Send a heartbeat
    try {
      heartbeat();
    } catch (IOException ioe) {
      throw RPCUtil.getRemoteException(ioe);
    }

    // Remove container and update status
    int ctr = 0;
    Container container = null;
    for (Iterator<Container> i = applicationContainers.iterator(); i
      .hasNext();) {
      container = i.next();
      if (container.getId().compareTo(containerID) == 0) {
        i.remove();
        ++ctr;
      }
    }

    if (ctr != 1) {
      throw new IllegalStateException("Container " + containerID
          + " stopped " + ctr + " times!");
    }

    Resources.addTo(available, container.getResource());
    Resources.subtractFrom(used, container.getResource());

    if (LOG.isDebugEnabled()) {
      LOG.debug("stopContainer:" + " node=" + containerManagerAddress
          + " application=" + applicationId + " container=" + containerID
          + " available=" + available + " used=" + used);
    }
  }
  return StopContainersResponse.newInstance(null,null);
}
 

Resource offer(Resource avail, ResourceCalculator rc,
    Resource clusterResource) {
 	
  Resource absMaxCapIdealAssignedDelta = Resources.componentwiseMax(
                  Resources.subtract(maxCapacity, idealAssigned),
                  Resource.newInstance(0, 0));
  
  // remain = avail - min(avail, (max - assigned), (current + pending - assigned))
  // we have bug here. in some case:
  //(current + pending - assigned).core > avail.core
  //(current + pending - assigned).memo < avail.memo
  //so we get least cores of the three and least memory of the three
  Resource possibleAccepted = 
      Resources.mins(rc, clusterResource, 
          absMaxCapIdealAssignedDelta,
      Resources.mins(rc, clusterResource, avail, Resources.subtract(
          Resources.add(current, pending), idealAssigned)));
  
  //final allocation resource
  Resource finalAccepted = Resources.clone(possibleAccepted);
  //in extrame case where avail cores are more less than the available memory, it may preempt mroe memory
  //Max:      1310720   320
  //avail:    542634    26
  //Delta:    734280    60  
  //Pending:  525312    120
  //current:  576512    260
  //ideal:    576512    260
  //then the accepted will be (525312,26) in which the memory is far more beyond the requirement
  
  if(isSuspended){
	  
  if(dominantResource == Resources.CPU && !Resources.equals(pending,Resources.none())){
	  //pending must be either none() or resource(int ,int)
	  if(avail.getVirtualCores() == 0){
	      //if the dominant resource is cpu, we will stop allocation even we have memory
		  finalAccepted.setMemory(0);
		  //but if we still have more available memory, we can allocate, to avoid preemption
		  //we set memory to current usage
		  int gapMemory = current.getMemory() - idealAssigned.getMemory();
		  if(gapMemory > 0 && possibleAccepted.getMemory() > gapMemory){
			  finalAccepted.setMemory(gapMemory);
			  LOG.info("gap memory: "+gapMemory);
		  }
		  
	  }else{
	  double memoryRatio   = pending.getMemory()*1.0/pending.getVirtualCores();
	  int    ratioedMemory = (int)(memoryRatio*possibleAccepted.getVirtualCores());
	  finalAccepted.setMemory(ratioedMemory < possibleAccepted.getMemory() ? 
			                   ratioedMemory:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" cpu dominant ");
	  
	  if(finalAccepted.getMemory() < possibleAccepted.getMemory()){
		  LOG.info("previous memory: "+possibleAccepted.getMemory()+"  final memory: "+finalAccepted.getMemory());
	  }
	  
	  
  }else if(dominantResource == Resources.MEMORY && !Resources.equals(pending, Resources.none())){
	  if(avail.getMemory() == 0){
		  finalAccepted.setVirtualCores(0);
		  int gapCores = current.getVirtualCores() - idealAssigned.getVirtualCores();
		  if(gapCores > 0 && possibleAccepted.getVirtualCores() > gapCores){
			  finalAccepted.setVirtualCores(gapCores);
			  LOG.info("gap cores: "+gapCores);
		  }
		  
	  }else{
	  double cpuRatio   = pending.getVirtualCores()*1.0/pending.getMemory();
	  int    ratioedcpu = (int)(cpuRatio*possibleAccepted.getMemory());
	  finalAccepted.setVirtualCores(ratioedcpu < possibleAccepted.getMemory() ? 
			                 ratioedcpu:possibleAccepted.getMemory());
	  }
	  LOG.info("queue: "+queueName+" memory dominant ");
  }else{
	  LOG.info("queue: "+queueName+" empty ");
  }
  
  }
 
  LOG.info("queueName:   "+queueName);
  LOG.info("beforeideal: "+idealAssigned);  
  Resource remain = Resources.subtract(avail, finalAccepted);
  Resources.addTo(idealAssigned, finalAccepted);
  LOG.info("avaul:       "+avail);
  LOG.info("absMaxDelta: "+absMaxCapIdealAssignedDelta);
  LOG.info("max:         "+maxCapacity);
  LOG.info("current:     "+current);
  LOG.info("pending:     "+pending);
  LOG.info("acceped:     "+finalAccepted);
  LOG.info("ideal:       "+idealAssigned);
  
  return remain;
  
}