Java Code Examples for org.apache.hadoop.yarn.api.records.Container#getResource()

/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
 

/**
 * launch a new container with the given life time
 */
public void addNewContainer(Container container, long lifeTimeMS) {
  LOG.debug(MessageFormat.format("NodeManager {0} launches a new " +
          "container ({1}).", node.getNodeID(), container.getId()));
  if (lifeTimeMS != -1) {
    // normal container
    ContainerSimulator cs = new ContainerSimulator(container.getId(),
            container.getResource(), lifeTimeMS + System.currentTimeMillis(),
            lifeTimeMS);
    containerQueue.add(cs);
    runningContainers.put(cs.getId(), cs);
  } else {
    // AM container
    // -1 means AMContainer
    synchronized(amContainerList) {
      amContainerList.add(container.getId());
    }
  }
}
 

private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
 

private CookieContainerRequest getMatchingRequestWithPriority(
    Container container,
    String location) {
  Priority priority = container.getPriority();
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> requestsList =
      amRmClient.getMatchingRequests(priority, location, capability);

  if (!requestsList.isEmpty()) {
    // pick first one
    for (Collection<CookieContainerRequest> requests : requestsList) {
      for (CookieContainerRequest cookieContainerRequest : requests) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          return cookieContainerRequest;
        }
      }
    }
  }

  return null;
}
 

/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}
 

/**
 * Assign a container to this node to facilitate {@code request}. If node does
 * not have enough memory, create a reservation. This is called once we are
 * sure the particular request should be facilitated by this node.
 *
 * @param node
 *     The node to try placing the container on.
 * @param request
 *     The ResourceRequest we're trying to satisfy.
 * @param type
 *     The locality of the assignment.
 * @param reserved
 *     Whether there's already a container reserved for this app on the node.
 * @return
 *     If an assignment was made, returns the resources allocated to the
 *     container.  If a reservation was made, returns
 *     FairScheduler.CONTAINER_RESERVED.  If no assignment or reservation was
 *     made, returns an empty resource.
 */
private Resource assignContainer(
    FSSchedulerNode node, ResourceRequest request, NodeType type,
    boolean reserved) {

  // How much does this request need?
  Resource capability = request.getCapability();

  // How much does the node have?
  Resource available = node.getAvailableResource();

  Container container = null;
  if (reserved) {
    container = node.getReservedContainer().getContainer();
  } else {
    container = createContainer(node, capability, request.getPriority());
  }

  // Can we allocate a container on this node?
  if (Resources.fitsIn(capability, available)) {
    // Inform the application of the new container for this request
    RMContainer allocatedContainer =
        allocate(type, node, request.getPriority(), request, container);
    if (allocatedContainer == null) {
      // Did the application need this resource?
      if (reserved) {
        unreserve(request.getPriority(), node);
      }
      return Resources.none();
    }

    // If we had previously made a reservation, delete it
    if (reserved) {
      unreserve(request.getPriority(), node);
    }

    // Inform the node
    node.allocateContainer(allocatedContainer);

    // If this container is used to run AM, update the leaf queue's AM usage
    if (getLiveContainers().size() == 1 && !getUnmanagedAM()) {
      getQueue().addAMResourceUsage(container.getResource());
      setAmRunning(true);
    }

    return container.getResource();
  } else {
    if (!FairScheduler.fitsInMaxShare(getQueue(), capability)) {
      return Resources.none();
    }

    // The desired container won't fit here, so reserve
    reserve(request.getPriority(), node, container, reserved);

    return FairScheduler.CONTAINER_RESERVED;
  }
}
 

@Override
public void completedContainer(Resource clusterResource, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer, 
    ContainerStatus containerStatus, RMContainerEventType event, CSQueue childQueue,
    boolean sortQueues) {
  if (application != null) {

    boolean removed = false;

    // Careful! Locking order is important!
    synchronized (this) {

      Container container = rmContainer.getContainer();
      
      Resource toRelease = null;

      // Inform the application & the node
      // Note: It's safe to assume that all state changes to RMContainer
      // happen under scheduler's lock... 
      // So, this is, in effect, a transaction across application & node
      if (rmContainer.getState() == RMContainerState.RESERVED) {
        removed = unreserve(application, rmContainer.getReservedPriority(),
            node, rmContainer);
        toRelease = container.getResource();
      } else {
        //for container suspend event	       
        if (event == RMContainerEventType.SUSPEND){
         removed =
      	application.containerSuspend(rmContainer, containerStatus, event);
      	//suspend and resume in fifo order
          if(!suspendedApps.contains(application.getApplicationAttemptId())){
          	LOG.info(application.getApplicationAttemptId()+"into suspending list");
          	suspendedApps.add(application.getApplicationAttemptId());
          }
          //we suspend the container on this node
      	node.suspendContainer(container,rmContainer.getLastPreemptedResource());
      	toRelease = rmContainer.getLastPreemptedResource();
      	
        }else{
      	toRelease = rmContainer.getCurrentUsedResource();
      	removed =
      	            application.containerCompleted(rmContainer, containerStatus, event);
      	            node.releaseContainer(container,toRelease);
      	          //for container suspend event
      	 //in case of completing a suspended container 
        }
      }
      // Book-keeping
      if (removed) {
         //更新资源试用情况
        if(event == RMContainerEventType.SUSPEND){
        releaseResource(clusterResource, application,
      		  toRelease, node.getLabels(),true);
        }else{
        releaseResource(clusterResource, application,
          		  toRelease, node.getLabels(),false);  
        }
        LOG.info("completedContainer" +
            " container=" + container +
            " queue=" + this +
            " cluster=" + clusterResource);
      }
    }

    if (removed) {
      // Inform the parent queue _outside_ of the leaf-queue lock
      getParent().completedContainer(clusterResource, application, node,
        rmContainer, null, event, this, sortQueues);
    }
  }
}
 

private CookieContainerRequest getMatchingRequestWithoutPriority(
    Container container,
    String location,
    boolean considerContainerAffinity) {
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> pRequestsList =
    amRmClient.getMatchingRequestsForTopPriority(location, capability);
  if (considerContainerAffinity && 
      !priorityHasAffinity.contains(amRmClient.getTopPriority())) {
    considerContainerAffinity = false;
  }
  if (pRequestsList == null || pRequestsList.isEmpty()) {
    return null;
  }
  CookieContainerRequest firstMatch = null;
  for (Collection<CookieContainerRequest> requests : pRequestsList) {
    for (CookieContainerRequest cookieContainerRequest : requests) {
      if (firstMatch == null || // we dont have a match. So look for one 
          // we have a match but are looking for a better container level match.
          // skip the expensive canAssignTaskToContainer() if the request is 
          // not affinitized to the container
          container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
          ) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          // request matched to container
          if (!considerContainerAffinity) {
            return cookieContainerRequest;
          }
          ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
          boolean canMatchTaskWithAffinity = true;
          if (affCId == null || 
              !heldContainers.containsKey(affCId) ||
              inUseContainers.contains(affCId)) {
            // affinity not specified
            // affinitized container is no longer held
            // affinitized container is in use
            canMatchTaskWithAffinity = false;
          }
          if (canMatchTaskWithAffinity) {
            if (container.getId().equals(
                cookieContainerRequest.getAffinitizedContainer())) {
              // container level match
              if (LOG.isDebugEnabled()) {
                LOG.debug("Matching with affinity for request: "
                    + cookieContainerRequest + " container: " + affCId);
              }
              return cookieContainerRequest;
            }
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping request for container " + container.getId()
                  + " due to affinity. Request: " + cookieContainerRequest
                  + " affContainer: " + affCId);
            }
          } else {
            firstMatch = cookieContainerRequest;
          }
        }
      }
    }
  }
  
  return firstMatch;
}
 

private CookieContainerRequest getMatchingRequestWithoutPriority(
    Container container,
    String location,
    boolean considerContainerAffinity) {
  Resource capability = container.getResource();
  List<? extends Collection<CookieContainerRequest>> pRequestsList =
    amRmClient.getMatchingRequestsForTopPriority(location, capability);
  if (considerContainerAffinity && 
      !priorityHasAffinity.contains(amRmClient.getTopPriority())) {
    considerContainerAffinity = false;
  }
  if (pRequestsList == null || pRequestsList.isEmpty()) {
    return null;
  }
  CookieContainerRequest firstMatch = null;
  for (Collection<CookieContainerRequest> requests : pRequestsList) {
    for (CookieContainerRequest cookieContainerRequest : requests) {
      if (firstMatch == null || // we dont have a match. So look for one 
          // we have a match but are looking for a better container level match.
          // skip the expensive canAssignTaskToContainer() if the request is 
          // not affinitized to the container
          container.getId().equals(cookieContainerRequest.getAffinitizedContainer())
          ) {
        if (canAssignTaskToContainer(cookieContainerRequest, container)) {
          // request matched to container
          if (!considerContainerAffinity) {
            return cookieContainerRequest;
          }
          ContainerId affCId = cookieContainerRequest.getAffinitizedContainer();
          boolean canMatchTaskWithAffinity = true;
          if (affCId == null || 
              !heldContainers.containsKey(affCId) ||
              inUseContainers.contains(affCId)) {
            // affinity not specified
            // affinitized container is no longer held
            // affinitized container is in use
            canMatchTaskWithAffinity = false;
          }
          if (canMatchTaskWithAffinity) {
            if (container.getId().equals(
                cookieContainerRequest.getAffinitizedContainer())) {
              // container level match
              if (LOG.isDebugEnabled()) {
                LOG.debug("Matching with affinity for request: "
                    + cookieContainerRequest + " container: " + affCId);
              }
              return cookieContainerRequest;
            }
            if (LOG.isDebugEnabled()) {
              LOG.debug("Skipping request for container " + container.getId()
                  + " due to affinity. Request: " + cookieContainerRequest
                  + " affContainer: " + affCId);
            }
          } else {
            firstMatch = cookieContainerRequest;
          }
        }
      }
    }
  }
  
  return firstMatch;
}