Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer#handle()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer#handle() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
public synchronized void adjusctContianerResouce(ContainerId contianerId, Resource resource){
 
 RMContainer container = getRMContainer(contianerId);
 //get original resource
 Resource allocatedResource = container.getAllocatedResource();
 Resource marginalResource=Resources.subtract(allocatedResource, resource);
 
 if(marginalResource.getMemory() > 0 ){		  
  LOG.info("increase memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());		  
 }else{		  
  LOG.info("decrease memory on continaer"+contianerId.getContainerId()+"by"+marginalResource.getMemory());
 }
 
 if(marginalResource.getVirtualCores() > 0){
  LOG.info("increase cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }else{
  LOG.info("dncrease cores on contianer"+contianerId.getContainerId()+"by"+marginalResource.getVirtualCores());
 }
 
 SchedulerNode schedulerNode = getSchedulerNode(container.getAllocatedNode());
 //now inform the schedulerNode to adjust resurce assumption
 schedulerNode.addAvailableResource(marginalResource);
 //now inform RMContinaer to adjuct its allocatedResource
 container.handle(new RMContainerResourceUpdateEvent(contianerId, RMContainerEventType.RESOURCE_UPDATE, resource));
 	  
}
 
Example 2
Source File: SchedulerApplicationAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public synchronized ContainersAndNMTokensAllocation
    pullNewlyAllocatedContainersAndNMTokens() {
  List<Container> returnContainerList =
      new ArrayList<Container>(newlyAllocatedContainers.size());
  List<NMToken> nmTokens = new ArrayList<NMToken>();
  for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
    .hasNext();) {
    RMContainer rmContainer = i.next();
    Container container = rmContainer.getContainer();
    try {
      // create container token and NMToken altogether.
      container.setContainerToken(rmContext.getContainerTokenSecretManager()
        .createContainerToken(container.getId(), container.getNodeId(),
          getUser(), container.getResource(), container.getPriority(),
          rmContainer.getCreationTime(), this.logAggregationContext));
      NMToken nmToken =
          rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
            getApplicationAttemptId(), container);
      if (nmToken != null) {
        nmTokens.add(nmToken);
      }
    } catch (IllegalArgumentException e) {
      // DNS might be down, skip returning this container.
      LOG.error("Error trying to assign container token and NM token to" +
          " an allocated container " + container.getId(), e);
      continue;
    }
    returnContainerList.add(container);
    i.remove();
    rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
      RMContainerEventType.ACQUIRED));
  }
  return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
}
 
Example 3
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
synchronized public void containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  
  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);
  
  // Remove from the list of containers
  liveContainers.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  Resources.subtractFrom(currentConsumption, containerResource);

  // remove from preemption map if it is completed
  preemptionMap.remove(rmContainer);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;
}
 
Example 4
Source File: SchedulerApplicationAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
public synchronized ContainersAndNMTokensAllocation
    pullNewlyAllocatedContainersAndNMTokens() {
  List<Container> returnContainerList =
      new ArrayList<Container>(newlyAllocatedContainers.size());
  List<NMToken> nmTokens = new ArrayList<NMToken>();
  for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
    .hasNext();) {
    RMContainer rmContainer = i.next();
    Container container = rmContainer.getContainer();
    try {
      // create container token and NMToken altogether.
      container.setContainerToken(rmContext.getContainerTokenSecretManager()
        .createContainerToken(container.getId(), container.getNodeId(),
          getUser(), container.getResource(), container.getPriority(),
          rmContainer.getCreationTime(), this.logAggregationContext));
      NMToken nmToken =
          rmContext.getNMTokenSecretManager().createAndGetNMToken(getUser(),
            getApplicationAttemptId(), container);
      if (nmToken != null) {
        nmTokens.add(nmToken);
      }
    } catch (IllegalArgumentException e) {
      // DNS might be down, skip returning this container.
      LOG.error("Error trying to assign container token and NM token to" +
          " an allocated container " + container.getId(), e);
      continue;
    }
    returnContainerList.add(container);
    i.remove();
    rmContainer.handle(new RMContainerEvent(rmContainer.getContainerId(),
      RMContainerEventType.ACQUIRED));
  }
  return new ContainersAndNMTokensAllocation(returnContainerList, nmTokens);
}
 
Example 5
Source File: SchedulerApplicationAttempt.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void containerLaunchedOnNode(ContainerId containerId,
    NodeId nodeId) {
  // Inform the container
  RMContainer rmContainer = getRMContainer(containerId);
  if (rmContainer == null) {
    // Some unknown container sneaked into the system. Kill it.
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
    return;
  }

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
}
 
Example 6
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
synchronized public void containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  
  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);
  
  // Remove from the list of containers
  liveContainers.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  this.attemptResourceUsage.decUsed(containerResource);

  // remove from preemption map if it is completed
  preemptionMap.remove(rmContainer);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;
}
 
Example 7
Source File: SchedulerApplicationAttempt.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public synchronized void containerLaunchedOnNode(ContainerId containerId,
    NodeId nodeId) {
  // Inform the container
  RMContainer rmContainer = getRMContainer(containerId);
  if (rmContainer == null) {
    // Some unknown container sneaked into the system. Kill it.
    rmContext.getDispatcher().getEventHandler()
      .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
    return;
  }

  rmContainer.handle(new RMContainerEvent(containerId,
      RMContainerEventType.LAUNCHED));
}
 
Example 8
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 4 votes vote down vote up
synchronized public boolean containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event,
    String partition) {

  // Remove from the list of containers
  if (null == liveContainers.remove(rmContainer.getContainerId())) {
    return false;
  }
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);

  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();

  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);

  containersToPreempt.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(),
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
      getApplicationId(), containerId);
  
  // Update usage metrics 
  Resource containerResource = rmContainer.getContainer().getResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  attemptResourceUsage.decUsed(partition, containerResource);

  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;

  return true;
}
 
Example 9
Source File: FiCaSchedulerApp.java    From hadoop with Apache License 2.0 4 votes vote down vote up
synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
    Priority priority, ResourceRequest request, 
    Container container) {

  if (isStopped) {
    return null;
  }
  
  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, this
      .getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), this.rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);

  attemptResourceUsage.incUsed(node.getPartition(), container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl)rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(),
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 
Example 10
Source File: AbstractYarnScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public synchronized void recoverContainersOnNode(
    List<NMContainerStatus> containerReports, RMNode nm) {
  if (!rmContext.isWorkPreservingRecoveryEnabled()
      || containerReports == null
      || (containerReports != null && containerReports.isEmpty())) {
    return;
  }

  for (NMContainerStatus container : containerReports) {
    ApplicationId appId =
        container.getContainerId().getApplicationAttemptId().getApplicationId();
    RMApp rmApp = rmContext.getRMApps().get(appId);
    if (rmApp == null) {
      LOG.error("Skip recovering container " + container
          + " for unknown application.");
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    // Unmanaged AM recovery is addressed in YARN-1815
    if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
      LOG.info("Skip recovering container " + container + " for unmanaged AM."
          + rmApp.getApplicationId());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    SchedulerApplication<T> schedulerApp = applications.get(appId);
    if (schedulerApp == null) {
      LOG.info("Skip recovering container  " + container
          + " for unknown SchedulerApplication. Application current state is "
          + rmApp.getState());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    LOG.info("Recovering container " + container);
    SchedulerApplicationAttempt schedulerAttempt =
        schedulerApp.getCurrentAppAttempt();

    if (!rmApp.getApplicationSubmissionContext()
      .getKeepContainersAcrossApplicationAttempts()) {
      // Do not recover containers for stopped attempt or previous attempt.
      if (schedulerAttempt.isStopped()
          || !schedulerAttempt.getApplicationAttemptId().equals(
            container.getContainerId().getApplicationAttemptId())) {
        LOG.info("Skip recovering container " + container
            + " for already stopped attempt.");
        killOrphanContainerOnNode(nm, container);
        continue;
      }
    }

    // create container
    RMContainer rmContainer = recoverAndCreateContainer(container, nm);

    // recover RMContainer
    rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(),
      container));

    // recover scheduler node
    SchedulerNode schedulerNode = nodes.get(nm.getNodeID());
    schedulerNode.recoverContainer(rmContainer);

    // recover queue: update headroom etc.
    Queue queue = schedulerAttempt.getQueue();
    queue.recoverContainer(clusterResource, schedulerAttempt, rmContainer);

    // recover scheduler attempt
    schedulerAttempt.recoverContainer(schedulerNode, rmContainer);
          
    // set master container for the current running AMContainer for this
    // attempt.
    RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
    if (appAttempt != null) {
      Container masterContainer = appAttempt.getMasterContainer();

      // Mark current running AMContainer's RMContainer based on the master
      // container ID stored in AppAttempt.
      if (masterContainer != null
          && masterContainer.getId().equals(rmContainer.getContainerId())) {
        ((RMContainerImpl)rmContainer).setAMContainer(true);
      }
    }

    synchronized (schedulerAttempt) {
      Set<ContainerId> releases = schedulerAttempt.getPendingRelease();
      if (releases.contains(container.getContainerId())) {
        // release the container
        rmContainer.handle(new RMContainerFinishedEvent(container
          .getContainerId(), SchedulerUtils.createAbnormalContainerStatus(
          container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER),
          RMContainerEventType.RELEASED));
        releases.remove(container.getContainerId());
        LOG.info(container.getContainerId() + " is released by application.");
      }
    }
  }
}
 
Example 11
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 4 votes vote down vote up
synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
    Priority priority, ResourceRequest request,
    Container container) {
  // Update allowed locality level
  NodeType allowed = allowedLocalityLevel.get(priority);
  if (allowed != null) {
    if (allowed.equals(NodeType.OFF_SWITCH) &&
        (type.equals(NodeType.NODE_LOCAL) ||
            type.equals(NodeType.RACK_LOCAL))) {
      this.resetAllowedLocalityLevel(priority, type);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL) &&
        type.equals(NodeType.NODE_LOCAL)) {
      this.resetAllowedLocalityLevel(priority, type);
    }
  }

  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, 
      getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  this.attemptResourceUsage.incUsed(container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 
Example 12
Source File: FSAppAttempt.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
    Priority priority, ResourceRequest request,
    Container container) {
  // Update allowed locality level
  NodeType allowed = allowedLocalityLevel.get(priority);
  if (allowed != null) {
    if (allowed.equals(NodeType.OFF_SWITCH) &&
        (type.equals(NodeType.NODE_LOCAL) ||
            type.equals(NodeType.RACK_LOCAL))) {
      this.resetAllowedLocalityLevel(priority, type);
    }
    else if (allowed.equals(NodeType.RACK_LOCAL) &&
        type.equals(NodeType.NODE_LOCAL)) {
      this.resetAllowedLocalityLevel(priority, type);
    }
  }

  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, 
      getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  Resources.addTo(currentConsumption, container.getResource());

  // Update resource requests related to "request" and store in RMContainer
  ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}
 
Example 13
Source File: AbstractYarnScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
public synchronized void recoverContainersOnNode(
    List<NMContainerStatus> containerReports, RMNode nm) {
  if (!rmContext.isWorkPreservingRecoveryEnabled()
      || containerReports == null
      || (containerReports != null && containerReports.isEmpty())) {
    return;
  }

  for (NMContainerStatus container : containerReports) {
    ApplicationId appId =
        container.getContainerId().getApplicationAttemptId().getApplicationId();
    RMApp rmApp = rmContext.getRMApps().get(appId);
    if (rmApp == null) {
      LOG.error("Skip recovering container " + container
          + " for unknown application.");
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    // Unmanaged AM recovery is addressed in YARN-1815
    if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
      LOG.info("Skip recovering container " + container + " for unmanaged AM."
          + rmApp.getApplicationId());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    SchedulerApplication<T> schedulerApp = applications.get(appId);
    if (schedulerApp == null) {
      LOG.info("Skip recovering container  " + container
          + " for unknown SchedulerApplication. Application current state is "
          + rmApp.getState());
      killOrphanContainerOnNode(nm, container);
      continue;
    }

    LOG.info("Recovering container " + container);
    SchedulerApplicationAttempt schedulerAttempt =
        schedulerApp.getCurrentAppAttempt();

    if (!rmApp.getApplicationSubmissionContext()
      .getKeepContainersAcrossApplicationAttempts()) {
      // Do not recover containers for stopped attempt or previous attempt.
      if (schedulerAttempt.isStopped()
          || !schedulerAttempt.getApplicationAttemptId().equals(
            container.getContainerId().getApplicationAttemptId())) {
        LOG.info("Skip recovering container " + container
            + " for already stopped attempt.");
        killOrphanContainerOnNode(nm, container);
        continue;
      }
    }

    // create container
    RMContainer rmContainer = recoverAndCreateContainer(container, nm);

    // recover RMContainer
    rmContainer.handle(new RMContainerRecoverEvent(container.getContainerId(),
      container));

    // recover scheduler node
    nodes.get(nm.getNodeID()).recoverContainer(rmContainer);

    // recover queue: update headroom etc.
    Queue queue = schedulerAttempt.getQueue();
    queue.recoverContainer(clusterResource, schedulerAttempt, rmContainer);

    // recover scheduler attempt
    schedulerAttempt.recoverContainer(rmContainer);
          
    // set master container for the current running AMContainer for this
    // attempt.
    RMAppAttempt appAttempt = rmApp.getCurrentAppAttempt();
    if (appAttempt != null) {
      Container masterContainer = appAttempt.getMasterContainer();

      // Mark current running AMContainer's RMContainer based on the master
      // container ID stored in AppAttempt.
      if (masterContainer != null
          && masterContainer.getId().equals(rmContainer.getContainerId())) {
        ((RMContainerImpl)rmContainer).setAMContainer(true);
      }
    }

    synchronized (schedulerAttempt) {
      Set<ContainerId> releases = schedulerAttempt.getPendingRelease();
      if (releases.contains(container.getContainerId())) {
        // release the container
        rmContainer.handle(new RMContainerFinishedEvent(container
          .getContainerId(), SchedulerUtils.createAbnormalContainerStatus(
          container.getContainerId(), SchedulerUtils.RELEASED_CONTAINER),
          RMContainerEventType.RELEASED));
        releases.remove(container.getContainerId());
        LOG.info(container.getContainerId() + " is released by application.");
      }
    }
  }
}
 
Example 14
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized public boolean containerSuspend(RMContainer rmContainer,
  ContainerStatus containerStatus, RMContainerEventType event){
 //we try to find it from live container list
 LOG.info("app suspend "+rmContainer.getContainerId());
 if (!liveContainers.keySet().contains(rmContainer.getContainerId())){
  LOG.info("container not found "+rmContainer.getContainerId());
  return false;
 }
 
 isSuspending = true;

 Container container = rmContainer.getContainer();
 ContainerId containerId = container.getId();
 
 
 if(!this.containersSuspended.contains(rmContainer.getContainerId())){
 //add to suspended set if this container is first suspended
 containersSuspended.add(containerId);
 }
	
 // Inform the container
 
 rmContainer.handle(
       new RMContainerFinishedEvent(
           containerId,
           containerStatus, 
           event)
 );

 // Update usage metrics,we release resource here,to support increamental suspension  
 Resource toPreempted = rmContainer.getLastPreemptedResource();
 queue.getMetrics().releaseResources(getUser(), 1, toPreempted);
 Resources.subtractFrom(currentConsumption, toPreempted);

 LOG.info("app suspend container: " + rmContainer.getContainerId() + 
        " in state: " + rmContainer.getState() + " resource:" + toPreempted);
 
 // Clear resource utilization metrics cache.
 lastMemoryAggregateAllocationUpdateTime = -1;
 
 return true; 	  
}
 
Example 15
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized public boolean containerCompleted(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {

  // Remove from the list of containers
  if (null == liveContainers.remove(rmContainer.getContainerId())) {
    return false;
  }
  
  // Remove from the list of newly allocated containers if found
  newlyAllocatedContainers.remove(rmContainer);

  Container container = rmContainer.getContainer();
  ContainerId containerId = container.getId();
  
  //we are trying to complete a suspeded container
  containersSuspended.remove(containerId);
  
  // Inform the container
  rmContainer.handle(
      new RMContainerFinishedEvent(
          containerId,
          containerStatus, 
          event)
      );
  
  LOG.info("Completed container: " + rmContainer.getContainerId() + 
      " in state: " + rmContainer.getState() + " event:" + event);

  containersToPreempt.remove(rmContainer.getContainerId());

  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.RELEASE_CONTAINER, "SchedulerApp", 
      getApplicationId(), containerId);
  
  Resource containerResource = rmContainer.getCurrentUsedResource();
  queue.getMetrics().releaseResources(getUser(), 1, containerResource);
  Resources.subtractFrom(currentConsumption, containerResource);
  
  LOG.info("Completed container: after substract resource is "+containerResource);
  
  // Clear resource utilization metrics cache.
  lastMemoryAggregateAllocationUpdateTime = -1;

  return true;
}
 
Example 16
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized public boolean containerResume(RMContainer rmContainer,Resource toResume){
 
 
 
 ContainerId containerId = rmContainer.getContainerId();
 
 if(isStopped){
  return false;
 }
 
 if(!isSuspending){
  return false;
 }
 
 if(!containersSuspended.contains(containerId)){
  return false;
 }
 //add resumed resource
 rmContainer.addResumedResource(toResume);
 
 //we try to update its resource consumption
 rmContainer.handle(
            new RMContainerEvent(containerId,RMContainerEventType.RESUME)  
     );

 //if all of its resource has been resumed
 if(!rmContainer.isSuspending()){
 //delete contaienr from containersSuspended
 this.containersSuspended.remove(containerId);
 }  
 
 //update resource usage
 queue.getMetrics().allocateResources(getUser(), 1, toResume, true);
 //update app resource usage
 Resources.addTo(currentConsumption,toResume);
 //inform RMContainer
 if(this.containersSuspended.size() == 0){  
  isSuspending = false;
  LOG.info("application "+this.getApplicationId()+"has been out of the suspended list");
 }
 
 LOG.info("app "+this.getApplicationAttemptId()+" consume resource"+currentConsumption);

   if (LOG.isDebugEnabled()) {
     LOG.debug("allocate: applicationAttemptId=" 
         + this.getApplicationAttemptId() 
         + " container=" + containerId + " host="
         + rmContainer.getContainer().getNodeId().getHost() 
       );
   }
 
 return true;
 
}
 
Example 17
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 4 votes vote down vote up
synchronized public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
    Priority priority, ResourceRequest request, 
    Container container) {

  if (isStopped) {
    return null;
  }
  
  // Required sanity check - AM can call 'allocate' to update resource 
  // request without locking the scheduler, hence we need to check
  if (getTotalRequiredResources(priority) <= 0) {
    return null;
  }
  
  // Create RMContainer
  RMContainer rmContainer = new RMContainerImpl(container, this
      .getApplicationAttemptId(), node.getNodeID(),
      appSchedulingInfo.getUser(), this.rmContext);

  // Add it to allContainers list.
  newlyAllocatedContainers.add(rmContainer);
  liveContainers.put(container.getId(), rmContainer);    

  // Update consumption and track allocations
  List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
      type, node, priority, request, container);
  Resources.addTo(currentConsumption, container.getResource());
  
  // Update resource requests related to "request" and store in RMContainer 
  ((RMContainerImpl)rmContainer).setResourceRequests(resourceRequestList);

  // Inform the container
  rmContainer.handle(
      new RMContainerEvent(container.getId(), RMContainerEventType.START));

  if (LOG.isDebugEnabled()) {
    LOG.debug("allocate: applicationAttemptId=" 
        + container.getId().getApplicationAttemptId() 
        + " container=" + container.getId() + " host="
        + container.getNodeId().getHost() + " type=" + type);
  }
  RMAuditLogger.logSuccess(getUser(), 
      AuditConstants.ALLOC_CONTAINER, "SchedulerApp", 
      getApplicationId(), container.getId());
  
  return rmContainer;
}