Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode#getLabels()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode#getLabels() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ParentQueue.java From hadoop with Apache License 2.0

6 votes

@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}

Example 2

Source File: ParentQueue.java From hadoop with Apache License 2.0

6 votes

@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}

Example 3

Source File: ParentQueue.java From hadoop with Apache License 2.0

6 votes

@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels());
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}

Example 4

Source File: ParentQueue.java From big-c with Apache License 2.0

6 votes

@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}

Example 5

Source File: ParentQueue.java From big-c with Apache License 2.0

6 votes

@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}

Example 6

Source File: ParentQueue.java From big-c with Apache License 2.0

6 votes

@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}

Example 7

Source File: ParentQueue.java From hadoop with Apache License 2.0

4 votes

@Override
public synchronized CSAssignment assignContainers(Resource clusterResource,
    FiCaSchedulerNode node, ResourceLimits resourceLimits) {
  CSAssignment assignment = 
      new CSAssignment(Resources.createResource(0, 0, 0), NodeType.NODE_LOCAL);
  Set<String> nodeLabels = node.getLabels();
  
  // if our queue cannot access this node, just return
  if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
    return assignment;
  }
  
  while (canAssign(clusterResource, node)) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to assign containers to child-queue of "
        + getQueueName());
    }
    
    // Are we over maximum-capacity for this queue?
    // This will also consider parent's limits and also continuous reservation
    // looking
    if (!super.canAssignToThisQueue(clusterResource, nodeLabels, resourceLimits,
        minimumAllocation, Resources.createResource(getMetrics()
            .getReservedMB(), getMetrics().getReservedVirtualCores(), getMetrics().getReservedGpuCores()))) {
      break;
    }
    
    // Schedule
    CSAssignment assignedToChild = 
        assignContainersToChildQueues(clusterResource, node, resourceLimits);
    assignment.setType(assignedToChild.getType());
    
    // Done if no child-queue assigned anything
    if (Resources.greaterThan(
            resourceCalculator, clusterResource, 
            assignedToChild.getResource(), Resources.none())) {
      // Track resource utilization for the parent-queue
      super.allocateResource(clusterResource, assignedToChild.getResource(),
          nodeLabels);
      
      // Track resource utilization in this pass of the scheduler
      Resources.addTo(assignment.getResource(), assignedToChild.getResource());
      
      LOG.info("assignedContainer" +
          " queue=" + getQueueName() + 
          " usedCapacity=" + getUsedCapacity() +
          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
          " used=" + queueUsage.getUsed() + 
          " cluster=" + clusterResource);

    } else {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("ParentQ=" + getQueueName()
        + " assignedSoFarInThisIteration=" + assignment.getResource()
        + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity());
    }

    // Do not assign more than one container if this isn't the root queue
    // or if we've already assigned an off-switch container
    if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) {
      if (LOG.isDebugEnabled()) {
        if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) {
          LOG.debug("Not assigning more than one off-switch container," +
              " assignments so far: " + assignment);
        }
      }
      break;
    }
  } 
  
  return assignment;
}

Example 8

Source File: ParentQueue.java From hadoop with Apache License 2.0

4 votes

@Override
public void completedContainer(Resource clusterResource,
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    RMContainer rmContainer, ContainerStatus containerStatus, 
    RMContainerEventType event, CSQueue completedChildQueue,
    boolean sortQueues) {
  if (application != null) {
    // Careful! Locking order is important!
    // Book keeping
    synchronized (this) {
      super.releaseResource(clusterResource, rmContainer.getContainer()
          .getResource(), node.getLabels());

      LOG.info("completedContainer" +
          " queue=" + getQueueName() + 
          " usedCapacity=" + getUsedCapacity() +
          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
          " used=" + queueUsage.getUsed() + 
          " cluster=" + clusterResource);

      // Note that this is using an iterator on the childQueues so this can't
      // be called if already within an iterator for the childQueues. Like
      // from assignContainersToChildQueues.
      if (sortQueues) {
        // reinsert the updated queue
        for (Iterator<CSQueue> iter = childQueues.iterator();
             iter.hasNext();) {
          CSQueue csqueue = iter.next();
          if(csqueue.equals(completedChildQueue)) {
            iter.remove();
            LOG.info("Re-sorting completed queue: " + csqueue.getQueuePath() +
                " stats: " + csqueue);
            childQueues.add(csqueue);
            break;
          }
        }
      }
    }

    // Inform the parent
    if (parent != null) {
      // complete my parent
      parent.completedContainer(clusterResource, application, 
          node, rmContainer, null, event, this, sortQueues);
    }    
  }
}

Example 9

Source File: ParentQueue.java From big-c with Apache License 2.0

4 votes

@Override
public synchronized CSAssignment assignContainers(Resource clusterResource,
    FiCaSchedulerNode node, ResourceLimits resourceLimits) {
  CSAssignment assignment = 
      new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
  Set<String> nodeLabels = node.getLabels();
  
  // if our queue cannot access this node, just return
  if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
    return assignment;
  }
  
  //每次分配一个container 给node节点上的应用，直到不能非配为止
  while (canAssign(clusterResource, node)) {
    if (LOG.isDebugEnabled()) 
    {
      LOG.info("Trying to assign containers to child-queue of "
        + getQueueName());
    }
    
    //LOG.info("cluster resource:"+clusterResource.getMemory()+"node:"+node.getNodeName()+"resource limits:"+resourceLimits.getLimit().getMemory());
    // Are we over maximum-capacity for this queue?
    // This will also consider parent's limits and also continuous reservation
    // looking,try to allocate at minimumAllocation to see if it's can be allocated
    if (!super.canAssignToThisQueue(clusterResource, nodeLabels, resourceLimits,
        minimumAllocation, Resources.createResource(getMetrics()
            .getReservedMB(), getMetrics().getReservedVirtualCores()))) {
  	LOG.info("over maximum-capacity of this queue "+getQueueName());
      break;
    }
    
    // Schedule
    CSAssignment assignedToChild = 
        assignContainersToChildQueues(clusterResource, node, resourceLimits);
    assignment.setType(assignedToChild.getType());
    
    // Done if no child-queue assigned anything
    //if get allocated in child queue
    if (Resources.greaterThan(
            resourceCalculator, clusterResource, 
            assignedToChild.getResource(), Resources.none())) {
  	  
      // Track resource utilization for the parent-queue
  	if(assignedToChild.getContainersToResume().size() > 0){
      super.allocateResource(clusterResource, assignedToChild.getResource(),
          nodeLabels,true);
  	}else{
  	super.allocateResource(clusterResource, assignedToChild.getResource(),
  	    nodeLabels,false);	
  	}
      
      // Track resource utilization in this pass of the scheduler
      //Resources.addTo(assignment.getResource(), assignedToChild.getResource());
      //we need to merge the resumed containers and accumulate the resource
      assignment.merge(assignedToChild);
      
      LOG.info("assignedContainer" +
          " queue=" + getQueueName() + 
          " usedCapacity=" + getUsedCapacity() +
          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
          " used=" + queueUsage.getUsed() + 
          " cluster=" + clusterResource+
          " resume size="+assignment.getContainersToResume().size());

    } else {
      break;
    }
    if (LOG.isDebugEnabled())
   {
      LOG.info("ParentQ=" + getQueueName()
        + " assignedSoFarInThisIteration=" + assignment.getResource()
        + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity());
    }

    // Do not assign more than one container if this isn't the root queue
    // or if we've already assigned an off-switch container why?
    if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) {
      if (LOG.isDebugEnabled()) {
        if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) {
          LOG.debug("Not assigning more than one off-switch container," +
              " assignments so far: " + assignment);
        }
      }
      break;
    }
   } 
  return assignment;
}

Example 10

Source File: ParentQueue.java From big-c with Apache License 2.0

4 votes

@Override
public void completedContainer(Resource clusterResource,
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    RMContainer rmContainer, ContainerStatus containerStatus, 
    RMContainerEventType event, CSQueue completedChildQueue,
    boolean sortQueues) {
  if (application != null) {
    // Careful! Locking order is important!
    // Book keeping
    synchronized (this) {
  	Resource toRelease;
  	
  	if(event == RMContainerEventType.SUSPEND){
  		toRelease = rmContainer.getLastPreemptedResource();
  		LOG.info("parent queue suspend resource "+toRelease);
  		super.releaseResource(clusterResource, toRelease ,node.getLabels(),true);
  	}else{
  		toRelease = rmContainer.getCurrentUsedResource(); 
  		LOG.info("parent queue release resource "+toRelease);
  		super.releaseResource(clusterResource, toRelease ,node.getLabels(),false);
  	}
  	
     

      LOG.info("completedContainer/suspend" +
          " queue=" + getQueueName() + 
          " usedCapacity=" + getUsedCapacity() +
          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
          " used=" + queueUsage.getUsed() + 
          " cluster=" + clusterResource+"released resource: "+toRelease);

      // Note that this is using an iterator on the childQueues so this can't
      // be called if already within an iterator for the childQueues. Like
      // from assignContainersToChildQueues.
      if (sortQueues) {
        // reinsert the updated queue
        for (Iterator<CSQueue> iter = childQueues.iterator();
             iter.hasNext();) {
          CSQueue csqueue = iter.next();
          if(csqueue.equals(completedChildQueue)) {
            iter.remove();
            LOG.info("Re-sorting completed queue: " + csqueue.getQueuePath() +
                " stats: " + csqueue);
            childQueues.add(csqueue);
            break;
          }
        }
      }
    }

    // Inform the parent
    if (parent != null) {
      // complete my parent
      parent.completedContainer(clusterResource, application, 
          node, rmContainer, null, event, this, sortQueues);
    }    
  }
}