org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example #2
Source File: TestProportionalCapacityPreemptionPolicy.java    From hadoop with Apache License 2.0 6 votes vote down vote up
void printString(CSQueue nq, String indent) {
  if (nq instanceof ParentQueue) {
    System.out.println(indent + nq.getQueueName()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (CSQueue q : ((ParentQueue)nq).getChildQueues()) {
      printString(q, indent + "  ");
    }
  } else {
    System.out.println(indent + nq.getQueueName()
        + " pen:" + ((LeafQueue) nq).getTotalResourcePending()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (FiCaSchedulerApp a : ((LeafQueue)nq).getApplications()) {
      System.out.println(indent + "  " + a.getApplicationId());
    }
  }
}
 
Example #3
Source File: FifoScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 
Example #4
Source File: ParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example #5
Source File: FifoScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private int assignNodeLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getNodeName());
  if (request != null) {
    // Don't allocate on this node if we don't need containers on this rack
    ResourceRequest rackRequest =
        application.getResourceRequest(priority, 
            node.getRMNode().getRackName());
    if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.NODE_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.NODE_LOCAL);
  }
  return assignedContainers;
}
 
Example #6
Source File: LeafQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Override
public Resource getPreemptedResource(){
 Resource preemptedResource = Resource.newInstance(0, 0);
 //try to resume containers which are suspended in fifo order
 for(ApplicationAttemptId appId: this.suspendedApps){
   FiCaSchedulerApp app = this.applicationAttemptMap.get(appId);
   	synchronized(app){
   	for(ContainerId cntId : app.getContainersSuspended()){
   		RMContainer rmContainer  =  app.getLiveContainersMap().get(cntId);
   		Resources.addTo(preemptedResource, rmContainer.getPreemptedResource());
   	   }
       }
   }
 
 return preemptedResource;
}
 
Example #7
Source File: LeafQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up
synchronized void releaseResource(Resource clusterResource, 
    FiCaSchedulerApp application, Resource resource, Set<String> nodeLabels,boolean isSuspend) {
  super.releaseResource(clusterResource, resource, nodeLabels,isSuspend);
  
  // Update user metrics
  String userName = application.getUser();
  User user = getUser(userName);
  user.releaseContainer(resource, nodeLabels);
  if(!isSuspend){
    metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
  }
  
  LOG.info(getQueueName() + 
          " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
          " headroom = " + application.getHeadroom() +
          " user-resources=" + user.getUsed()+"allocate resource:"+resource+
          " absUsed= "+getAbsoluteUsedCapacity()
          );
}
 
Example #8
Source File: ParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example #9
Source File: TestLeafQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testUserQueueAcl() throws Exception {

  // Manipulate queue 'a'
  LeafQueue d = stubLeafQueue((LeafQueue) queues.get(D));

  // Users
  final String user_d = "user_d";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 = TestUtils
      .getMockApplicationAttemptId(0, 1);
  FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_d, d, null,
      spyRMContext);
  d.submitApplicationAttempt(app_0, user_d);

  // Attempt the same application again
  final ApplicationAttemptId appAttemptId_1 = TestUtils
      .getMockApplicationAttemptId(0, 2);
  FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_d, d, null,
      spyRMContext);
  d.submitApplicationAttempt(app_1, user_d); // same user
}
 
Example #10
Source File: LeafQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void addApplicationAttempt(FiCaSchedulerApp application,
    User user) {
  // Accept 
  user.submitApplication();
  pendingApplications.add(application);
  applicationAttemptMap.put(application.getApplicationAttemptId(), application);

  // Activate applications
  activateApplications();
  
  LOG.info("Application added -" +
      " appId: " + application.getApplicationId() +
      " user: " + user + "," + " leaf-queue: " + getQueueName() +
      " #user-pending-applications: " + user.getPendingApplications() +
      " #user-active-applications: " + user.getActiveApplications() +
      " #queue-pending-applications: " + getNumPendingApplications() +
      " #queue-active-applications: " + getNumActiveApplications()
      );
}
 
Example #11
Source File: MyriadOperationsTest.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up
private RMContext generateRMContext(AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler) throws Exception {
  Configuration conf = new Configuration();
  MockRMContext context = null;
  Dispatcher dispatcher = new MockDispatcher();

  RMApplicationHistoryWriter rmApplicationHistoryWriter = new RMApplicationHistoryWriter(); 
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(dispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(dispatcher);    
  RMDelegationTokenSecretManager delegationTokenSecretManager = new RMDelegationTokenSecretManager(1, 1, 1, 1, context);

  context = new MockRMContext();
  context.setStateStore(TestObjectFactory.getStateStore(conf, "tmp/myriad-operations-test"));
  context.setAmLivelinessMonitor(amLivelinessMonitor);
  context.setAmFinishingMonitor(amFinishingMonitor);
  context.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  context.setRMDelegationTokenSecretManager(delegationTokenSecretManager);
  return context;
}
 
Example #12
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
boolean shouldAllocOrReserveNewContainer(FiCaSchedulerApp application,
    Priority priority, Resource required) {
  int requiredContainers = application.getTotalRequiredResources(priority);
  int reservedContainers = application.getNumReservedContainers(priority);
  int starvation = 0;
  if (reservedContainers > 0) {
    float nodeFactor = 
        Resources.ratio(
            resourceCalculator, required, getMaximumAllocation()
            );
    
    // Use percentage of node required to bias against large containers...
    // Protect against corner case where you need the whole node with
    // Math.min(nodeFactor, minimumAllocationFactor)
    // 在该优先级reserve被call的次数。
    starvation = 
        (int)((application.getReReservations(priority) / (float)reservedContainers) * 
              (1.0f - (Math.min(nodeFactor, getMinimumAllocationFactor())))
             );
    
    if (LOG.isDebugEnabled()) {
      LOG.debug("needsContainers:" +
          " app.#re-reserve=" + application.getReReservations(priority) + 
          " reserved=" + reservedContainers + 
          " nodeFactor=" + nodeFactor + 
          " minAllocFactor=" + getMinimumAllocationFactor() +
          " starvation=" + starvation);
    }
  }

  return (((starvation + requiredContainers) - reservedContainers) > 0);
}
 
Example #13
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized String moveApplication(ApplicationId appId,
    String targetQueueName) throws YarnException {
  FiCaSchedulerApp app =
      getApplicationAttempt(ApplicationAttemptId.newInstance(appId, 0));
  String sourceQueueName = app.getQueue().getQueueName();
  LeafQueue source = getAndCheckLeafQueue(sourceQueueName);
  String destQueueName = handleMoveToPlanQueue(targetQueueName);
  LeafQueue dest = getAndCheckLeafQueue(destQueueName);
  // Validation check - ACLs, submission limits for user & queue
  String user = app.getUser();
  try {
    dest.submitApplication(appId, user, destQueueName);
  } catch (AccessControlException e) {
    throw new YarnException(e);
  }
  // Move all live containers
  for (RMContainer rmContainer : app.getLiveContainers()) {
    source.detachContainer(clusterResource, app, rmContainer);
    // attach the Container to another queue
    dest.attachContainer(clusterResource, app, rmContainer);
  }
  // Detach the application..
  source.finishApplicationAttempt(app, sourceQueueName);
  source.getParent().finishApplication(appId, app.getUser());
  // Finish app & update metrics
  app.move(dest);
  // Submit to a new queue
  dest.submitApplicationAttempt(app, user);
  applications.get(appId).setQueue(dest);
  LOG.info("App: " + app.getApplicationId() + " successfully moved from "
      + sourceQueueName + " to: " + destQueueName);
  return targetQueueName;
}
 
Example #14
Source File: FifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  int maxContainers = 0;
  
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchRequest != null) {
    maxContainers = offSwitchRequest.getNumContainers();
  }

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 
Example #15
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {
  
  // Mock some methods for ease in these unit tests
  
  // 1. LeafQueue.createContainer to return dummy containers
  doAnswer(
      new Answer<Container>() {
        @Override
        public Container answer(InvocationOnMock invocation) 
            throws Throwable {
          final FiCaSchedulerApp application = 
              (FiCaSchedulerApp)(invocation.getArguments()[0]);
          final ContainerId containerId =                 
              TestUtils.getMockContainerId(application);

          Container container = TestUtils.getMockContainer(
              containerId,
              ((FiCaSchedulerNode)(invocation.getArguments()[1])).getNodeID(), 
              (Resource)(invocation.getArguments()[2]),
              ((Priority)invocation.getArguments()[3]));
          return container;
        }
      }
    ).
    when(queue).createContainer(
            any(FiCaSchedulerApp.class), 
            any(FiCaSchedulerNode.class), 
            any(Resource.class),
            any(Priority.class)
            );
  
  // 2. Stub out LeafQueue.parent.completedContainer
  CSQueue parent = queue.getParent();
  doNothing().when(parent).completedContainer(
      any(Resource.class), any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class), 
      any(RMContainer.class), any(ContainerStatus.class), 
      any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());
  
  return queue;
}
 
Example #16
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Resource assignRackLocalContainers(Resource clusterResource,
    ResourceRequest rackLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.RACK_LOCAL,
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        rackLocalResourceRequest, NodeType.RACK_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 
Example #17
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public synchronized Resource getTotalResourcePending() {
  Resource ret = BuilderUtils.newResource(0, 0, 0);
  for (FiCaSchedulerApp f : activeApplications) {
    Resources.addTo(ret, f.getTotalPendingRequests());
  }
  return ret;
}
 
Example #18
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private Resource assignRackLocalContainers(Resource clusterResource,
    ResourceRequest rackLocalResourceRequest, FiCaSchedulerNode node,
    FiCaSchedulerApp application, Priority priority,
    RMContainer reservedContainer, MutableObject allocatedContainer,
    ResourceLimits currentResoureLimits) {
  if (canAssign(application, priority, node, NodeType.RACK_LOCAL,
      reservedContainer)) {
    return assignContainer(clusterResource, node, application, priority,
        rackLocalResourceRequest, NodeType.RACK_LOCAL, reservedContainer,
        allocatedContainer, currentResoureLimits);
  }
  
  return Resources.none();
}
 
Example #19
Source File: TestReservations.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {

    // Mock some methods for ease in these unit tests

    // 1. LeafQueue.createContainer to return dummy containers
    doAnswer(new Answer<Container>() {
      @Override
      public Container answer(InvocationOnMock invocation) throws Throwable {
        final FiCaSchedulerApp application = (FiCaSchedulerApp) (invocation
            .getArguments()[0]);
        final ContainerId containerId = TestUtils
            .getMockContainerId(application);

        Container container = TestUtils.getMockContainer(containerId,
            ((FiCaSchedulerNode) (invocation.getArguments()[1])).getNodeID(),
            (Resource) (invocation.getArguments()[2]),
            ((Priority) invocation.getArguments()[3]));
        return container;
      }
    }).when(queue).createContainer(any(FiCaSchedulerApp.class),
        any(FiCaSchedulerNode.class), any(Resource.class), any(Priority.class));

    // 2. Stub out LeafQueue.parent.completedContainer
    CSQueue parent = queue.getParent();
    doNothing().when(parent).completedContainer(any(Resource.class),
        any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class),
        any(RMContainer.class), any(ContainerStatus.class),
        any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());

    return queue;
  }
 
Example #20
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
public synchronized void removeApplicationAttempt(
    FiCaSchedulerApp application, User user) {
  boolean wasActive = activeApplications.remove(application);
  if (!wasActive) {
    pendingApplications.remove(application);
  } else {
    queueUsage.decAMUsed(application.getAMResource());
    user.getResourceUsage().decAMUsed(application.getAMResource());
  }
  applicationAttemptMap.remove(application.getApplicationAttemptId());
  
  //remove from suspended list
  this.suspendedApps.remove(application.getApplicationAttemptId());
  
  user.finishApplication(wasActive);
  if (user.getTotalApplications() == 0) {
    users.remove(application.getUser());
  }

  // Check if we can activate more applications
  activateApplications();

  LOG.info("Application removed -" +
      " appId: " + application.getApplicationId() + 
      " user: " + application.getUser() + 
      " queue: " + getQueueName() +
      " #user-pending-applications: " + user.getPendingApplications() +
      " #user-active-applications: " + user.getActiveApplications() +
      " #queue-pending-applications: " + getNumPendingApplications() +
      " #queue-active-applications: " + getNumActiveApplications()
      );
}
 
Example #21
Source File: MyriadOperationsTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up
private MyriadOperations initialize() throws Exception {
  resetStoreState();
  SchedulerState sState = TestObjectFactory.getSchedulerState(cfg, "tmp/myriad-operations-test");
  sState.setFrameworkId(FrameworkID.newBuilder().setValue("mock-framework").build());
  AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler = TestObjectFactory.getYarnScheduler();
  MyriadDriverManager manager = TestObjectFactory.getMyriadDriverManager();
  MyriadWebServer webServer = TestObjectFactory.getMyriadWebServer(cfg);
  CompositeInterceptor registry = new CompositeInterceptor();
  LeastAMNodesFirstPolicy policy = new LeastAMNodesFirstPolicy(registry, scheduler, sState);

  manager.startDriver();

  return new MyriadOperations(cfg, sState, policy, manager, webServer, generateRMContext(scheduler));
}
 
Example #22
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
synchronized void releaseResource(Resource clusterResource, 
    FiCaSchedulerApp application, Resource resource, Set<String> nodeLabels) {
  super.releaseResource(clusterResource, resource, nodeLabels);
  
  // Update user metrics
  String userName = application.getUser();
  User user = getUser(userName);
  user.releaseContainer(resource, nodeLabels);
  metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
    
  LOG.info(getQueueName() +
      " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
      " user=" + userName + " user-resources=" + user.getUsed());
}
 
Example #23
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Lock({LeafQueue.class, FiCaSchedulerApp.class})
Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
    Resource clusterResource, Resource required, Set<String> requestedLabels) {
  String user = application.getUser();
  User queueUser = getUser(user);

  // Compute user limit respect requested labels,
  // TODO, need consider headroom respect labels also
  Resource userLimit =
      computeUserLimit(application, clusterResource, required,
          queueUser, requestedLabels);

  setQueueResourceLimitsInfo(clusterResource);
  
  Resource headroom =
      getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(),
          clusterResource, userLimit);
  
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for user " + user + ": " + 
        " userLimit=" + userLimit + 
        " queueMaxAvailRes=" + cachedResourceLimitsForHeadroom.getLimit() +
        " consumed=" + queueUser.getUsed() + 
        " headroom=" + headroom);
  }
  
  CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider(
    queueUser, this, application, required, queueResourceLimitsInfo);
  
  application.setHeadroomProvider(headroomProvider);

  metrics.setAvailableResourcesToUser(user, headroom);
  
  return userLimit;
}
 
Example #24
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public synchronized void removeApplicationAttempt(
    FiCaSchedulerApp application, User user) {
  String partitionName = application.getAppAMNodePartitionName();
  boolean wasActive = activeApplications.remove(application);
  if (!wasActive) {
    pendingApplications.remove(application);
  } else {
    queueUsage.decAMUsed(partitionName,
        application.getAMResource(partitionName));
    user.getResourceUsage().decAMUsed(partitionName,
        application.getAMResource(partitionName));
  }
  applicationAttemptMap.remove(application.getApplicationAttemptId());

  user.finishApplication(wasActive);
  if (user.getTotalApplications() == 0) {
    users.remove(application.getUser());
  }

  // Check if we can activate more applications
  activateApplications();

  LOG.info("Application removed -" +
      " appId: " + application.getApplicationId() + 
      " user: " + application.getUser() + 
      " queue: " + getQueueName() +
      " #user-pending-applications: " + user.getPendingApplications() +
      " #user-active-applications: " + user.getActiveApplications() +
      " #queue-pending-applications: " + getNumPendingApplications() +
      " #queue-active-applications: " + getNumActiveApplications()
      );
}
 
Example #25
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) {
  // Careful! Locking order is important!
  synchronized (this) {
    removeApplicationAttempt(application, getUser(application.getUser()));
  }
  getParent().finishApplicationAttempt(application, queue);
}
 
Example #26
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Container getContainer(RMContainer rmContainer, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    Resource capability, Priority priority) {
 //是否是reserve 的情况
  return (rmContainer != null) ? rmContainer.getContainer() :
    createContainer(application, node, capability, priority);
}
 
Example #27
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void reserve(FiCaSchedulerApp application, Priority priority, 
    FiCaSchedulerNode node, RMContainer rmContainer, Container container) {
  // Update reserved metrics if this is the first reservation
  if (rmContainer == null) {
    getMetrics().reserveResource(
        application.getUser(), container.getResource());
  }

  // Inform the application 
  rmContainer = application.reserve(node, priority, rmContainer, container);
  
  // Update the node
  node.reserveResource(application, priority, rmContainer);
}
 
Example #28
Source File: CSAssignment.java    From big-c with Apache License 2.0 5 votes vote down vote up
public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) {
  this.resource = excessReservation.getContainer().getResource();
  this.type = NodeType.NODE_LOCAL;
  this.application = application;
  this.excessReservation = excessReservation;
  this.skipped = false;
}
 
Example #29
Source File: CapacityScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Lock(CapacityScheduler.class)
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }
  
  Container container = rmContainer.getContainer();
  
  // Get the application for the finished container
  FiCaSchedulerApp application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  if (application == null) {
    LOG.info("Container " + container + " of" + " unknown application "
        + appId + " completed with event " + event);
    return;
  }
  
  // Get the node on which the container was allocated
  FiCaSchedulerNode node = getNode(container.getNodeId());
  
  // Inform the queue
  LeafQueue queue = (LeafQueue)application.getQueue();
  queue.completedContainer(clusterResource, application, node, 
      rmContainer, containerStatus, event, null, true);

  LOG.info("Application attempt " + application.getApplicationAttemptId()
      + " released container " + container.getId() + " on node: " + node
      + " with event: " + event);
}
 
Example #30
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void updateClusterResource(Resource clusterResource,
    ResourceLimits currentResourceLimits) {
  updateCurrentResourceLimits(currentResourceLimits, clusterResource);
  lastClusterResource = clusterResource;
  updateAbsoluteCapacityResource(clusterResource);
  
  // Update headroom info based on new cluster resource value
  // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
  // during allocation
  setQueueResourceLimitsInfo(clusterResource);
  
  // Update metrics
  CSQueueUtils.updateQueueStatistics(
      resourceCalculator, this, getParent(), clusterResource, 
      minimumAllocation);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();

  // Update application properties
  for (FiCaSchedulerApp application : activeApplications) {
    synchronized (application) {
      computeUserLimitAndSetHeadroom(application, clusterResource, 
          Resources.none(), null);
    }
  }
}