Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: incubator-myriad   Source File: MyriadOperationsTest.java    License: Apache License 2.0 6 votes vote down vote up
private RMContext generateRMContext(AbstractYarnScheduler<FiCaSchedulerApp, FiCaSchedulerNode> scheduler) throws Exception {
  Configuration conf = new Configuration();
  MockRMContext context = null;
  Dispatcher dispatcher = new MockDispatcher();

  RMApplicationHistoryWriter rmApplicationHistoryWriter = new RMApplicationHistoryWriter(); 
  AMLivelinessMonitor amLivelinessMonitor = new AMLivelinessMonitor(dispatcher);
  AMLivelinessMonitor amFinishingMonitor = new AMLivelinessMonitor(dispatcher);    
  RMDelegationTokenSecretManager delegationTokenSecretManager = new RMDelegationTokenSecretManager(1, 1, 1, 1, context);

  context = new MockRMContext();
  context.setStateStore(TestObjectFactory.getStateStore(conf, "tmp/myriad-operations-test"));
  context.setAmLivelinessMonitor(amLivelinessMonitor);
  context.setAmFinishingMonitor(amFinishingMonitor);
  context.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
  context.setRMDelegationTokenSecretManager(delegationTokenSecretManager);
  return context;
}
 
Example 2
Source Project: hadoop   Source File: ParentQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example 3
void printString(CSQueue nq, String indent) {
  if (nq instanceof ParentQueue) {
    System.out.println(indent + nq.getQueueName()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (CSQueue q : ((ParentQueue)nq).getChildQueues()) {
      printString(q, indent + "  ");
    }
  } else {
    System.out.println(indent + nq.getQueueName()
        + " pen:" + ((LeafQueue) nq).getTotalResourcePending()
        + " cur:" + nq.getAbsoluteUsedCapacity()
        + " guar:" + nq.getAbsoluteCapacity()
        );
    for (FiCaSchedulerApp a : ((LeafQueue)nq).getApplications()) {
      System.out.println(indent + "  " + a.getApplicationId());
    }
  }
}
 
Example 4
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Resource getPreemptedResource(){
 Resource preemptedResource = Resource.newInstance(0, 0);
 //try to resume containers which are suspended in fifo order
 for(ApplicationAttemptId appId: this.suspendedApps){
   FiCaSchedulerApp app = this.applicationAttemptMap.get(appId);
   	synchronized(app){
   	for(ContainerId cntId : app.getContainersSuspended()){
   		RMContainer rmContainer  =  app.getLiveContainersMap().get(cntId);
   		Resources.addTo(preemptedResource, rmContainer.getPreemptedResource());
   	   }
       }
   }
 
 return preemptedResource;
}
 
Example 5
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void addApplicationAttempt(FiCaSchedulerApp application,
    User user) {
  // Accept 
  user.submitApplication();
  pendingApplications.add(application);
  applicationAttemptMap.put(application.getApplicationAttemptId(), application);

  // Activate applications
  activateApplications();
  
  LOG.info("Application added -" +
      " appId: " + application.getApplicationId() +
      " user: " + user + "," + " leaf-queue: " + getQueueName() +
      " #user-pending-applications: " + user.getPendingApplications() +
      " #user-active-applications: " + user.getActiveApplications() +
      " #queue-pending-applications: " + getNumPendingApplications() +
      " #queue-active-applications: " + getNumActiveApplications()
      );
}
 
Example 6
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initScheduler(Configuration conf) {
  validateConf(conf);
  //Use ConcurrentSkipListMap because applications need to be ordered
  this.applications =
      new ConcurrentSkipListMap<ApplicationId, SchedulerApplication<FiCaSchedulerApp>>();
  this.minimumAllocation =
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  initMaximumResourceCapability(
      Resources.createResource(conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
        conf.getInt(
          YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
          YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES)));
  this.usePortForNodeName = conf.getBoolean(
      YarnConfiguration.RM_SCHEDULER_INCLUDE_PORT_IN_NODE_NAME,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_USE_PORT_FOR_NODE_NAME);
  this.metrics = QueueMetrics.forQueue(DEFAULT_QUEUE_NAME, null, false,
      conf);
  this.activeUsersManager = new ActiveUsersManager(metrics);
}
 
Example 7
Source Project: hadoop   Source File: TestLeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testUserQueueAcl() throws Exception {

  // Manipulate queue 'a'
  LeafQueue d = stubLeafQueue((LeafQueue) queues.get(D));

  // Users
  final String user_d = "user_d";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 = TestUtils
      .getMockApplicationAttemptId(0, 1);
  FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_d, d, null,
      spyRMContext);
  d.submitApplicationAttempt(app_0, user_d);

  // Attempt the same application again
  final ApplicationAttemptId appAttemptId_1 = TestUtils
      .getMockApplicationAttemptId(0, 2);
  FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_d, d, null,
      spyRMContext);
  d.submitApplicationAttempt(app_1, user_d); // same user
}
 
Example 8
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 6 votes vote down vote up
synchronized void releaseResource(Resource clusterResource, 
    FiCaSchedulerApp application, Resource resource, Set<String> nodeLabels,boolean isSuspend) {
  super.releaseResource(clusterResource, resource, nodeLabels,isSuspend);
  
  // Update user metrics
  String userName = application.getUser();
  User user = getUser(userName);
  user.releaseContainer(resource, nodeLabels);
  if(!isSuspend){
    metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
  }
  
  LOG.info(getQueueName() + 
          " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
          " headroom = " + application.getHeadroom() +
          " user-resources=" + user.getUsed()+"allocate resource:"+resource+
          " absUsed= "+getAbsoluteUsedCapacity()
          );
}
 
Example 9
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private int assignNodeLocalContainers(FiCaSchedulerNode node, 
    FiCaSchedulerApp application, Priority priority) {
  int assignedContainers = 0;
  ResourceRequest request = 
    application.getResourceRequest(priority, node.getNodeName());
  if (request != null) {
    // Don't allocate on this node if we don't need containers on this rack
    ResourceRequest rackRequest =
        application.getResourceRequest(priority, 
            node.getRMNode().getRackName());
    if (rackRequest == null || rackRequest.getNumContainers() <= 0) {
      return 0;
    }
    
    int assignableContainers = 
      Math.min(
          getMaxAllocatableContainers(application, priority, node, 
              NodeType.NODE_LOCAL), 
              request.getNumContainers());
    assignedContainers = 
      assignContainer(node, application, priority, 
          assignableContainers, request, NodeType.NODE_LOCAL);
  }
  return assignedContainers;
}
 
Example 10
Source Project: big-c   Source File: ParentQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void detachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.releaseResource(clusterResource,
        rmContainer.getContainer().getResource(),
        node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.detachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example 11
Source Project: big-c   Source File: ParentQueue.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void attachContainer(Resource clusterResource,
    FiCaSchedulerApp application, RMContainer rmContainer) {
  if (application != null) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
    LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
        + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
        + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
        + clusterResource);
    // Inform the parent
    if (parent != null) {
      parent.attachContainer(clusterResource, application, rmContainer);
    }
  }
}
 
Example 12
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void initScheduler(Configuration configuration) throws
    IOException {
  this.conf = loadCapacitySchedulerConfiguration(configuration);
  validateConf(this.conf);
  this.minimumAllocation = this.conf.getMinimumAllocation();
  initMaximumResourceCapability(this.conf.getMaximumAllocation());
  this.calculator = this.conf.getResourceCalculator();
  this.usePortForNodeName = this.conf.getUsePortForNodeName();
  
  this.nodeContainerUpdateMap = new HashMap<NodeId, ConcurrentLinkedQueue<NodeContainerUpdate>>();
 
  this.applications =
      new ConcurrentHashMap<ApplicationId,
          SchedulerApplication<FiCaSchedulerApp>>();
  this.labelManager = rmContext.getNodeLabelManager();
  authorizer = YarnAuthorizationProvider.getInstance(yarnConf);
  initializeQueues(this.conf);

  scheduleAsynchronously = this.conf.getScheduleAynschronously();
  asyncScheduleInterval =
      this.conf.getLong(ASYNC_SCHEDULER_INTERVAL,
          DEFAULT_ASYNC_SCHEDULER_INTERVAL);
  if (scheduleAsynchronously) {
    asyncSchedulerThread = new AsyncScheduleThread(this);
  }

  LOG.info("Initialized CapacityScheduler with " +
      "calculator=" + getResourceCalculator().getClass() + ", " +
      "minimumAllocation=<" + getMinimumResourceCapability() + ">, " +
      "maximumAllocation=<" + getMaximumResourceCapability() + ">, " +
      "asynchronousScheduling=" + scheduleAsynchronously + ", " +
      "asyncScheduleInterval=" + asyncScheduleInterval + "ms");
}
 
Example 13
Source Project: hadoop   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@VisibleForTesting
public synchronized void
    addApplicationAttempt(ApplicationAttemptId appAttemptId,
        boolean transferStateFromPreviousAttempt,
        boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(appAttemptId.getApplicationId());
  String user = application.getUser();
  // TODO: Fix store
  FiCaSchedulerApp schedulerApp =
      new FiCaSchedulerApp(appAttemptId, user, DEFAULT_QUEUE,
        activeUsersManager, this.rmContext);

  if (transferStateFromPreviousAttempt) {
    schedulerApp.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(schedulerApp);

  metrics.submitAppAttempt(user);
  LOG.info("Added Application Attempt " + appAttemptId
      + " to scheduler from user " + application.getUser());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(appAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(appAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 14
Source Project: hadoop   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private int getMaxAllocatableContainers(FiCaSchedulerApp application,
    Priority priority, FiCaSchedulerNode node, NodeType type) {
  int maxContainers = 0;
  
  ResourceRequest offSwitchRequest = 
    application.getResourceRequest(priority, ResourceRequest.ANY);
  if (offSwitchRequest != null) {
    maxContainers = offSwitchRequest.getNumContainers();
  }

  if (type == NodeType.OFF_SWITCH) {
    return maxContainers;
  }

  if (type == NodeType.RACK_LOCAL) {
    ResourceRequest rackLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getRackName());
    if (rackLocalRequest == null) {
      return maxContainers;
    }

    maxContainers = Math.min(maxContainers, rackLocalRequest.getNumContainers());
  }

  if (type == NodeType.NODE_LOCAL) {
    ResourceRequest nodeLocalRequest = 
      application.getResourceRequest(priority, node.getRMNode().getNodeAddress());
    if (nodeLocalRequest != null) {
      maxContainers = Math.min(maxContainers, nodeLocalRequest.getNumContainers());
    }
  }

  return maxContainers;
}
 
Example 15
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized List<ApplicationAttemptId>
    getAppsInQueue(String queueName) {
  if (queueName.equals(DEFAULT_QUEUE.getQueueName())) {
    List<ApplicationAttemptId> attempts =
        new ArrayList<ApplicationAttemptId>(applications.size());
    for (SchedulerApplication<FiCaSchedulerApp> app : applications.values()) {
      attempts.add(app.getCurrentAppAttempt().getApplicationAttemptId());
    }
    return attempts;
  } else {
    return null;
  }
}
 
Example 16
Source Project: hadoop   Source File: TestLeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private List<FiCaSchedulerApp> createListOfApps(int noOfApps, String user,
    LeafQueue defaultQueue) {
  List<FiCaSchedulerApp> appsLists = new ArrayList<FiCaSchedulerApp>();
  for (int i = 0; i < noOfApps; i++) {
    ApplicationAttemptId appAttemptId_0 =
        TestUtils.getMockApplicationAttemptId(i, 0);
    FiCaSchedulerApp app_0 =
        new FiCaSchedulerApp(appAttemptId_0, user, defaultQueue,
            mock(ActiveUsersManager.class), spyRMContext);
    appsLists.add(app_0);
  }
  return appsLists;
}
 
Example 17
Source Project: big-c   Source File: TestApplicationLimits.java    License: Apache License 2.0 5 votes vote down vote up
private FiCaSchedulerApp getMockApplication(int appId, String user,
  Resource amResource) {
  FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
  ApplicationAttemptId applicationAttemptId =
      TestUtils.getMockApplicationAttemptId(appId, 0);
  doReturn(applicationAttemptId.getApplicationId()).
      when(application).getApplicationId();
  doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
  doReturn(user).when(application).getUser();
  doReturn(amResource).when(application).getAMResource();
  return application;
}
 
Example 18
FiCaSchedulerApp mockApp(int qid, int id, int used, int pending, int reserved,
    int gran) {
  FiCaSchedulerApp app = mock(FiCaSchedulerApp.class);

  ApplicationId appId = ApplicationId.newInstance(TS, id);
  ApplicationAttemptId appAttId = ApplicationAttemptId.newInstance(appId, 0);
  when(app.getApplicationId()).thenReturn(appId);
  when(app.getApplicationAttemptId()).thenReturn(appAttId);

  int cAlloc = 0;
  Resource unit = Resource.newInstance(gran, 0);
  List<RMContainer> cReserved = new ArrayList<RMContainer>();
  for (int i = 0; i < reserved; i += gran) {
    cReserved.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
        .getValue()));
    ++cAlloc;
  }
  when(app.getReservedContainers()).thenReturn(cReserved);

  List<RMContainer> cLive = new ArrayList<RMContainer>();
  for (int i = 0; i < used; i += gran) {
    if(setAMContainer && i == 0){
      cLive.add(mockContainer(appAttId, cAlloc, unit, priority.AMCONTAINER
          .getValue()));
    }else if(setLabeledContainer && i ==1){
      cLive.add(mockContainer(appAttId, cAlloc, unit,
          priority.LABELEDCONTAINER.getValue()));
      ++used;
    }
    else{
      cLive.add(mockContainer(appAttId, cAlloc, unit, priority.CONTAINER
          .getValue()));
    }
    ++cAlloc;
  }
  when(app.getLiveContainers()).thenReturn(cLive);
  return app;
}
 
Example 19
Source Project: big-c   Source File: TestLeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private List<FiCaSchedulerApp> createListOfApps(int noOfApps, String user,
    LeafQueue defaultQueue) {
  List<FiCaSchedulerApp> appsLists = new ArrayList<FiCaSchedulerApp>();
  for (int i = 0; i < noOfApps; i++) {
    ApplicationAttemptId appAttemptId_0 =
        TestUtils.getMockApplicationAttemptId(i, 0);
    FiCaSchedulerApp app_0 =
        new FiCaSchedulerApp(appAttemptId_0, user, defaultQueue,
            mock(ActiveUsersManager.class), spyRMContext);
    appsLists.add(app_0);
  }
  return appsLists;
}
 
Example 20
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void addApplicationAttempt(
    ApplicationAttemptId applicationAttemptId,
    boolean transferStateFromPreviousAttempt,
    boolean isAttemptRecovering) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationAttemptId.getApplicationId());
  CSQueue queue = (CSQueue) application.getQueue();

  FiCaSchedulerApp attempt =
      new FiCaSchedulerApp(applicationAttemptId, application.getUser(),
        queue, queue.getActiveUsersManager(), rmContext);
  if (transferStateFromPreviousAttempt) {
    attempt.transferStateFromPreviousAttempt(application
      .getCurrentAppAttempt());
  }
  application.setCurrentAppAttempt(attempt);

  queue.submitApplicationAttempt(attempt, application.getUser());
  LOG.info("Added Application Attempt " + applicationAttemptId
      + " to scheduler from user " + application.getUser() + " in queue "
      + queue.getQueueName());
  if (isAttemptRecovering) {
    if (LOG.isDebugEnabled()) {
      LOG.debug(applicationAttemptId
          + " is recovering. Skipping notifying ATTEMPT_ADDED");
    }
  } else {
    rmContext.getDispatcher().getEventHandler().handle(
      new RMAppAttemptEvent(applicationAttemptId,
          RMAppAttemptEventType.ATTEMPT_ADDED));
  }
}
 
Example 21
Source Project: big-c   Source File: FifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationId);
  if (application == null){
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }

  // Inform the activeUsersManager
  activeUsersManager.deactivateApplication(application.getUser(),
    applicationId);
  application.stop(finalState);
  applications.remove(applicationId);
}
 
Example 22
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void updateClusterResource(Resource clusterResource,
    ResourceLimits currentResourceLimits) {
  updateCurrentResourceLimits(currentResourceLimits, clusterResource);
  lastClusterResource = clusterResource;
  updateAbsoluteCapacityResource(clusterResource);
  
  // Update headroom info based on new cluster resource value
  // absoluteMaxCapacity now,  will be replaced with absoluteMaxAvailCapacity
  // during allocation
  setQueueResourceLimitsInfo(clusterResource);
  
  // Update metrics
  CSQueueUtils.updateQueueStatistics(
      resourceCalculator, this, getParent(), clusterResource, 
      minimumAllocation);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();

  // Update application properties
  for (FiCaSchedulerApp application : activeApplications) {
    synchronized (application) {
      computeUserLimitAndSetHeadroom(application, clusterResource, 
          Resources.none(), null);
    }
  }
}
 
Example 23
Source Project: hadoop   Source File: CapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Lock(CapacityScheduler.class)
@Override
protected synchronized void completedContainer(RMContainer rmContainer,
    ContainerStatus containerStatus, RMContainerEventType event) {
  if (rmContainer == null) {
    LOG.info("Null container completed...");
    return;
  }
  
  Container container = rmContainer.getContainer();
  
  // Get the application for the finished container
  FiCaSchedulerApp application =
      getCurrentAttemptForContainer(container.getId());
  ApplicationId appId =
      container.getId().getApplicationAttemptId().getApplicationId();
  if (application == null) {
    LOG.info("Container " + container + " of" + " unknown application "
        + appId + " completed with event " + event);
    return;
  }
  
  // Get the node on which the container was allocated
  FiCaSchedulerNode node = getNode(container.getNodeId());
  
  // Inform the queue
  LeafQueue queue = (LeafQueue)application.getQueue();
  queue.completedContainer(clusterResource, application, node, 
      rmContainer, containerStatus, event, null, true);

  LOG.info("Application attempt " + application.getApplicationAttemptId()
      + " released container " + container.getId() + " on node: " + node
      + " with event: " + event);
}
 
Example 24
Source Project: big-c   Source File: CSAssignment.java    License: Apache License 2.0 5 votes vote down vote up
public CSAssignment(FiCaSchedulerApp application, RMContainer excessReservation) {
  this.resource = excessReservation.getContainer().getResource();
  this.type = NodeType.NODE_LOCAL;
  this.application = application;
  this.excessReservation = excessReservation;
  this.skipped = false;
}
 
Example 25
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private void reserve(FiCaSchedulerApp application, Priority priority, 
    FiCaSchedulerNode node, RMContainer rmContainer, Container container) {
  // Update reserved metrics if this is the first reservation
  if (rmContainer == null) {
    getMetrics().reserveResource(
        application.getUser(), container.getResource());
  }

  // Inform the application 
  rmContainer = application.reserve(node, priority, rmContainer, container);
  
  // Update the node
  node.reserveResource(application, priority, rmContainer);
}
 
Example 26
Source Project: big-c   Source File: TestReservations.java    License: Apache License 2.0 5 votes vote down vote up
static LeafQueue stubLeafQueue(LeafQueue queue) {

    // Mock some methods for ease in these unit tests

    // 1. LeafQueue.createContainer to return dummy containers
    doAnswer(new Answer<Container>() {
      @Override
      public Container answer(InvocationOnMock invocation) throws Throwable {
        final FiCaSchedulerApp application = (FiCaSchedulerApp) (invocation
            .getArguments()[0]);
        final ContainerId containerId = TestUtils
            .getMockContainerId(application);

        Container container = TestUtils.getMockContainer(containerId,
            ((FiCaSchedulerNode) (invocation.getArguments()[1])).getNodeID(),
            (Resource) (invocation.getArguments()[2]),
            ((Priority) invocation.getArguments()[3]));
        return container;
      }
    }).when(queue).createContainer(any(FiCaSchedulerApp.class),
        any(FiCaSchedulerNode.class), any(Resource.class), any(Priority.class));

    // 2. Stub out LeafQueue.parent.completedContainer
    CSQueue parent = queue.getParent();
    doNothing().when(parent).completedContainer(any(Resource.class),
        any(FiCaSchedulerApp.class), any(FiCaSchedulerNode.class),
        any(RMContainer.class), any(ContainerStatus.class),
        any(RMContainerEventType.class), any(CSQueue.class), anyBoolean());

    return queue;
  }
 
Example 27
Source Project: big-c   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
private Container getContainer(RMContainer rmContainer, 
    FiCaSchedulerApp application, FiCaSchedulerNode node, 
    Resource capability, Priority priority) {
 //是否是reserve 的情况
  return (rmContainer != null) ? rmContainer.getContainer() :
    createContainer(application, node, capability, priority);
}
 
Example 28
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) {
  // Careful! Locking order is important!
  synchronized (this) {
    removeApplicationAttempt(application, getUser(application.getUser()));
  }
  getParent().finishApplicationAttempt(application, queue);
}
 
Example 29
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
public synchronized void removeApplicationAttempt(
    FiCaSchedulerApp application, User user) {
  String partitionName = application.getAppAMNodePartitionName();
  boolean wasActive = activeApplications.remove(application);
  if (!wasActive) {
    pendingApplications.remove(application);
  } else {
    queueUsage.decAMUsed(partitionName,
        application.getAMResource(partitionName));
    user.getResourceUsage().decAMUsed(partitionName,
        application.getAMResource(partitionName));
  }
  applicationAttemptMap.remove(application.getApplicationAttemptId());

  user.finishApplication(wasActive);
  if (user.getTotalApplications() == 0) {
    users.remove(application.getUser());
  }

  // Check if we can activate more applications
  activateApplications();

  LOG.info("Application removed -" +
      " appId: " + application.getApplicationId() + 
      " user: " + application.getUser() + 
      " queue: " + getQueueName() +
      " #user-pending-applications: " + user.getPendingApplications() +
      " #user-active-applications: " + user.getActiveApplications() +
      " #queue-pending-applications: " + getNumPendingApplications() +
      " #queue-active-applications: " + getNumActiveApplications()
      );
}
 
Example 30
Source Project: hadoop   Source File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Lock({LeafQueue.class, FiCaSchedulerApp.class})
Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
    Resource clusterResource, Resource required, Set<String> requestedLabels) {
  String user = application.getUser();
  User queueUser = getUser(user);

  // Compute user limit respect requested labels,
  // TODO, need consider headroom respect labels also
  Resource userLimit =
      computeUserLimit(application, clusterResource, required,
          queueUser, requestedLabels);

  setQueueResourceLimitsInfo(clusterResource);
  
  Resource headroom =
      getHeadroom(queueUser, cachedResourceLimitsForHeadroom.getLimit(),
          clusterResource, userLimit);
  
  if (LOG.isDebugEnabled()) {
    LOG.debug("Headroom calculation for user " + user + ": " + 
        " userLimit=" + userLimit + 
        " queueMaxAvailRes=" + cachedResourceLimitsForHeadroom.getLimit() +
        " consumed=" + queueUser.getUsed() + 
        " headroom=" + headroom);
  }
  
  CapacityHeadroomProvider headroomProvider = new CapacityHeadroomProvider(
    queueUser, this, application, required, queueResourceLimitsInfo);
  
  application.setHeadroomProvider(headroomProvider);

  metrics.setAvailableResourcesToUser(user, headroom);
  
  return userLimit;
}