org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels());
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}
Example #2
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public static void waitForNumContainersToRecover(int num, MockRM rm,
    ApplicationAttemptId attemptId) throws Exception {
  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm.getResourceScheduler();
  SchedulerApplicationAttempt attempt =
      scheduler.getApplicationAttempt(attemptId);
  while (attempt == null) {
    System.out.println("Wait for scheduler attempt " + attemptId
        + " to be created");
    Thread.sleep(200);
    attempt = scheduler.getApplicationAttempt(attemptId);
  }
  while (attempt.getLiveContainers().size() < num) {
    System.out.println("Wait for " + num
        + " containers to recover. currently: "
        + attempt.getLiveContainers().size());
    Thread.sleep(200);
  }
}
Example #3
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
Example #4
Source File: FiCaSchedulerNode.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {

  // adding NP checks as this can now be called for preemption
  if (getReservedContainer() != null
      && getReservedContainer().getContainer() != null
      && getReservedContainer().getContainer().getId() != null
      && getReservedContainer().getContainer().getId()
        .getApplicationAttemptId() != null) {

    // Cannot unreserve for wrong application...
    ApplicationAttemptId reservedApplication =
        getReservedContainer().getContainer().getId()
          .getApplicationAttemptId();
    if (!reservedApplication.equals(
        application.getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to unreserve " +
          " for application " + application.getApplicationAttemptId() +
          " when currently reserved " +
          " for application " + reservedApplication.getApplicationId() +
          " on node " + this);
    }
  }
  setReservedContainer(null);
}
Example #5
Source File: TestCapacityScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
Example #6
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 6 votes vote down vote up 
public static void waitForNumContainersToRecover(int num, MockRM rm,
    ApplicationAttemptId attemptId) throws Exception {
  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm.getResourceScheduler();
  SchedulerApplicationAttempt attempt =
      scheduler.getApplicationAttempt(attemptId);
  while (attempt == null) {
    System.out.println("Wait for scheduler attempt " + attemptId
        + " to be created");
    Thread.sleep(200);
    attempt = scheduler.getApplicationAttempt(attemptId);
  }
  while (attempt.getLiveContainers().size() < num) {
    System.out.println("Wait for " + num
        + " containers to recover. currently: "
        + attempt.getLiveContainers().size());
    Thread.sleep(200);
  }
}
Example #7
Source File: ParentQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    super.allocateResource(clusterResource, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
  }
  if (parent != null) {
    parent.recoverContainer(clusterResource, attempt, rmContainer);
  }
}
Example #8
Source File: LeafQueue.java    From big-c with Apache License 2.0 6 votes vote down vote up 
synchronized void allocateResource(Resource clusterResource,
    SchedulerApplicationAttempt application, Resource resource,
    Set<String> nodeLabels,boolean isResume) {
  super.allocateResource(clusterResource, resource, nodeLabels,isResume);
  
  // Update user metrics
  String userName = application.getUser();
  User user = getUser(userName);
  user.assignContainer(resource, nodeLabels);
  // Note this is a bit unconventional since it gets the object and modifies
  // it here, rather then using set routine
  if(!isResume){
  Resources.subtractFrom(application.getHeadroom(), resource); // headroom
  metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
  }
  //if (LOG.isDebugEnabled()) {
    LOG.info(getQueueName() + 
        " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
        " headroom = " + application.getHeadroom() +
        " user-resources=" + user.getUsed()+"allocate resource:"+resource+
        " absUsed= "+getAbsoluteUsedCapacity()
        );
//  }
}
Example #9
Source File: FSSchedulerNode.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {
  // Cannot unreserve for wrong application...
  ApplicationAttemptId reservedApplication = 
      getReservedContainer().getContainer().getId().getApplicationAttemptId(); 
  if (!reservedApplication.equals(
      application.getApplicationAttemptId())) {
    throw new IllegalStateException("Trying to unreserve " +  
        " for application " + application.getApplicationId() + 
        " when currently reserved " + 
        " for application " + reservedApplication.getApplicationId() + 
        " on node " + this);
  }
  
  setReservedContainer(null);
  this.reservedAppSchedulable = null;
}
Example #10
Source File: LeafQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
synchronized void allocateResource(Resource clusterResource,
    SchedulerApplicationAttempt application, Resource resource,
    Set<String> nodeLabels) {
  super.allocateResource(clusterResource, resource, nodeLabels);
  
  // Update user metrics
  String userName = application.getUser();
  User user = getUser(userName);
  user.assignContainer(resource, nodeLabels);
  // Note this is a bit unconventional since it gets the object and modifies
  // it here, rather then using set routine
  Resources.subtractFrom(application.getHeadroom(), resource); // headroom
  metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
  
  if (LOG.isDebugEnabled()) {
    LOG.info(getQueueName() + 
        " user=" + userName + 
        " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
        " headroom = " + application.getHeadroom() +
        " user-resources=" + user.getUsed()
        );
  }
}
Example #11
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private void checkCSLeafQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers) {
  LeafQueue leafQueue = (LeafQueue) app.getQueue();
  // assert queue used resources.
  assertEquals(usedResource, leafQueue.getUsedResources());
  assertEquals(numContainers, leafQueue.getNumContainers());

  ResourceCalculator calc =
      ((CapacityScheduler) rm.getResourceScheduler()).getResourceCalculator();
  float usedCapacity =
      Resources.divide(calc, clusterResource, usedResource, queueResource);
  // assert queue used capacity
  assertEquals(usedCapacity, leafQueue.getUsedCapacity(), 1e-8);
  float absoluteUsedCapacity =
      Resources.divide(calc, clusterResource, usedResource, clusterResource);
  // assert queue absolute capacity
  assertEquals(absoluteUsedCapacity, leafQueue.getAbsoluteUsedCapacity(),
    1e-8);
  // assert user consumed resources.
  assertEquals(usedResource, leafQueue.getUser(app.getUser())
    .getUsed());
}
Example #12
Source File: FiCaSchedulerNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {

  // adding NP checks as this can now be called for preemption
  if (getReservedContainer() != null
      && getReservedContainer().getContainer() != null
      && getReservedContainer().getContainer().getId() != null
      && getReservedContainer().getContainer().getId()
        .getApplicationAttemptId() != null) {

    // Cannot unreserve for wrong application...
    ApplicationAttemptId reservedApplication =
        getReservedContainer().getContainer().getId()
          .getApplicationAttemptId();
    if (!reservedApplication.equals(
        application.getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to unreserve " +
          " for application " + application.getApplicationAttemptId() +
          " when currently reserved " +
          " for application " + reservedApplication.getApplicationId() +
          " on node " + this);
    }
  }
  setReservedContainer(null);
}
Example #13
Source File: FSSchedulerNode.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Override
public synchronized void unreserveResource(
    SchedulerApplicationAttempt application) {
  // Cannot unreserve for wrong application...
  ApplicationAttemptId reservedApplication = 
      getReservedContainer().getContainer().getId().getApplicationAttemptId(); 
  if (!reservedApplication.equals(
      application.getApplicationAttemptId())) {
    throw new IllegalStateException("Trying to unreserve " +  
        " for application " + application.getApplicationId() + 
        " when currently reserved " + 
        " for application " + reservedApplication.getApplicationId() + 
        " on node " + this);
  }
  
  setReservedContainer(null);
  this.reservedAppSchedulable = null;
}
Example #14
Source File: CompositeInterceptor.java    From incubator-myriad with Apache License 2.0 6 votes vote down vote up 
@Override
public void beforeReleaseContainers(List<ContainerId> containers, SchedulerApplicationAttempt attempt){
  if (containers != null && attempt != null) {
    for (YarnSchedulerInterceptor interceptor : interceptors.values()) {
      List<ContainerId> filteredContainers = new ArrayList<>();
      for (ContainerId containerId: containers) {
        NodeId nodeId = attempt.getRMContainer(containerId).getContainer().getNodeId();
        if ((nodeId != null && interceptor.getCallBackFilter().allowCallBacksForNode(nodeId))) {
          filteredContainers.add(containerId);
        }
      }
      if (!filteredContainers.isEmpty()) {
        interceptor.beforeReleaseContainers(filteredContainers, attempt);
      }
    }
  }
}
Example #15
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromCapacityScheduler() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  setupQueueConfiguration(conf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> cs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();

  SchedulerApplication<SchedulerApplicationAttempt> app =
      TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
        cs.getSchedulerApplications(), cs, "a1");
  Assert.assertEquals("a1", app.getQueue().getQueueName());
}
Example #16
Source File: FifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  increaseUsedResources(rmContainer);
  updateAppHeadRoom(schedulerAttempt);
  updateAvailableResourcesMetrics();
}
Example #17
Source File: FSSchedulerNode.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public synchronized void reserveResource(
    SchedulerApplicationAttempt application, Priority priority,
    RMContainer container) {
  // Check if it's already reserved
  RMContainer reservedContainer = getReservedContainer();
  if (reservedContainer != null) {
    // Sanity check
    if (!container.getContainer().getNodeId().equals(getNodeID())) {
      throw new IllegalStateException("Trying to reserve" +
          " container " + container +
          " on node " + container.getReservedNode() + 
          " when currently" + " reserved resource " + reservedContainer +
          " on node " + reservedContainer.getReservedNode());
    }
    
    // Cannot reserve more than one application on a given node!
    if (!reservedContainer.getContainer().getId().getApplicationAttemptId()
        .equals(container.getContainer().getId().getApplicationAttemptId())) {
      throw new IllegalStateException("Trying to reserve" +
          " container " + container + 
          " for application " + application.getApplicationId() + 
          " when currently" +
          " reserved container " + reservedContainer +
          " on node " + this);
    }

    LOG.info("Updated reserved container " + 
        container.getContainer().getId() + " on node " + 
        this + " for application " + application);
  } else {
    LOG.info("Reserved container " + container.getContainer().getId() + 
        " on node " + this + " for application " + application);
  }
  setReservedContainer(container);
  this.reservedAppSchedulable = (FSAppAttempt) application;
}
Example #18
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
Example #19
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
Example #20
Source File: FiCaSchedulerApp.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public synchronized void transferStateFromPreviousAttempt(
    SchedulerApplicationAttempt appAttempt) {
  super.transferStateFromPreviousAttempt(appAttempt);
  this.headroomProvider = 
    ((FiCaSchedulerApp) appAttempt).getHeadroomProvider();
}
Example #21
Source File: LeafQueue.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt attempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  // Careful! Locking order is important! 
  synchronized (this) {
    FiCaSchedulerNode node =
        scheduler.getNode(rmContainer.getContainer().getNodeId());
    allocateResource(clusterResource, attempt, rmContainer.getContainer()
        .getResource(), node.getLabels(),false);
  }
  getParent().recoverContainer(clusterResource, attempt, rmContainer);
}
Example #22
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFiFoScheduler() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
    ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  @SuppressWarnings("unchecked")
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> fs =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) rm
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    fs.getSchedulerApplications(), fs, "queue");
}
Example #23
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testAddAndRemoveAppFromFairScheduler() throws Exception {
  AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode> scheduler =
      (AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>) resourceManager
        .getResourceScheduler();
  TestSchedulerUtils.verifyAppAddedAndRemovedFromScheduler(
    scheduler.getSchedulerApplications(), scheduler, "default");
}
Example #24
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}
Example #25
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStart() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).start();
  super.serviceStart();
}
Example #26
Source File: FifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Override
public void recoverContainer(Resource clusterResource,
    SchedulerApplicationAttempt schedulerAttempt, RMContainer rmContainer) {
  if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
    return;
  }
  increaseUsedResources(rmContainer);
  updateAppHeadRoom(schedulerAttempt);
  updateAvailableResourcesMetrics();
}
Example #27
Source File: ResourceSchedulerWrapper.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
Example #28
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceStop() throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).stop();
  super.serviceStop();
}
Example #29
Source File: ResourceSchedulerWrapper.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@SuppressWarnings("unchecked")
@Override
public void serviceInit(Configuration conf) throws Exception {
  ((AbstractYarnScheduler<SchedulerApplicationAttempt, SchedulerNode>)
      scheduler).init(conf);
  super.serviceInit(conf);
}
Example #30
Source File: TestWorkPreservingRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void checkCSQueue(MockRM rm,
    SchedulerApplication<SchedulerApplicationAttempt> app,
    Resource clusterResource, Resource queueResource, Resource usedResource,
    int numContainers)
    throws Exception {
  checkCSLeafQueue(rm, app, clusterResource, queueResource, usedResource,
      numContainers);

  LeafQueue queue = (LeafQueue) app.getQueue();
  Resource availableResources =
      Resources.subtract(queueResource, usedResource);
  // ************ check app headroom ****************
  SchedulerApplicationAttempt schedulerAttempt = app.getCurrentAppAttempt();
  assertEquals(availableResources, schedulerAttempt.getHeadroom());

  // ************* check Queue metrics ************
  QueueMetrics queueMetrics = queue.getMetrics();
  assertMetrics(queueMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());

  // ************ check user metrics ***********
  QueueMetrics userMetrics =
      queueMetrics.getUserMetrics(app.getUser());
  assertMetrics(userMetrics, 1, 0, 1, 0, 2, availableResources.getMemory(),
      availableResources.getVirtualCores(), usedResource.getMemory(),
      usedResource.getVirtualCores());
}