Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils#normalizeRequests()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils#normalizeRequests() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: FifoScheduler.java From hadoop with Apache License 2.0

4 votes

@Override
public Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
  FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
  if (application == null) {
    LOG.error("Calling allocate on removed " +
        "or non existant application " + applicationAttemptId);
    return EMPTY_ALLOCATION;
  }

  // Sanity check
  SchedulerUtils.normalizeRequests(ask, resourceCalculator, 
      clusterResource, minimumAllocation, getMaximumResourceCapability());

  // Release containers
  releaseContainers(release, application);

  synchronized (application) {

    // make sure we aren't stopping/removing the application
    // when the allocate comes in
    if (application.isStopped()) {
      LOG.info("Calling allocate on a stopped " +
          "application " + applicationAttemptId);
      return EMPTY_ALLOCATION;
    }

    if (!ask.isEmpty()) {
      LOG.debug("allocate: pre-update" +
          " applicationId=" + applicationAttemptId + 
          " application=" + application);
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      LOG.debug("allocate: post-update" +
          " applicationId=" + applicationAttemptId + 
          " application=" + application);
      application.showRequests();

      LOG.debug("allocate:" +
          " applicationId=" + applicationAttemptId + 
          " #ask=" + ask.size());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    ContainersAndNMTokensAllocation allocation =
        application.pullNewlyAllocatedContainersAndNMTokens();
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(allocation.getContainerList(), headroom, null,
        null, null, allocation.getNMTokenList());
  }
}

Example 2

Source File: FairScheduler.java From hadoop with Apache License 2.0

4 votes

@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
    List<ResourceRequest> ask, List<ContainerId> release,
    List<String> blacklistAdditions, List<String> blacklistRemovals) {

  // Make sure this application exists
  FSAppAttempt application = getSchedulerApp(appAttemptId);
  if (application == null) {
    LOG.info("Calling allocate on removed " +
        "or non existant application " + appAttemptId);
    return EMPTY_ALLOCATION;
  }

  // Sanity check
  SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR,
      clusterResource, minimumAllocation, getMaximumResourceCapability(),
      incrAllocation);

  // Set amResource for this app
  if (!application.getUnmanagedAM() && ask.size() == 1
      && application.getLiveContainers().isEmpty()) {
    application.setAMResource(ask.get(0).getCapability());
  }

  // Release containers
  releaseContainers(release, application);

  synchronized (application) {
    if (!ask.isEmpty()) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: pre-update" +
            " applicationAttemptId=" + appAttemptId +
            " application=" + application.getApplicationId());
      }
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      application.showRequests();
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("allocate: post-update" +
          " applicationAttemptId=" + appAttemptId +
          " #ask=" + ask.size() +
          " reservation= " + application.getCurrentReservation());

      LOG.debug("Preempting " + application.getPreemptionContainers().size()
          + " container(s)");
    }
    
    Set<ContainerId> preemptionContainerIds = new HashSet<ContainerId>();
    for (RMContainer container : application.getPreemptionContainers()) {
      preemptionContainerIds.add(container.getContainerId());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    ContainersAndNMTokensAllocation allocation =
        application.pullNewlyAllocatedContainersAndNMTokens();
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(allocation.getContainerList(), headroom,
        preemptionContainerIds, null, null, allocation.getNMTokenList());
  }
}

Example 3

Source File: CapacityScheduler.java From hadoop with Apache License 2.0

4 votes

@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
    List<ResourceRequest> ask, List<ContainerId> release, 
    List<String> blacklistAdditions, List<String> blacklistRemovals) {

  FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
  if (application == null) {
    LOG.info("Calling allocate on removed " +
        "or non existant application " + applicationAttemptId);
    return EMPTY_ALLOCATION;
  }
  
  // Sanity check
  SchedulerUtils.normalizeRequests(
      ask, getResourceCalculator(), getClusterResource(),
      getMinimumResourceCapability(), getMaximumResourceCapability());

  // Release containers
  releaseContainers(release, application);

  synchronized (application) {

    // make sure we aren't stopping/removing the application
    // when the allocate comes in
    if (application.isStopped()) {
      LOG.info("Calling allocate on a stopped " +
          "application " + applicationAttemptId);
      return EMPTY_ALLOCATION;
    }

    if (!ask.isEmpty()) {

      if(LOG.isDebugEnabled()) {
        LOG.debug("allocate: pre-update" +
          " applicationAttemptId=" + applicationAttemptId + 
          " application=" + application);
      }
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      LOG.debug("allocate: post-update");
      application.showRequests();
    }

    if(LOG.isDebugEnabled()) {
      LOG.debug("allocate:" +
        " applicationAttemptId=" + applicationAttemptId + 
        " #ask=" + ask.size());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);

    return application.getAllocation(getResourceCalculator(),
                 clusterResource, getMinimumResourceCapability());
  }
}

Example 4

Source File: FifoScheduler.java From big-c with Apache License 2.0

4 votes

@Override
public Allocation allocate(
    ApplicationAttemptId applicationAttemptId, List<ResourceRequest> ask,
    List<ContainerId> release, List<String> blacklistAdditions, List<String> blacklistRemovals) {
  FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
  if (application == null) {
    LOG.error("Calling allocate on removed " +
        "or non existant application " + applicationAttemptId);
    return EMPTY_ALLOCATION;
  }

  // Sanity check
  SchedulerUtils.normalizeRequests(ask, resourceCalculator, 
      clusterResource, minimumAllocation, getMaximumResourceCapability());

  // Release containers
  releaseContainers(release, application);

  synchronized (application) {

    // make sure we aren't stopping/removing the application
    // when the allocate comes in
    if (application.isStopped()) {
      LOG.info("Calling allocate on a stopped " +
          "application " + applicationAttemptId);
      return EMPTY_ALLOCATION;
    }

    if (!ask.isEmpty()) {
      LOG.debug("allocate: pre-update" +
          " applicationId=" + applicationAttemptId + 
          " application=" + application);
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      LOG.debug("allocate: post-update" +
          " applicationId=" + applicationAttemptId + 
          " application=" + application);
      application.showRequests();

      LOG.debug("allocate:" +
          " applicationId=" + applicationAttemptId + 
          " #ask=" + ask.size());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    ContainersAndNMTokensAllocation allocation =
        application.pullNewlyAllocatedContainersAndNMTokens();
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(allocation.getContainerList(), headroom, null,
        null, null, allocation.getNMTokenList());
  }
}

Example 5

Source File: FairScheduler.java From big-c with Apache License 2.0

4 votes

@Override
public Allocation allocate(ApplicationAttemptId appAttemptId,
    List<ResourceRequest> ask, List<ContainerId> release,
    List<String> blacklistAdditions, List<String> blacklistRemovals) {

  // Make sure this application exists
  FSAppAttempt application = getSchedulerApp(appAttemptId);
  if (application == null) {
    LOG.info("Calling allocate on removed " +
        "or non existant application " + appAttemptId);
    return EMPTY_ALLOCATION;
  }

  // Sanity check
  SchedulerUtils.normalizeRequests(ask, DOMINANT_RESOURCE_CALCULATOR,
      clusterResource, minimumAllocation, getMaximumResourceCapability(),
      incrAllocation);

  // Set amResource for this app
  if (!application.getUnmanagedAM() && ask.size() == 1
      && application.getLiveContainers().isEmpty()) {
    application.setAMResource(ask.get(0).getCapability());
  }

  // Release containers
  releaseContainers(release, application);

  synchronized (application) {
    if (!ask.isEmpty()) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("allocate: pre-update" +
            " applicationAttemptId=" + appAttemptId +
            " application=" + application.getApplicationId());
      }
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      application.showRequests();
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("allocate: post-update" +
          " applicationAttemptId=" + appAttemptId +
          " #ask=" + ask.size() +
          " reservation= " + application.getCurrentReservation());

      LOG.debug("Preempting " + application.getPreemptionContainers().size()
          + " container(s)");
    }
    
    Set<ContainerId> preemptionContainerIds = new HashSet<ContainerId>();
    for (RMContainer container : application.getPreemptionContainers()) {
      preemptionContainerIds.add(container.getContainerId());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);
    ContainersAndNMTokensAllocation allocation =
        application.pullNewlyAllocatedContainersAndNMTokens();
    Resource headroom = application.getHeadroom();
    application.setApplicationHeadroomForMetrics(headroom);
    return new Allocation(allocation.getContainerList(), headroom,
        preemptionContainerIds, null, null, allocation.getNMTokenList());
  }
}

Example 6

Source File: CapacityScheduler.java From big-c with Apache License 2.0

4 votes

@Override
@Lock(Lock.NoLock.class)
public Allocation allocate(ApplicationAttemptId applicationAttemptId,
    List<ResourceRequest> ask, List<ContainerId> release, 
    List<String> blacklistAdditions, List<String> blacklistRemovals) {

  FiCaSchedulerApp application = getApplicationAttempt(applicationAttemptId);
  if (application == null) {
    LOG.info("Calling allocate on removed " +
        "or non existant application " + applicationAttemptId);
    return EMPTY_ALLOCATION;
  }
  
  // Sanity check
  SchedulerUtils.normalizeRequests(
      ask, getResourceCalculator(), getClusterResource(),
      getMinimumResourceCapability(), getMaximumResourceCapability());

  // Release containers
 
  releaseContainers(release, application);

  synchronized (application) {

    // make sure we aren't stopping/removing the application
    // when the allocate comes in
    if (application.isStopped()) {
      LOG.info("Calling allocate on a stopped " +
          "application " + applicationAttemptId);
      return EMPTY_ALLOCATION;
    }

    if (!ask.isEmpty()) {

      if(LOG.isDebugEnabled()) {
        LOG.debug("allocate: pre-update" +
          " applicationAttemptId=" + applicationAttemptId + 
          " application=" + application);
      }
      application.showRequests();

      // Update application requests
      application.updateResourceRequests(ask);

      LOG.debug("allocate: post-update");
      application.showRequests();
    }

    if(LOG.isDebugEnabled()) {
      LOG.debug("allocate:" +
        " applicationAttemptId=" + applicationAttemptId + 
        " #ask=" + ask.size());
    }

    application.updateBlacklist(blacklistAdditions, blacklistRemovals);

    return application.getAllocation(getResourceCalculator(),
                 clusterResource, getMinimumResourceCapability());
  }
}