Java Code Examples for org.apache.hadoop.yarn.api.records.Resource#getGpuCores()

The following examples show how to use org.apache.hadoop.yarn.api.records.Resource#getGpuCores() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: RMContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }

  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = resource.getMemory()
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long gcoreSeconds = resource.getGpuCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds, gcoreSeconds);
  }
}
 
Example 2
Source Project: hadoop   File: AbstractYarnScheduler.java    License: Apache License 2.0 6 votes vote down vote up
protected void refreshMaximumAllocation(Resource newMaxAlloc) {
  maxAllocWriteLock.lock();
  try {
    configuredMaximumAllocation = Resources.clone(newMaxAlloc);
    int maxMemory = newMaxAlloc.getMemory();
    if (maxNodeMemory != -1) {
      maxMemory = Math.min(maxMemory, maxNodeMemory);
    }
    int maxVcores = newMaxAlloc.getVirtualCores();
    if (maxNodeVCores != -1) {
      maxVcores = Math.min(maxVcores, maxNodeVCores);
    }
    int maxGcores = newMaxAlloc.getGpuCores();
    if (maxNodeGCores != -1) {
      maxGcores = Math.min(maxGcores, maxNodeGCores);
    }
    maximumAllocation = Resources.createResource(maxMemory, maxVcores, maxGcores);
  } finally {
    maxAllocWriteLock.unlock();
  }
}
 
Example 3
Source Project: hadoop   File: ResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource other) {
  int diff = this.getMemory() - other.getMemory();
  if (diff == 0) {
    diff = this.getVirtualCores() - other.getVirtualCores();
    if (diff == 0) {
      diff = this.getGpuCores() - other.getGpuCores();
    }
  }
  return diff;
}
 
Example 4
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 
Example 5
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compareTo(Resource o) {
  int diff = 0 - o.getMemory();
  if (diff == 0) {
    diff = 0 - o.getVirtualCores();
    if (diff == 0) {
      diff = 0 - o.getGpuCores();
    }
  }
  return diff;
}
 
Example 6
@Override
public int compare(Resource clusterResource, Resource lhs, Resource rhs) {
  
  if (lhs.equals(rhs)) {
    return 0;
  }

  float[] lValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) lhs.getMemory() / clusterResource.getMemory() : lhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) lhs.getVirtualCores() / clusterResource.getVirtualCores() : lhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) lhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(lValues);

  float[] rValues = new float[] {
    (clusterResource.getMemory() != 0) ? (float) rhs.getMemory() / clusterResource.getMemory() : rhs.getMemory(),
    (clusterResource.getVirtualCores() != 0) ? (float) rhs.getVirtualCores() / clusterResource.getVirtualCores() : rhs.getVirtualCores(),
    (clusterResource.getGpuCores() != 0) ? (float) rhs.getGpuCores() / clusterResource.getGpuCores() : 0.0f };
  Arrays.sort(rValues);

  int diff = 0;
  for(int i = 0; i < 3; i++) {
    float l = lValues[i];
    float r = rValues[i];
    if (l < r) {
      diff = -1;
    } else if (l > r) {
      diff = 1;
    }
  }
  
  return diff;
}
 
Example 7
Source Project: hadoop   File: ResourceCalculatorUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static int divideAndCeilContainers(Resource required, Resource factor,
    EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (factor.getGpuCores() == 0) {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()));
    }
    else {
      return Math.max(divideAndCeil(required.getMemory(), factor.getMemory()),
              Math.max(divideAndCeil(required.getVirtualCores(), factor.getVirtualCores()),
                      divideAndCeil(required.getGpuCores(), factor.getGpuCores())));
    }
  }
  return divideAndCeil(required.getMemory(), factor.getMemory());
}
 
Example 8
@Override
public int computeAvailableContainers(Resource available, Resource required) {
  int min = Math.min(
      available.getMemory() / required.getMemory(),
      available.getVirtualCores() / required.getVirtualCores());
  if (required.getGpuCores() != 0) {
    min = Math.min(min,
        available.getGpuCores() / required.getGpuCores());
  }
  return min;
}
 
Example 9
@Override
public boolean isInvalidDivisor(Resource r) {
  if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f || r.getGpuCores() == 0.0f) {
    return true;
  }
  return false;
}
 
Example 10
Source Project: hadoop   File: AMRMClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compare(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  if(mem0 == mem1) {
    if(cpu0 == cpu1) {
      if(gpu0 == gpu1) {
        return 0;
      }
      if(gpu0 < gpu1) {
        return 1;
      }
      return -1;
    }
    if(cpu0 < cpu1) {
      return 1;
    }
    return -1;
  }
  if(mem0 < mem1) { 
    return 1;
  }
  return -1;
}
 
Example 11
Source Project: hadoop   File: LeafQueue.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void reinitialize(
    CSQueue newlyParsedQueue, Resource clusterResource) 
throws IOException {
  // Sanity check
  if (!(newlyParsedQueue instanceof LeafQueue) || 
      !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
    throw new IOException("Trying to reinitialize " + getQueuePath() + 
        " from " + newlyParsedQueue.getQueuePath());
  }

  LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;

  // don't allow the maximum allocation to be decreased in size
  // since we have already told running AM's the size
  Resource oldMax = getMaximumAllocation();
  Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
  if (newMax.getMemory() < oldMax.getMemory()
      || newMax.getVirtualCores() < oldMax.getVirtualCores()
      || newMax.getGpuCores() < oldMax.getGpuCores()) {
    throw new IOException(
        "Trying to reinitialize "
            + getQueuePath()
            + " the maximum allocation size can not be decreased!"
            + " Current setting: " + oldMax
            + ", trying to set it to: " + newMax);
  }

  setupQueueConfigs(clusterResource);

  // queue metrics are updated, more resource may be available
  // activate the pending applications if possible
  activateApplications();
}
 
Example 12
Source Project: hadoop   File: AMRMClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
static boolean canFit(Resource arg0, Resource arg1) {
  int mem0 = arg0.getMemory();
  int mem1 = arg1.getMemory();
  int cpu0 = arg0.getVirtualCores();
  int cpu1 = arg1.getVirtualCores();
  int gpu0 = arg0.getGpuCores();
  int gpu1 = arg1.getGpuCores();
  
  if(mem0 <= mem1 && cpu0 <= cpu1 && gpu0 <= gpu1) {
    return true;
  }
  return false; 
}
 
Example 13
Source Project: hadoop   File: ResourceCalculatorUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static int computeAvailableContainers(Resource available,
    Resource required, EnumSet<SchedulerResourceTypes> resourceTypes) {
  if (resourceTypes.contains(SchedulerResourceTypes.CPU)) {
    if (required.getGpuCores() == 0) {
      return Math.min(available.getMemory() / required.getMemory(),
              available.getVirtualCores() / required.getVirtualCores());
    }
    else {
      return Math.min(available.getMemory() / required.getMemory(),
              Math.min(available.getVirtualCores() / required.getVirtualCores(),
                      available.getGpuCores() / required.getGpuCores()));
    }
  }
  return available.getMemory() / required.getMemory();
}
 
Example 14
Source Project: hadoop   File: ComputeFairShares.java    License: Apache License 2.0 5 votes vote down vote up
private static int getResourceValue(Resource resource, ResourceType type) {
  switch (type) {
  case MEMORY:
    return resource.getMemory();
  case CPU:
    return resource.getVirtualCores();
  case GPU:
    return resource.getGpuCores();
  default:
    throw new IllegalArgumentException("Invalid resource");
  }
}
 
Example 15
Source Project: hadoop   File: Resources.java    License: Apache License 2.0 4 votes vote down vote up
public static boolean fitsIn(Resource smaller, Resource bigger) {
  return smaller.getMemory() <= bigger.getMemory() &&
      smaller.getVirtualCores() <= bigger.getVirtualCores() &&
      smaller.getGpuCores() <= bigger.getGpuCores();
}
 
Example 16
Source Project: hadoop   File: AppInfo.java    License: Apache License 2.0 4 votes vote down vote up
@SuppressWarnings({ "rawtypes", "unchecked" })
public AppInfo(ResourceManager rm, RMApp app, Boolean hasAccess,
    String schemePrefix) {
  this.schemePrefix = schemePrefix;
  if (app != null) {
    String trackingUrl = app.getTrackingUrl();
    this.state = app.createApplicationState();
    this.trackingUrlIsNotReady = trackingUrl == null || trackingUrl.isEmpty()
        || YarnApplicationState.NEW == this.state
        || YarnApplicationState.NEW_SAVING == this.state
        || YarnApplicationState.SUBMITTED == this.state
        || YarnApplicationState.ACCEPTED == this.state;
    this.trackingUI = this.trackingUrlIsNotReady ? "UNASSIGNED" : (app
        .getFinishTime() == 0 ? "ApplicationMaster" : "History");
    if (!trackingUrlIsNotReady) {
      this.trackingUrl =
          WebAppUtils.getURLWithScheme(schemePrefix,
              trackingUrl);
      this.trackingUrlPretty = this.trackingUrl;
    } else {
      this.trackingUrlPretty = "UNASSIGNED";
    }
    this.applicationId = app.getApplicationId();
    this.applicationType = app.getApplicationType();
    this.appIdNum = String.valueOf(app.getApplicationId().getId());
    this.id = app.getApplicationId().toString();
    this.user = app.getUser().toString();
    this.name = app.getName().toString();
    this.queue = app.getQueue().toString();
    this.progress = app.getProgress() * 100;
    this.diagnostics = app.getDiagnostics().toString();
    if (diagnostics == null || diagnostics.isEmpty()) {
      this.diagnostics = "";
    }
    if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
      this.applicationTags = Joiner.on(',').join(app.getApplicationTags());
    }
    this.finalStatus = app.getFinalApplicationStatus();
    this.clusterId = ResourceManager.getClusterTimeStamp();
    if (hasAccess) {
      this.startedTime = app.getStartTime();
      this.finishedTime = app.getFinishTime();
      this.elapsedTime = Times.elapsed(app.getStartTime(),
          app.getFinishTime());

      RMAppAttempt attempt = app.getCurrentAppAttempt();
      if (attempt != null) {
        Container masterContainer = attempt.getMasterContainer();
        if (masterContainer != null) {
          this.amContainerLogsExist = true;
          this.amContainerLogs = WebAppUtils.getRunningLogURL(
              schemePrefix + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()),
              app.getUser());
          this.amHostHttpAddress = masterContainer.getNodeHttpAddress();
        }
        
        ApplicationResourceUsageReport resourceReport = attempt
            .getApplicationResourceUsageReport();
        if (resourceReport != null) {
          Resource usedResources = resourceReport.getUsedResources();
          allocatedMB = usedResources.getMemory();
          allocatedVCores = usedResources.getVirtualCores();
          allocatedGCores = usedResources.getGpuCores();
          runningContainers = resourceReport.getNumUsedContainers();
        }
        resourceRequests =
            ((AbstractYarnScheduler) rm.getRMContext().getScheduler())
              .getPendingResourceRequestsForAttempt(attempt.getAppAttemptId());
      }
    }

    // copy preemption info fields
    RMAppMetrics appMetrics = app.getRMAppMetrics();
    numAMContainerPreempted =
        appMetrics.getNumAMContainersPreempted();
    preemptedResourceMB =
        appMetrics.getResourcePreempted().getMemory();
    numNonAMContainerPreempted =
        appMetrics.getNumNonAMContainersPreempted();
    preemptedResourceVCores =
        appMetrics.getResourcePreempted().getVirtualCores();
    preemptedResourceGCores =
        appMetrics.getResourcePreempted().getGpuCores();
    memorySeconds = appMetrics.getMemorySeconds();
    vcoreSeconds = appMetrics.getVcoreSeconds();
    gcoreSeconds = appMetrics.getGcoreSeconds();
  }
}
 
Example 17
Source Project: hadoop   File: CapacityOverTimePolicy.java    License: Apache License 2.0 4 votes vote down vote up
public void add(Resource r) {
  memory += r.getMemory();
  vcores += r.getVirtualCores();
  gcores += r.getGpuCores();
}
 
Example 18
Source Project: hadoop   File: CapacityOverTimePolicy.java    License: Apache License 2.0 4 votes vote down vote up
public void subtract(Resource r) {
  memory -= r.getMemory();
  vcores -= r.getVirtualCores();
  gcores -= r.getGpuCores();
}
 
Example 19
Source Project: hadoop   File: SchedulerUtils.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore/gcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  if (resReq.getCapability().getGpuCores() < 0 ||
      resReq.getCapability().getGpuCores() >
      maximumResource.getGpuCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested gpu cores < 0"
        + ", or requested gpu cores > max configured"
        + ", requestedGpuCores="
        + resReq.getCapability().getGpuCores()
        + ", maxGpuCores=" + maximumResource.getGpuCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}
 
Example 20
/**
 * Get the per queue setting for the maximum limit to allocate to
 * each container request.
 *
 * @param queue
 *          name of the queue
 * @return setting specified per queue else falls back to the cluster setting
 */
public Resource getMaximumAllocationPerQueue(String queue) {
  String queuePrefix = getQueuePrefix(queue);
  int maxAllocationMbPerQueue = getInt(queuePrefix + MAXIMUM_ALLOCATION_MB,
      (int)UNDEFINED);
  int maxAllocationVcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_VCORES, (int)UNDEFINED);
  int maxAllocationGcoresPerQueue = getInt(
      queuePrefix + MAXIMUM_ALLOCATION_GCORES, (int)UNDEFINED);
  if (LOG.isDebugEnabled()) {
    LOG.debug("max alloc mb per queue for " + queue + " is "
        + maxAllocationMbPerQueue);
    LOG.debug("max alloc vcores per queue for " + queue + " is "
        + maxAllocationVcoresPerQueue);
    LOG.debug("max alloc gcores per queue for " + queue + " is "
        + maxAllocationGcoresPerQueue);
  }
  Resource clusterMax = getMaximumAllocation();
  if (maxAllocationMbPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc mb per queue for " + queue + " is undefined");
    maxAllocationMbPerQueue = clusterMax.getMemory();
  }
  if (maxAllocationVcoresPerQueue == (int)UNDEFINED) {
     LOG.info("max alloc vcore per queue for " + queue + " is undefined");
    maxAllocationVcoresPerQueue = clusterMax.getVirtualCores();
  }
  if (maxAllocationGcoresPerQueue == (int)UNDEFINED) {
    LOG.info("max alloc gcore per queue for " + queue + " is undefined");
    maxAllocationGcoresPerQueue = clusterMax.getGpuCores();
  }
  Resource result = Resources.createResource(maxAllocationMbPerQueue,
      maxAllocationVcoresPerQueue, maxAllocationGcoresPerQueue);
  if (maxAllocationMbPerQueue > clusterMax.getMemory()
      || maxAllocationVcoresPerQueue > clusterMax.getVirtualCores()
      || maxAllocationGcoresPerQueue > clusterMax.getGpuCores()) {
    throw new IllegalArgumentException(
        "Queue maximum allocation cannot be larger than the cluster setting"
        + " for queue " + queue
        + " max allocation per queue: " + result
        + " cluster setting: " + clusterMax);
  }
  return result;
}