Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#getNodeLabelExpression()

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceRequest#getNodeLabelExpression() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SchedulerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
private static void normalizeNodeLabelExpressionInRequest(
    ResourceRequest resReq, QueueInfo queueInfo) {

  String labelExp = resReq.getNodeLabelExpression();

  // if queue has default label expression, and RR doesn't have, use the
  // default label expression of queue
  if (labelExp == null && queueInfo != null && ResourceRequest.ANY
      .equals(resReq.getResourceName())) {
    labelExp = queueInfo.getDefaultNodeLabelExpression();
  }

  // If labelExp still equals to null, set it to be NO_LABEL
  if (labelExp == null) {
    labelExp = RMNodeLabelsManager.NO_LABEL;
  }
  resReq.setNodeLabelExpression(labelExp);
}
Example 2
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up 
private static void normalizeNodeLabelExpressionInRequest(
    ResourceRequest resReq, QueueInfo queueInfo) {

  String labelExp = resReq.getNodeLabelExpression();

  // if queue has default label expression, and RR doesn't have, use the
  // default label expression of queue
  if (labelExp == null && queueInfo != null && ResourceRequest.ANY
      .equals(resReq.getResourceName())) {
    labelExp = queueInfo.getDefaultNodeLabelExpression();
  }

  // If labelExp still equals to null, set it to be NO_LABEL
  if (labelExp == null) {
    labelExp = RMNodeLabelsManager.NO_LABEL;
  }
  resReq.setNodeLabelExpression(labelExp);
}
Example 3
Source File: LeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private boolean checkResourceRequestMatchingNodeLabel(ResourceRequest offswitchResourceRequest,
    FiCaSchedulerNode node) {
  String askedNodeLabel = offswitchResourceRequest.getNodeLabelExpression();
  if (null == askedNodeLabel) {
    askedNodeLabel = RMNodeLabelsManager.NO_LABEL;
  }
  return askedNodeLabel.equals(node.getPartition());
}
Example 4
Source File: AppSchedulingInfo.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private boolean isRequestLabelChanged(ResourceRequest requestOne,
    ResourceRequest requestTwo) {
  String requestOneLabelExp = requestOne.getNodeLabelExpression();
  String requestTwoLabelExp = requestTwo.getNodeLabelExpression();
  return (!(requestOneLabelExp.equals(requestTwoLabelExp)));
}
Example 5
Source File: SchedulerUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore/gcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  if (resReq.getCapability().getGpuCores() < 0 ||
      resReq.getCapability().getGpuCores() >
      maximumResource.getGpuCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested gpu cores < 0"
        + ", or requested gpu cores > max configured"
        + ", requestedGpuCores="
        + resReq.getCapability().getGpuCores()
        + ", maxGpuCores=" + maximumResource.getGpuCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}
Example 6
Source File: RMAppManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
private ResourceRequest validateAndCreateResourceRequest(
    ApplicationSubmissionContext submissionContext, boolean isRecovery)
    throws InvalidResourceRequestException {
  // Validation of the ApplicationSubmissionContext needs to be completed
  // here. Only those fields that are dependent on RM's configuration are
  // checked here as they have to be validated whether they are part of new
  // submission or just being recovered.

  // Check whether AM resource requirements are within required limits
  if (!submissionContext.getUnmanagedAM()) {
    ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
    if (amReq == null) {
      amReq = BuilderUtils
          .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
              ResourceRequest.ANY, submissionContext.getResource(), 1);
    }

    // set label expression for AM container
    if (null == amReq.getNodeLabelExpression()) {
      amReq.setNodeLabelExpression(submissionContext
          .getNodeLabelExpression());
    }

    try {
      SchedulerUtils.normalizeAndValidateRequest(amReq,
          scheduler.getMaximumResourceCapability(),
          submissionContext.getQueue(), scheduler, isRecovery, rmContext);
    } catch (InvalidResourceRequestException e) {
      LOG.warn("RM app submission failed in validating AM resource request"
          + " for application " + submissionContext.getApplicationId(), e);
      throw e;
    }

    SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(),
        scheduler.getClusterResource(),
        scheduler.getMinimumResourceCapability(),
        scheduler.getMaximumResourceCapability(),
        scheduler.getMinimumResourceCapability());
    return amReq;
  }
  
  return null;
}
Example 7
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
/**
 * Utility method to validate a resource request, by insuring that the
 * requested memory/vcore is non-negative and not greater than max
 * 
 * @throws InvalidResourceRequestException when there is invalid request
 */
private static void validateResourceRequest(ResourceRequest resReq,
    Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
    throws InvalidResourceRequestException {
  if (resReq.getCapability().getMemory() < 0 ||
      resReq.getCapability().getMemory() > maximumResource.getMemory()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested memory < 0"
        + ", or requested memory > max configured"
        + ", requestedMemory=" + resReq.getCapability().getMemory()
        + ", maxMemory=" + maximumResource.getMemory());
  }
  if (resReq.getCapability().getVirtualCores() < 0 ||
      resReq.getCapability().getVirtualCores() >
      maximumResource.getVirtualCores()) {
    throw new InvalidResourceRequestException("Invalid resource request"
        + ", requested virtual cores < 0"
        + ", or requested virtual cores > max configured"
        + ", requestedVirtualCores="
        + resReq.getCapability().getVirtualCores()
        + ", maxVirtualCores=" + maximumResource.getVirtualCores());
  }
  String labelExp = resReq.getNodeLabelExpression();

  // we don't allow specify label expression other than resourceName=ANY now
  if (!ResourceRequest.ANY.equals(resReq.getResourceName())
      && labelExp != null && !labelExp.trim().isEmpty()) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified node label expression in a "
            + "resource request has resource name = "
            + resReq.getResourceName());
  }
  
  // we don't allow specify label expression with more than one node labels now
  if (labelExp != null && labelExp.contains("&&")) {
    throw new InvalidResourceRequestException(
        "Invailid resource request, queue=" + queueInfo.getQueueName()
            + " specified more than one node label "
            + "in a node label expression, node label expression = "
            + labelExp);
  }
  
  if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
    if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
        labelExp, rmContext)) {
      throw new InvalidResourceRequestException("Invalid resource request"
          + ", queue="
          + queueInfo.getQueueName()
          + " doesn't have permission to access all labels "
          + "in resource request. labelExpression of resource request="
          + labelExp
          + ". Queue labels="
          + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
              .getAccessibleNodeLabels().iterator(), ',')));
    }
  }
}
Example 8
Source File: RMAppManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
private ResourceRequest validateAndCreateResourceRequest(
    ApplicationSubmissionContext submissionContext, boolean isRecovery)
    throws InvalidResourceRequestException {
  // Validation of the ApplicationSubmissionContext needs to be completed
  // here. Only those fields that are dependent on RM's configuration are
  // checked here as they have to be validated whether they are part of new
  // submission or just being recovered.

  // Check whether AM resource requirements are within required limits
  if (!submissionContext.getUnmanagedAM()) {
    ResourceRequest amReq = submissionContext.getAMContainerResourceRequest();
    if (amReq == null) {
      amReq = BuilderUtils
          .newResourceRequest(RMAppAttemptImpl.AM_CONTAINER_PRIORITY,
              ResourceRequest.ANY, submissionContext.getResource(), 1);
    }

    // set label expression for AM container
    if (null == amReq.getNodeLabelExpression()) {
      amReq.setNodeLabelExpression(submissionContext
          .getNodeLabelExpression());
    }

    try {
      SchedulerUtils.normalizeAndValidateRequest(amReq,
          scheduler.getMaximumResourceCapability(),
          submissionContext.getQueue(), scheduler, isRecovery, rmContext);
    } catch (InvalidResourceRequestException e) {
      LOG.warn("RM app submission failed in validating AM resource request"
          + " for application " + submissionContext.getApplicationId(), e);
      throw e;
    }

    SchedulerUtils.normalizeRequest(amReq, scheduler.getResourceCalculator(),
        scheduler.getClusterResource(),
        scheduler.getMinimumResourceCapability(),
        scheduler.getMaximumResourceCapability(),
        scheduler.getMinimumResourceCapability());
    return amReq;
  }
  
  return null;
}