Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#setCapability()

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceRequest#setCapability() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MockAM.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
public ResourceRequest createResourceReq(String resource, int memory, int priority,
    int containers, String labelExpression) throws Exception {
  ResourceRequest req = Records.newRecord(ResourceRequest.class);
  req.setResourceName(resource);
  req.setNumContainers(containers);
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(priority);
  req.setPriority(pri);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  req.setCapability(capability);
  if (labelExpression != null) {
   req.setNodeLabelExpression(labelExpression); 
  }
  return req;
}
Example 2
Source File: RMContainerRequestor.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void addResourceRequest(Priority priority, String resourceName,
    Resource capability) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  if (remoteRequests == null) {
    remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
    this.remoteRequestsTable.put(priority, remoteRequests);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Added priority=" + priority);
    }
  }
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    reqMap = new HashMap<Resource, ResourceRequest>();
    remoteRequests.put(resourceName, reqMap);
  }
  ResourceRequest remoteRequest = reqMap.get(capability);
  if (remoteRequest == null) {
    remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
    remoteRequest.setPriority(priority);
    remoteRequest.setResourceName(resourceName);
    remoteRequest.setCapability(capability);
    remoteRequest.setNumContainers(0);
    reqMap.put(capability, remoteRequest);
  }
  remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);

  // Note this down for next interaction with ResourceManager
  addResourceRequestToAsk(remoteRequest);
  if (LOG.isDebugEnabled()) {
    LOG.debug("addResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
Example 3
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private ResourceRequest createResourceRequest(int memory, String host,
    int priority, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setCapability(Resources.createResource(memory));
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  return request;
}
Example 4
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
  ResourceCalculator resourceCalculator = new DominantResourceCalculator();
  
  Resource minResource = Resources.createResource(1024, 1);
  Resource maxResource = Resources.createResource(10240, 10);
  Resource clusterResource = Resources.createResource(10 * 1024, 10);
  
  ResourceRequest ask = new ResourceRequestPBImpl();

  // case negative memory/vcores
  ask.setCapability(Resources.createResource(-1024, -1));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());

  // case zero memory/vcores
  ask.setCapability(Resources.createResource(0, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(1024, ask.getCapability().getMemory());

  // case non-zero memory & zero cores
  ask.setCapability(Resources.createResource(1536, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(Resources.createResource(2048, 1), ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(2048, ask.getCapability().getMemory());
}
Example 5
Source File: AMSimulator.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
        Resource resource, String host, int priority, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setCapability(resource);
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  return request;
}
Example 6
Source File: SchedulerUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Utility method to normalize a resource request, by insuring that the
 * requested memory is a multiple of minMemory and is not zero.
 */
public static void normalizeRequest(
  ResourceRequest ask,
  ResourceCalculator resourceCalculator,
  Resource clusterResource,
  Resource minimumResource,
  Resource maximumResource) {
  Resource normalized =
    Resources.normalize(
      resourceCalculator, ask.getCapability(), minimumResource,
      maximumResource, minimumResource);
  ask.setCapability(normalized);
}
Example 7
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(ResourceRequest r) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(r.getPriority());
  request.setResourceName(r.getResourceName());
  request.setCapability(r.getCapability());
  request.setNumContainers(r.getNumContainers());
  return request;
}
Example 8
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(Priority priority,
    String hostName, Resource capability, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(priority);
  request.setResourceName(hostName);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  return request;
}
Example 9
Source File: TestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 10
Source File: RMContainerRequestor.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void addResourceRequest(Priority priority, String resourceName,
    Resource capability, String nodeLabelExpression) {
  Map<String, Map<Resource, ResourceRequest>> remoteRequests =
    this.remoteRequestsTable.get(priority);
  if (remoteRequests == null) {
    remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
    this.remoteRequestsTable.put(priority, remoteRequests);
    if (LOG.isDebugEnabled()) {
      LOG.debug("Added priority=" + priority);
    }
  }
  Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
  if (reqMap == null) {
    reqMap = new HashMap<Resource, ResourceRequest>();
    remoteRequests.put(resourceName, reqMap);
  }
  ResourceRequest remoteRequest = reqMap.get(capability);
  if (remoteRequest == null) {
    remoteRequest = recordFactory.newRecordInstance(ResourceRequest.class);
    remoteRequest.setPriority(priority);
    remoteRequest.setResourceName(resourceName);
    remoteRequest.setCapability(capability);
    remoteRequest.setNumContainers(0);
    remoteRequest.setNodeLabelExpression(nodeLabelExpression);
    reqMap.put(capability, remoteRequest);
  }
  remoteRequest.setNumContainers(remoteRequest.getNumContainers() + 1);

  // Note this down for next interaction with ResourceManager
  addResourceRequestToAsk(remoteRequest);
  if (LOG.isDebugEnabled()) {
    LOG.debug("addResourceRequest:" + " applicationId="
        + applicationId.getId() + " priority=" + priority.getPriority()
        + " resourceName=" + resourceName + " numContainers="
        + remoteRequest.getNumContainers() + " #asks=" + ask.size());
  }
}
Example 11
Source File: TestUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
Example 12
Source File: FairSchedulerTestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
    int memory, int vcores, int gcores, String host, int priority, int numContainers,
    boolean relaxLocality) {
  ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
  request.setCapability(BuilderUtils.newResource(memory, vcores, gcores));
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  request.setRelaxLocality(relaxLocality);
  request.setNodeLabelExpression(RMNodeLabelsManager.NO_LABEL);
  return request;
}
Example 13
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test (timeout = 30000)
public void testNormalizeRequestWithDominantResourceCalculator() {
  ResourceCalculator resourceCalculator = new DominantResourceCalculator();
  
  Resource minResource = Resources.createResource(1024, 1, 0);
  Resource maxResource = Resources.createResource(10240, 10, 10);
  Resource clusterResource = Resources.createResource(10 * 1024, 10, 10);
  
  ResourceRequest ask = new ResourceRequestPBImpl();

  // case negative memory/vcores/gcores
  ask.setCapability(Resources.createResource(-1024, -1, -1));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());

  // case zero memory/vcores/gcores
  ask.setCapability(Resources.createResource(0, 0, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(minResource, ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(1024, ask.getCapability().getMemory());
  assertEquals(0, ask.getCapability().getGpuCores());

  // case non-zero memory & zero cores
  ask.setCapability(Resources.createResource(1536, 0, 0));
  SchedulerUtils.normalizeRequest(
      ask, resourceCalculator, clusterResource, minResource, maxResource);
  assertEquals(Resources.createResource(2048, 1, 0), ask.getCapability());
  assertEquals(1, ask.getCapability().getVirtualCores());
  assertEquals(2048, ask.getCapability().getMemory());
  assertEquals(0, ask.getCapability().getGpuCores());
}
Example 14
Source File: FairSchedulerTestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up 
protected ResourceRequest createResourceRequest(
    int memory, int vcores, String host, int priority, int numContainers,
    boolean relaxLocality) {
  ResourceRequest request = recordFactory.newRecordInstance(ResourceRequest.class);
  request.setCapability(BuilderUtils.newResource(memory, vcores));
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  request.setRelaxLocality(relaxLocality);
  return request;
}
Example 15
Source File: TestFifoScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private ResourceRequest createResourceRequest(int memory, String host,
    int priority, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setCapability(Resources.createResource(memory));
  request.setResourceName(host);
  request.setNumContainers(numContainers);
  Priority prio = recordFactory.newRecordInstance(Priority.class);
  prio.setPriority(priority);
  request.setPriority(prio);
  return request;
}
Example 16
Source File: SchedulerUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Utility method to normalize a resource request, by insuring that the
 * requested memory is a multiple of minMemory and is not zero.
 */
public static void normalizeRequest(
    ResourceRequest ask, 
    ResourceCalculator resourceCalculator, 
    Resource clusterResource,
    Resource minimumResource,
    Resource maximumResource,
    Resource incrementResource) {
  Resource normalized = 
      Resources.normalize(
          resourceCalculator, ask.getCapability(), minimumResource,
          maximumResource, incrementResource);
  ask.setCapability(normalized);
}
Example 17
Source File: SchedulerUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Utility method to normalize a resource request, by insuring that the
 * requested memory is a multiple of minMemory and is not zero.
 */
public static void normalizeRequest(
  ResourceRequest ask,
  ResourceCalculator resourceCalculator,
  Resource clusterResource,
  Resource minimumResource,
  Resource maximumResource) {
  Resource normalized =
    Resources.normalize(
      resourceCalculator, ask.getCapability(), minimumResource,
      maximumResource, minimumResource);
  ask.setCapability(normalized);
}
Example 18
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(ResourceRequest r) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(r.getPriority());
  request.setResourceName(r.getResourceName());
  request.setCapability(r.getCapability());
  request.setNumContainers(r.getNumContainers());
  return request;
}
Example 19
Source File: BuilderUtils.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
public static ResourceRequest newResourceRequest(Priority priority,
    String hostName, Resource capability, int numContainers) {
  ResourceRequest request = recordFactory
      .newRecordInstance(ResourceRequest.class);
  request.setPriority(priority);
  request.setResourceName(hostName);
  request.setCapability(capability);
  request.setNumContainers(numContainers);
  return request;
}
Example 20
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test (timeout = 30000)
public void testNormalizeRequest() {
  ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
  
  final int minMemory = 1024;
  final int maxMemory = 8192;
  Resource minResource = Resources.createResource(minMemory, 0);
  Resource maxResource = Resources.createResource(maxMemory, 0);
  
  ResourceRequest ask = new ResourceRequestPBImpl();

  // case negative memory
  ask.setCapability(Resources.createResource(-1024));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(minMemory, ask.getCapability().getMemory());

  // case zero memory
  ask.setCapability(Resources.createResource(0));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(minMemory, ask.getCapability().getMemory());

  // case memory is a multiple of minMemory
  ask.setCapability(Resources.createResource(2 * minMemory));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(2 * minMemory, ask.getCapability().getMemory());

  // case memory is not a multiple of minMemory
  ask.setCapability(Resources.createResource(minMemory + 10));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(2 * minMemory, ask.getCapability().getMemory());

  // case memory is equal to max allowed
  ask.setCapability(Resources.createResource(maxMemory));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxMemory, ask.getCapability().getMemory());

  // case memory is just less than max
  ask.setCapability(Resources.createResource(maxMemory - 10));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxMemory, ask.getCapability().getMemory());

  // max is not a multiple of min
  maxResource = Resources.createResource(maxMemory - 10, 0);
  ask.setCapability(Resources.createResource(maxMemory - 100));
  // multiple of minMemory > maxMemory, then reduce to maxMemory
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());

  // ask is more than max
  maxResource = Resources.createResource(maxMemory, 0);
  ask.setCapability(Resources.createResource(maxMemory + 100));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}