Java Code Examples for org.apache.hadoop.yarn.api.records.ResourceRequest#ANY

The following examples show how to use org.apache.hadoop.yarn.api.records.ResourceRequest#ANY . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: RMServerUtils.java From hadoop with Apache License 2.0

5 votes

public static void validateBlacklistRequest(
    ResourceBlacklistRequest blacklistRequest)
    throws InvalidResourceBlacklistRequestException {
  if (blacklistRequest != null) {
    List<String> plus = blacklistRequest.getBlacklistAdditions();
    if (plus != null && plus.contains(ResourceRequest.ANY)) {
      throw new InvalidResourceBlacklistRequestException(
          "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
    }
  }
}

Example 2

Source File: RMServerUtils.java From big-c with Apache License 2.0

5 votes

public static void validateBlacklistRequest(
    ResourceBlacklistRequest blacklistRequest)
    throws InvalidResourceBlacklistRequestException {
  if (blacklistRequest != null) {
    List<String> plus = blacklistRequest.getBlacklistAdditions();
    if (plus != null && plus.contains(ResourceRequest.ANY)) {
      throw new InvalidResourceBlacklistRequestException(
          "Cannot add " + ResourceRequest.ANY + " to the blacklist!");
    }
  }
}

Example 3

Source File: YarnTaskSchedulerService.java From incubator-tez with Apache License 2.0

5 votes

@Override
public CookieContainerRequest assignNewContainer(Container container) {
  String location = ResourceRequest.ANY;
  CookieContainerRequest assigned = getMatchingRequestWithPriority(container,
      location);
  doBookKeepingForAssignedContainer(assigned, container, location, false);
  return assigned;
}

Example 4

Source File: YarnTaskSchedulerService.java From incubator-tez with Apache License 2.0

5 votes

@Override
public CookieContainerRequest assignReUsedContainer(Container container,
    boolean honorLocality) {
  if (!honorLocality) {
    String location = ResourceRequest.ANY;
    CookieContainerRequest assigned = getMatchingRequestWithoutPriority(
      container, location, false);
    doBookKeepingForAssignedContainer(assigned, container, location,
        honorLocality);
    return assigned;
  }
  return null;
}

Example 5

Source File: TezAMRMClientAsync.java From incubator-tez with Apache License 2.0

5 votes

public synchronized List<? extends Collection<T>>
  getMatchingRequestsForTopPriority(
      String resourceName, Resource capability) {
  // Sort based on reverse order. By default, Priority ordering is based on
  // highest numeric value being considered to be lowest priority.
  Iterator<Priority> iter =
    knownRequestsByPriority.descendingKeySet().iterator();
  if (!iter.hasNext()) {
    return Collections.emptyList();
  }
  Priority p = iter.next();
  LocalityRequestCounter lrc = knownRequestsByPriority.get(p);
  if (lrc.localityRequests.get() == 0) {
    // Fallback to ANY if there are no pending requests that require
    // locality matching
    if (LOG.isDebugEnabled()) {
      LOG.debug("Over-ridding location request for matching containers as"
        + " there are no pending requests that require locality at this"
        + " priority"
        + ", priority=" + p
        + ", localityRequests=" + lrc.localityRequests
        + ", noLocalityRequests=" + lrc.noLocalityRequests);
    }
    resourceName = ResourceRequest.ANY;
  }
  List<? extends Collection<T>> matched =
    getMatchingRequests(p, resourceName, capability);
  if (matched != null && !matched.isEmpty()) {
    return matched;
  }
  return Collections.emptyList();
}

Example 6

Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0

5 votes

@Override
public CookieContainerRequest assignNewContainer(Container container) {
  String location = ResourceRequest.ANY;
  CookieContainerRequest assigned = getMatchingRequestWithPriority(container,
      location);
  doBookKeepingForAssignedContainer(assigned, container, location, false);
  return assigned;
}

Example 7

Source File: YarnTaskSchedulerService.java From tez with Apache License 2.0

5 votes

@Override
public CookieContainerRequest assignReUsedContainer(Container container,
    boolean honorLocality) {
  if (!honorLocality) {
    String location = ResourceRequest.ANY;
    CookieContainerRequest assigned = getMatchingRequestWithoutPriority(
      container, location, false);
    doBookKeepingForAssignedContainer(assigned, container, location,
        honorLocality);
    return assigned;
  }
  return null;
}

Example 8

Source File: TezAMRMClientAsync.java From tez with Apache License 2.0

5 votes

public synchronized List<? extends Collection<T>>
  getMatchingRequestsForTopPriority(
      String resourceName, Resource capability) {
  // Sort based on reverse order. By default, Priority ordering is based on
  // highest numeric value being considered to be lowest priority.
  Map.Entry<Priority,LocalityRequestCounter> entry = knownRequestsByPriority.lastEntry();
  if (entry == null || entry.getValue() == null) {
    return Collections.emptyList();
  }
  Priority p = entry.getKey();
  LocalityRequestCounter lrc = entry.getValue();
  if (lrc.localityRequests.get() == 0) {
    // Fallback to ANY if there are no pending requests that require
    // locality matching
    if (LOG.isDebugEnabled()) {
      LOG.debug("Over-ridding location request for matching containers as"
        + " there are no pending requests that require locality at this"
        + " priority"
        + ", priority=" + p
        + ", localityRequests=" + lrc.localityRequests
        + ", noLocalityRequests=" + lrc.noLocalityRequests);
    }
    resourceName = ResourceRequest.ANY;
  }
  List<? extends Collection<T>> matched =
    getMatchingRequests(p, resourceName, capability);
  if (matched != null && !matched.isEmpty()) {
    return matched;
  }
  return Collections.emptyList();
}

Example 9

Source File: DagAwareYarnTaskScheduler.java From tez with Apache License 2.0

5 votes

@GuardedBy("this")
@Nullable
private TaskRequest tryAssignReuseContainerAppRunning(HeldContainer hc) {
  if (!hc.isAssignable()) {
    LOG.debug("Skipping scheduling of container {} because it state is {}", hc.getId(), hc.getState());
    return null;
  }

  TaskRequest assignedRequest = tryAssignReuseContainerForAffinity(hc);
  if (assignedRequest != null) {
    return assignedRequest;
  }

  for (Entry<Priority,RequestPriorityStats> entry : requestTracker.getStatsEntries()) {
    Priority priority = entry.getKey();
    RequestPriorityStats stats = entry.getValue();
    if (!stats.allowedVertices.intersects(stats.vertices)) {
      LOG.debug("Skipping requests at priority {} because all requesting vertices are blocked by higher priority requests",
          priority);
      continue;
    }

    String matchLocation = hc.getMatchingLocation();
    if (stats.localityCount <= 0) {
      LOG.debug("Overriding locality match of container {} to ANY since there are no locality requests at priority {}",
          hc.getId(), priority);
      matchLocation = ResourceRequest.ANY;
    }
    assignedRequest = tryAssignReuseContainerForPriority(hc, matchLocation,
        priority, stats.allowedVertices);
    if (assignedRequest != null) {
      break;
    }
  }
  return assignedRequest;
}

Example 10

Source File: DagAwareYarnTaskScheduler.java From tez with Apache License 2.0

5 votes

String getMatchingLocation() {
  switch (state) {
  case MATCHING_LOCAL:
    return getHost();
  case MATCHING_RACK:
    return getRack();
  case MATCHING_ANY:
    return ResourceRequest.ANY;
  default:
    throw new IllegalStateException("Container " + getId() + " trying to match in state " + state);
  }
}

Example 11

Source File: TestResourceManager.java From hadoop with Apache License 2.0

4 votes

@Test
public void testResourceAllocation() throws IOException,
    YarnException, InterruptedException {
  LOG.info("--- START: testResourceAllocation ---");
      
  final int memory = 4 * 1024;
  final int vcores = 4;
  
  // Register node1
  String host1 = "host1";
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 = 
    registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, 
        Resources.createResource(memory, vcores));
  
  // Register node2
  String host2 = "host2";
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm2 = 
    registerNode(host2, 1234, 2345, NetworkTopology.DEFAULT_RACK, 
        Resources.createResource(memory/2, vcores/2));

  // Submit an application
  Application application = new Application("user1", resourceManager);
  application.submit();
  
  application.addNodeManager(host1, 1234, nm1);
  application.addNodeManager(host2, 1234, nm2);
  
  // Application resource requirements
  final int memory1 = 1024;
  Resource capability1 = Resources.createResource(memory1, 1);
  Priority priority1 = 
    org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.create(1);
  application.addResourceRequestSpec(priority1, capability1);
  
  Task t1 = new Task(application, priority1, new String[] {host1, host2});
  application.addTask(t1);
  
  final int memory2 = 2048;
  Resource capability2 = Resources.createResource(memory2, 1);
  Priority priority0 = 
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.create(0); // higher
  application.addResourceRequestSpec(priority0, capability2);
  
  // Send resource requests to the scheduler
  application.schedule();

 // Send a heartbeat to kick the tires on the Scheduler
  nodeUpdate(nm1);
  
  // Get allocations from the scheduler
  application.schedule();
  
  checkResourceUsage(nm1, nm2);
  
  LOG.info("Adding new tasks...");
  
  Task t2 = new Task(application, priority1, new String[] {host1, host2});
  application.addTask(t2);

  Task t3 = new Task(application, priority0, new String[] {ResourceRequest.ANY});
  application.addTask(t3);

  // Send resource requests to the scheduler
  application.schedule();
  checkResourceUsage(nm1, nm2);
  
  // Send heartbeats to kick the tires on the Scheduler
  nodeUpdate(nm2);
  nodeUpdate(nm2);
  nodeUpdate(nm1);
  nodeUpdate(nm1);
  
  // Get allocations from the scheduler
  LOG.info("Trying to allocate...");
  application.schedule();

  checkResourceUsage(nm1, nm2);
  
  // Complete tasks
  LOG.info("Finishing up tasks...");
  application.finishTask(t1);
  application.finishTask(t2);
  application.finishTask(t3);
  
  // Notify scheduler application is finished.
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(
        application.getApplicationAttemptId(), RMAppAttemptState.FINISHED, false);
  resourceManager.getResourceScheduler().handle(appRemovedEvent1);
  
  checkResourceUsage(nm1, nm2);
  
  LOG.info("--- END: testResourceAllocation ---");
}

Example 12

Source File: TestResourceManager.java From big-c with Apache License 2.0

4 votes

@Test
public void testResourceAllocation() throws IOException,
    YarnException, InterruptedException {
  LOG.info("--- START: testResourceAllocation ---");
      
  final int memory = 4 * 1024;
  final int vcores = 4;
  
  // Register node1
  String host1 = "host1";
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm1 = 
    registerNode(host1, 1234, 2345, NetworkTopology.DEFAULT_RACK, 
        Resources.createResource(memory, vcores));
  
  // Register node2
  String host2 = "host2";
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm2 = 
    registerNode(host2, 1234, 2345, NetworkTopology.DEFAULT_RACK, 
        Resources.createResource(memory/2, vcores/2));

  // Submit an application
  Application application = new Application("user1", resourceManager);
  application.submit();
  
  application.addNodeManager(host1, 1234, nm1);
  application.addNodeManager(host2, 1234, nm2);
  
  // Application resource requirements
  final int memory1 = 1024;
  Resource capability1 = Resources.createResource(memory1, 1);
  Priority priority1 = 
    org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.create(1);
  application.addResourceRequestSpec(priority1, capability1);
  
  Task t1 = new Task(application, priority1, new String[] {host1, host2});
  application.addTask(t1);
  
  final int memory2 = 2048;
  Resource capability2 = Resources.createResource(memory2, 1);
  Priority priority0 = 
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.create(0); // higher
  application.addResourceRequestSpec(priority0, capability2);
  
  // Send resource requests to the scheduler
  application.schedule();

 // Send a heartbeat to kick the tires on the Scheduler
  nodeUpdate(nm1);
  
  // Get allocations from the scheduler
  application.schedule();
  
  checkResourceUsage(nm1, nm2);
  
  LOG.info("Adding new tasks...");
  
  Task t2 = new Task(application, priority1, new String[] {host1, host2});
  application.addTask(t2);

  Task t3 = new Task(application, priority0, new String[] {ResourceRequest.ANY});
  application.addTask(t3);

  // Send resource requests to the scheduler
  application.schedule();
  checkResourceUsage(nm1, nm2);
  
  // Send heartbeats to kick the tires on the Scheduler
  nodeUpdate(nm2);
  nodeUpdate(nm2);
  nodeUpdate(nm1);
  nodeUpdate(nm1);
  
  // Get allocations from the scheduler
  LOG.info("Trying to allocate...");
  application.schedule();

  checkResourceUsage(nm1, nm2);
  
  // Complete tasks
  LOG.info("Finishing up tasks...");
  application.finishTask(t1);
  application.finishTask(t2);
  application.finishTask(t3);
  
  // Notify scheduler application is finished.
  AppAttemptRemovedSchedulerEvent appRemovedEvent1 =
      new AppAttemptRemovedSchedulerEvent(
        application.getApplicationAttemptId(), RMAppAttemptState.FINISHED, false);
  resourceManager.getResourceScheduler().handle(appRemovedEvent1);
  
  checkResourceUsage(nm1, nm2);
  
  LOG.info("--- END: testResourceAllocation ---");
}