Java Code Examples for org.apache.hadoop.yarn.util.resource.Resources#createResource()

The following examples show how to use org.apache.hadoop.yarn.util.resource.Resources#createResource() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FSParentQueue.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void updateDemand() {
  // Compute demand by iterating through apps in the queue
  // Limit demand to maxResources
  Resource maxRes = scheduler.getAllocationConfiguration()
      .getMaxResources(getName());
  demand = Resources.createResource(0);
  for (FSQueue childQueue : childQueues) {
    childQueue.updateDemand();
    Resource toAdd = childQueue.getDemand();
    if (LOG.isDebugEnabled()) {
      LOG.debug("Counting resource from " + childQueue.getName() + " " + 
          toAdd + "; Total resource consumption for " + getName() +
          " now " + demand);
    }
    demand = Resources.add(demand, toAdd);
    demand = Resources.componentwiseMin(demand, maxRes);
    if (Resources.equals(demand, maxRes)) {
      break;
    }
  }
  if (LOG.isDebugEnabled()) {
    LOG.debug("The updated demand for " + getName() + " is " + demand +
        "; the max is " + maxRes);
  }    
}
 
Example 2
Source File: FSAppAttempt.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Override
public void updateDemand() {
  demand = Resources.createResource(0);
  // Demand is current consumption plus outstanding requests
  Resources.addTo(demand, getCurrentConsumption());

  // Add up outstanding resource requests
  synchronized (this) {
    for (Priority p : getPriorities()) {
      for (ResourceRequest r : getResourceRequests(p).values()) {
        Resource total = Resources.multiply(r.getCapability(), r.getNumContainers());
        Resources.addTo(demand, total);
      }
    }
  }
}
 
Example 3
Source File: FairSchedulerConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Resource getMinimumAllocation() {
  int mem = getInt(
      YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
  int cpu = getInt(
      YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
  return Resources.createResource(mem, cpu);
}
 
Example 4
Source File: TestLeafQueue.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testMaxAMResourcePerQueuePercentAfterQueueRefresh()
    throws Exception {
  CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
  Resource clusterResource = Resources
      .createResource(100 * 16 * GB, 100 * 32);
  CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource);
  when(csContext.getRMContext()).thenReturn(rmContext);
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f);
  ParentQueue root = new ParentQueue(csContext, 
      CapacitySchedulerConfiguration.ROOT, null, null);
  csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
  LeafQueue a = new LeafQueue(csContext, A, root, null);
  assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
  
  csConf.setFloat(CapacitySchedulerConfiguration.
      MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
  LeafQueue newA = new LeafQueue(csContext, A, root, null);
  a.reinitialize(newA, clusterResource);
  assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
  assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));

  Resource newClusterResource = Resources.createResource(100 * 20 * GB,
      100 * 32);
  a.updateClusterResource(newClusterResource, 
      new ResourceLimits(newClusterResource));
  //  100 * 20 * 0.2 = 400
  assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
}
 
Example 5
Source File: TestUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ResourceRequest createResourceRequest(
    String resourceName, int memory, int numContainers, boolean relaxLocality,
    Priority priority, RecordFactory recordFactory) {
  ResourceRequest request = 
      recordFactory.newRecordInstance(ResourceRequest.class);
  Resource capability = Resources.createResource(memory, 1);
  
  request.setNumContainers(numContainers);
  request.setResourceName(resourceName);
  request.setCapability(capability);
  request.setRelaxLocality(relaxLocality);
  request.setPriority(priority);
  return request;
}
 
Example 6
Source File: TestDagAwareYarnTaskScheduler.java    From tez with Apache License 2.0 5 votes vote down vote up
@Override
public RegisterApplicationMasterResponse registerApplicationMaster(String appHostName, int appHostPort,
    String appTrackingUrl) {
  mockRegResponse = mock(RegisterApplicationMasterResponse.class);
  Resource mockMaxResource = Resources.createResource(1024*1024, 1024);
  Map<ApplicationAccessType, String> mockAcls = Collections.emptyMap();
  when(mockRegResponse.getMaximumResourceCapability()).thenReturn(
      mockMaxResource);
  when(mockRegResponse.getApplicationACLs()).thenReturn(mockAcls);
  when(mockRegResponse.getSchedulerResourceTypes()).thenReturn(
      EnumSet.of(SchedulerResourceTypes.MEMORY, SchedulerResourceTypes.CPU));
  return mockRegResponse;
}
 
Example 7
Source File: FairSchedulerConfiguration.java    From big-c with Apache License 2.0 5 votes vote down vote up
public Resource getIncrementAllocation() {
  int incrementMemory = getInt(
    RM_SCHEDULER_INCREMENT_ALLOCATION_MB,
    DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_MB);
  int incrementCores = getInt(
    RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES,
    DEFAULT_RM_SCHEDULER_INCREMENT_ALLOCATION_VCORES);
  return Resources.createResource(incrementMemory, incrementCores);
}
 
Example 8
Source File: TestRMWebServicesCapacitySched.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testResourceInfo() {
  Resource res = Resources.createResource(10, 1, 1);
  // If we add a new resource (e.g disks), then
  // CapacitySchedulerPage and these RM WebServices + docs need to be updated
  // eg. ResourceInfo
  assertEquals("<memory:10, vCores:1, gCores:1>", res.toString());
}
 
Example 9
Source File: TestDominantResourceFairnessPolicy.java    From big-c with Apache License 2.0 5 votes vote down vote up
private Schedulable createSchedulable(int memUsage, int cpuUsage,
    ResourceWeights weights, int minMemShare, int minCpuShare) {
  Resource usage = BuilderUtils.newResource(memUsage, cpuUsage);
  Resource minShare = BuilderUtils.newResource(minMemShare, minCpuShare);
  return new FakeSchedulable(minShare,
      Resources.createResource(Integer.MAX_VALUE, Integer.MAX_VALUE),
      weights, Resources.none(), usage, 0l);
}
 
Example 10
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testAllocateContainerOnNodeWithoutOffSwitchSpecified()
    throws Exception {
  // Manipulate queue 'a'
  LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B));

  // Users
  final String user_0 = "user_0";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 =
      TestUtils.getMockApplicationAttemptId(0, 0);
  FiCaSchedulerApp app_0 =
      new FiCaSchedulerApp(appAttemptId_0, user_0, a,
          mock(ActiveUsersManager.class), spyRMContext);
  a.submitApplicationAttempt(app_0, user_0);

  final ApplicationAttemptId appAttemptId_1 =
      TestUtils.getMockApplicationAttemptId(1, 0);
  FiCaSchedulerApp app_1 =
      new FiCaSchedulerApp(appAttemptId_1, user_0, a,
          mock(ActiveUsersManager.class), spyRMContext);
  a.submitApplicationAttempt(app_1, user_0); // same user

  // Setup some nodes
  String host_0 = "127.0.0.1";
  FiCaSchedulerNode node_0 =
      TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, 8 * GB);

  final int numNodes = 1;
  Resource clusterResource =
      Resources.createResource(numNodes * (8 * GB), numNodes * 16);
  when(csContext.getNumClusterNodes()).thenReturn(numNodes);

  // Setup resource-requests
  Priority priority = TestUtils.createMockPriority(1);
  app_0.updateResourceRequests(Arrays.asList(TestUtils.createResourceRequest(
      "127.0.0.1", 1 * GB, 3, true, priority, recordFactory), TestUtils
      .createResourceRequest(DEFAULT_RACK, 1 * GB, 3, true, priority,
          recordFactory)));

  try {
    a.assignContainers(clusterResource, node_0, 
        new ResourceLimits(clusterResource));
  } catch (NullPointerException e) {
    Assert.fail("NPE when allocating container on node but "
        + "forget to set off-switch request should be handled");
  }
}
 
Example 11
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testNormalizeRequest() {
  ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
  
  final int minMemory = 1024;
  final int maxMemory = 8192;
  Resource minResource = Resources.createResource(minMemory, 0);
  Resource maxResource = Resources.createResource(maxMemory, 0);
  
  ResourceRequest ask = new ResourceRequestPBImpl();

  // case negative memory
  ask.setCapability(Resources.createResource(-1024));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(minMemory, ask.getCapability().getMemory());

  // case zero memory
  ask.setCapability(Resources.createResource(0));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(minMemory, ask.getCapability().getMemory());

  // case memory is a multiple of minMemory
  ask.setCapability(Resources.createResource(2 * minMemory));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(2 * minMemory, ask.getCapability().getMemory());

  // case memory is not a multiple of minMemory
  ask.setCapability(Resources.createResource(minMemory + 10));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(2 * minMemory, ask.getCapability().getMemory());

  // case memory is equal to max allowed
  ask.setCapability(Resources.createResource(maxMemory));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxMemory, ask.getCapability().getMemory());

  // case memory is just less than max
  ask.setCapability(Resources.createResource(maxMemory - 10));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxMemory, ask.getCapability().getMemory());

  // max is not a multiple of min
  maxResource = Resources.createResource(maxMemory - 10, 0);
  ask.setCapability(Resources.createResource(maxMemory - 100));
  // multiple of minMemory > maxMemory, then reduce to maxMemory
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());

  // ask is more than max
  maxResource = Resources.createResource(maxMemory, 0);
  ask.setCapability(Resources.createResource(maxMemory + 100));
  SchedulerUtils.normalizeRequest(ask, resourceCalculator, null, minResource,
      maxResource);
  assertEquals(maxResource.getMemory(), ask.getCapability().getMemory());
}
 
Example 12
Source File: FakeSchedulable.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public FakeSchedulable(int minShare, int maxShare, double weight, int fairShare, int usage,
    long startTime) {
  this(Resources.createResource(minShare, 0, 0), Resources.createResource(maxShare, 0, 0),
      new ResourceWeights((float)weight), Resources.createResource(fairShare, 0, 0),
      Resources.createResource(usage, 0, 0), startTime);
}
 
Example 13
Source File: ParentQueue.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public synchronized CSAssignment assignContainers(Resource clusterResource,
    FiCaSchedulerNode node, ResourceLimits resourceLimits) {
  CSAssignment assignment = 
      new CSAssignment(Resources.createResource(0, 0, 0), NodeType.NODE_LOCAL);
  Set<String> nodeLabels = node.getLabels();
  
  // if our queue cannot access this node, just return
  if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
    return assignment;
  }
  
  while (canAssign(clusterResource, node)) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Trying to assign containers to child-queue of "
        + getQueueName());
    }
    
    // Are we over maximum-capacity for this queue?
    // This will also consider parent's limits and also continuous reservation
    // looking
    if (!super.canAssignToThisQueue(clusterResource, nodeLabels, resourceLimits,
        minimumAllocation, Resources.createResource(getMetrics()
            .getReservedMB(), getMetrics().getReservedVirtualCores(), getMetrics().getReservedGpuCores()))) {
      break;
    }
    
    // Schedule
    CSAssignment assignedToChild = 
        assignContainersToChildQueues(clusterResource, node, resourceLimits);
    assignment.setType(assignedToChild.getType());
    
    // Done if no child-queue assigned anything
    if (Resources.greaterThan(
            resourceCalculator, clusterResource, 
            assignedToChild.getResource(), Resources.none())) {
      // Track resource utilization for the parent-queue
      super.allocateResource(clusterResource, assignedToChild.getResource(),
          nodeLabels);
      
      // Track resource utilization in this pass of the scheduler
      Resources.addTo(assignment.getResource(), assignedToChild.getResource());
      
      LOG.info("assignedContainer" +
          " queue=" + getQueueName() + 
          " usedCapacity=" + getUsedCapacity() +
          " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
          " used=" + queueUsage.getUsed() + 
          " cluster=" + clusterResource);

    } else {
      break;
    }

    if (LOG.isDebugEnabled()) {
      LOG.debug("ParentQ=" + getQueueName()
        + " assignedSoFarInThisIteration=" + assignment.getResource()
        + " usedCapacity=" + getUsedCapacity()
        + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity());
    }

    // Do not assign more than one container if this isn't the root queue
    // or if we've already assigned an off-switch container
    if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) {
      if (LOG.isDebugEnabled()) {
        if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) {
          LOG.debug("Not assigning more than one off-switch container," +
              " assignments so far: " + assignment);
        }
      }
      break;
    }
  } 
  
  return assignment;
}
 
Example 14
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
 public void testUserHeadroomMultiApp() throws Exception {
   // Mock the queue
   LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
   //unset maxCapacity
   a.setMaxCapacity(1.0f);

   // Users
   final String user_0 = "user_0";
   final String user_1 = "user_1";

   // Submit applications
   final ApplicationAttemptId appAttemptId_0 =
       TestUtils.getMockApplicationAttemptId(0, 0);
   FiCaSchedulerApp app_0 =
       new FiCaSchedulerApp(appAttemptId_0, user_0, a,
           a.getActiveUsersManager(), spyRMContext);
   a.submitApplicationAttempt(app_0, user_0);

   final ApplicationAttemptId appAttemptId_1 =
       TestUtils.getMockApplicationAttemptId(1, 0);
   FiCaSchedulerApp app_1 =
       new FiCaSchedulerApp(appAttemptId_1, user_0, a,
           a.getActiveUsersManager(), spyRMContext);
   a.submitApplicationAttempt(app_1, user_0);  // same user

   final ApplicationAttemptId appAttemptId_2 =
       TestUtils.getMockApplicationAttemptId(2, 0);
   FiCaSchedulerApp app_2 =
       new FiCaSchedulerApp(appAttemptId_2, user_1, a,
           a.getActiveUsersManager(), spyRMContext);
   a.submitApplicationAttempt(app_2, user_1);

   // Setup some nodes
   String host_0 = "127.0.0.1";
   FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 
     0, 16*GB);
   String host_1 = "127.0.0.2";
   FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 
     0, 16*GB);

   final int numNodes = 2;
   Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1);
   when(csContext.getNumClusterNodes()).thenReturn(numNodes);

   Priority priority = TestUtils.createMockPriority(1);

   app_0.updateResourceRequests(Collections.singletonList(
           TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true,
               priority, recordFactory)));

   a.assignContainers(clusterResource, node_0,
       new ResourceLimits(clusterResource));
   assertEquals(1*GB, a.getUsedResources().getMemory());
   assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
   assertEquals(0*GB, app_1.getCurrentConsumption().getMemory());
   //Now, headroom is the same for all apps for a given user + queue combo
   //and a change to any app's headroom is reflected for all the user's apps
   //once those apps are active/have themselves calculated headroom for 
   //allocation at least one time
   assertEquals(2*GB, app_0.getHeadroom().getMemory());
   assertEquals(0*GB, app_1.getHeadroom().getMemory());//not yet active
   assertEquals(0*GB, app_2.getHeadroom().getMemory());//not yet active

   app_1.updateResourceRequests(Collections.singletonList(
       TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true,
           priority, recordFactory)));

   a.assignContainers(clusterResource, node_0,
       new ResourceLimits(clusterResource));
   assertEquals(2*GB, a.getUsedResources().getMemory());
   assertEquals(1*GB, app_0.getCurrentConsumption().getMemory());
   assertEquals(1*GB, app_1.getCurrentConsumption().getMemory());
   assertEquals(1*GB, app_0.getHeadroom().getMemory());
   assertEquals(1*GB, app_1.getHeadroom().getMemory());//now active
   assertEquals(0*GB, app_2.getHeadroom().getMemory());//not yet active

   //Complete container and verify that headroom is updated, for both apps 
   //for the user
   RMContainer rmContainer = app_0.getLiveContainers().iterator().next();
   a.completedContainer(clusterResource, app_0, node_0, rmContainer,
   ContainerStatus.newInstance(rmContainer.getContainerId(),
ContainerState.COMPLETE, "",
ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
   RMContainerEventType.KILL, null, true);

   assertEquals(2*GB, app_0.getHeadroom().getMemory());
   assertEquals(2*GB, app_1.getHeadroom().getMemory());
 }
 
Example 15
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testNormalizeNodeLabelExpression()
    throws IOException {
  // mock queue and scheduler
  YarnScheduler scheduler = mock(YarnScheduler.class);
  Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
  QueueInfo queueInfo = mock(QueueInfo.class);
  when(queueInfo.getQueueName()).thenReturn("queue");
  when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
  when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x ");
  when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean()))
      .thenReturn(queueInfo);
  
  Resource maxResource = Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_GCORES);

  // queue has labels, success cases
  try {
    // set queue accessible node labels to [x, y]
    queueAccessibleNodeLabels.clear();
    queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
    rmContext.getNodeLabelManager().addToCluserNodeLabels(
        ImmutableSet.of("x", "y"));
    Resource resource = Resources.createResource(
        0,
        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_GCORES);
    ResourceRequest resReq = BuilderUtils.newResourceRequest(
        mock(Priority.class), ResourceRequest.ANY, resource, 1);
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
        scheduler, rmContext);
    Assert.assertTrue(resReq.getNodeLabelExpression().equals("x"));
    
    resReq.setNodeLabelExpression(" y ");
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
        scheduler, rmContext);
    Assert.assertTrue(resReq.getNodeLabelExpression().equals("y"));
  } catch (InvalidResourceRequestException e) {
    e.printStackTrace();
    fail("Should be valid when request labels is a subset of queue labels");
  } finally {
    rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
        Arrays.asList("x", "y"));
  }
}
 
Example 16
Source File: FakeSchedulable.java    From big-c with Apache License 2.0 4 votes vote down vote up
public FakeSchedulable(int minShare, int maxShare, double weight, int fairShare, int usage,
    long startTime) {
  this(Resources.createResource(minShare, 0), Resources.createResource(maxShare, 0),
      new ResourceWeights((float)weight), Resources.createResource(fairShare, 0),
      Resources.createResource(usage, 0), startTime);
}
 
Example 17
Source File: TestCapacityScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(expected = YarnException.class)
public void testMoveAppViolateQueueState() throws Exception {
  resourceManager = new ResourceManager() {
     @Override
      protected RMNodeLabelsManager createNodeLabelManager() {
        RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
        mgr.init(getConfig());
        return mgr;
      }
  };
  CapacitySchedulerConfiguration csConf =
      new CapacitySchedulerConfiguration();
  setupQueueConfiguration(csConf);
  StringBuilder qState = new StringBuilder();
  qState.append(CapacitySchedulerConfiguration.PREFIX).append(B)
      .append(CapacitySchedulerConfiguration.DOT)
      .append(CapacitySchedulerConfiguration.STATE);
  csConf.set(qState.toString(), QueueState.STOPPED.name());
  YarnConfiguration conf = new YarnConfiguration(csConf);
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  resourceManager.init(conf);
  resourceManager.getRMContext().getContainerTokenSecretManager()
      .rollMasterKey();
  resourceManager.getRMContext().getNMTokenSecretManager().rollMasterKey();
  ((AsyncDispatcher) resourceManager.getRMContext().getDispatcher()).start();
  mockContext = mock(RMContext.class);
  when(mockContext.getConfigurationProvider()).thenReturn(
      new LocalConfigurationProvider());

  ResourceScheduler scheduler = resourceManager.getResourceScheduler();

  // Register node1
  String host_0 = "host_0";
  NodeManager nm_0 =
      registerNode(host_0, 1234, 2345, NetworkTopology.DEFAULT_RACK,
          Resources.createResource(6 * GB, 1));

  // ResourceRequest priorities
  Priority priority_0 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(0);
  Priority priority_1 =
      org.apache.hadoop.yarn.server.resourcemanager.resource.Priority
          .create(1);

  // Submit application_0
  Application application_0 =
      new Application("user_0", "a1", resourceManager);
  application_0.submit(); // app + app attempt event sent to scheduler

  application_0.addNodeManager(host_0, 1234, nm_0);

  Resource capability_0_0 = Resources.createResource(3 * GB, 1);
  application_0.addResourceRequestSpec(priority_1, capability_0_0);

  Resource capability_0_1 = Resources.createResource(2 * GB, 1);
  application_0.addResourceRequestSpec(priority_0, capability_0_1);

  Task task_0_0 =
      new Task(application_0, priority_1, new String[] { host_0 });
  application_0.addTask(task_0_0);

  // Send resource requests to the scheduler
  application_0.schedule(); // allocate

  // task_0_0 allocated
  nodeUpdate(nm_0);

  // Get allocations from the scheduler
  application_0.schedule(); // task_0_0
  checkApplicationResourceUsage(3 * GB, application_0);

  checkNodeResourceUsage(3 * GB, nm_0);
  // b2 queue contains 3GB consumption app,
  // add another 3GB will hit max capacity limit on queue b
  scheduler.moveApplication(application_0.getApplicationId(), "b1");

}
 
Example 18
Source File: TestLeafQueue.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testSingleQueueOneUserMetrics() throws Exception {

  // Manipulate queue 'a'
  LeafQueue a = stubLeafQueue((LeafQueue)queues.get(B));

  // Users
  final String user_0 = "user_0";

  // Submit applications
  final ApplicationAttemptId appAttemptId_0 = 
      TestUtils.getMockApplicationAttemptId(0, 0); 
  FiCaSchedulerApp app_0 = 
      new FiCaSchedulerApp(appAttemptId_0, user_0, a, 
          mock(ActiveUsersManager.class), spyRMContext);
  a.submitApplicationAttempt(app_0, user_0);

  final ApplicationAttemptId appAttemptId_1 = 
      TestUtils.getMockApplicationAttemptId(1, 0); 
  FiCaSchedulerApp app_1 = 
      new FiCaSchedulerApp(appAttemptId_1, user_0, a, 
          mock(ActiveUsersManager.class), spyRMContext);
  a.submitApplicationAttempt(app_1, user_0);  // same user

  
  // Setup some nodes
  String host_0 = "127.0.0.1";
  FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
      8*GB);

  final int numNodes = 1;
  Resource clusterResource = 
      Resources.createResource(numNodes * (8*GB), numNodes * 16);
  when(csContext.getNumClusterNodes()).thenReturn(numNodes);

  // Setup resource-requests
  Priority priority = TestUtils.createMockPriority(1);
  app_0.updateResourceRequests(Collections.singletonList(
          TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 3, true,
              priority, recordFactory)));

  // Start testing...
  
  // Only 1 container
  a.assignContainers(clusterResource, node_0, new ResourceLimits(
      clusterResource));
  assertEquals(
      (int)(node_0.getTotalResource().getMemory() * a.getCapacity()) - (1*GB),
      a.getMetrics().getAvailableMB());
}
 
Example 19
Source File: TestSchedulerUtils.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test (timeout = 30000)
public void testNormalizeNodeLabelExpression()
    throws IOException {
  // mock queue and scheduler
  YarnScheduler scheduler = mock(YarnScheduler.class);
  Set<String> queueAccessibleNodeLabels = Sets.newHashSet();
  QueueInfo queueInfo = mock(QueueInfo.class);
  when(queueInfo.getQueueName()).thenReturn("queue");
  when(queueInfo.getAccessibleNodeLabels()).thenReturn(queueAccessibleNodeLabels);
  when(queueInfo.getDefaultNodeLabelExpression()).thenReturn(" x ");
  when(scheduler.getQueueInfo(any(String.class), anyBoolean(), anyBoolean()))
      .thenReturn(queueInfo);
  
  Resource maxResource = Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES);

  // queue has labels, success cases
  try {
    // set queue accessible node labels to [x, y]
    queueAccessibleNodeLabels.clear();
    queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
    rmContext.getNodeLabelManager().addToCluserNodeLabels(
        ImmutableSet.of("x", "y"));
    Resource resource = Resources.createResource(
        0,
        YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
    ResourceRequest resReq = BuilderUtils.newResourceRequest(
        mock(Priority.class), ResourceRequest.ANY, resource, 1);
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
        scheduler, rmContext);
    Assert.assertTrue(resReq.getNodeLabelExpression().equals("x"));
    
    resReq.setNodeLabelExpression(" y ");
    SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
        scheduler, rmContext);
    Assert.assertTrue(resReq.getNodeLabelExpression().equals("y"));
  } catch (InvalidResourceRequestException e) {
    e.printStackTrace();
    fail("Should be valid when request labels is a subset of queue labels");
  } finally {
    rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
        Arrays.asList("x", "y"));
  }
}
 
Example 20
Source File: TestFifoScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test(timeout=2000)
public void testUpdateResourceOnNode() throws Exception {
  AsyncDispatcher dispatcher = new InlineDispatcher();
  Configuration conf = new Configuration();
  RMContainerTokenSecretManager containerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  containerTokenSecretManager.rollMasterKey();
  NMTokenSecretManagerInRM nmTokenSecretManager =
      new NMTokenSecretManagerInRM(conf);
  nmTokenSecretManager.rollMasterKey();
  RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
  
  FifoScheduler scheduler = new FifoScheduler(){
    @SuppressWarnings("unused")
    public Map<NodeId, FiCaSchedulerNode> getNodes(){
      return nodes;
    }
  };
  RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
      null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
      scheduler);
  ((RMContextImpl) rmContext).setSystemMetricsPublisher(
      mock(SystemMetricsPublisher.class));

  scheduler.setRMContext(rmContext);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(new Configuration(), rmContext);
  RMNode node0 = MockNodes.newNodeInfo(1,
      Resources.createResource(2048, 4), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
  scheduler.handle(nodeEvent1);
  
  Method method = scheduler.getClass().getDeclaredMethod("getNodes");
  @SuppressWarnings("unchecked")
  Map<NodeId, FiCaSchedulerNode> schedulerNodes = 
      (Map<NodeId, FiCaSchedulerNode>) method.invoke(scheduler);
  assertEquals(schedulerNodes.values().size(), 1);
  
  Resource newResource = Resources.createResource(1024, 4);
  
  NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new 
      NodeResourceUpdateSchedulerEvent(node0, ResourceOption.newInstance(
          newResource, RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT));
  scheduler.handle(node0ResourceUpdate);
  
  // SchedulerNode's total resource and available resource are changed.
  assertEquals(schedulerNodes.get(node0.getNodeID()).getTotalResource()
      .getMemory(), 1024);
  assertEquals(schedulerNodes.get(node0.getNodeID()).
      getAvailableResource().getMemory(), 1024);
  QueueInfo queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(0.0f, queueInfo.getCurrentCapacity(), 0.0f);
  
  int _appId = 1;
  int _appAttemptId = 1;
  ApplicationAttemptId appAttemptId = createAppAttemptId(_appId,
      _appAttemptId);
  createMockRMApp(appAttemptId, rmContext);

  AppAddedSchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appAttemptId.getApplicationId(), "queue1",
        "user1");
  scheduler.handle(appEvent);
  AppAttemptAddedSchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  scheduler.handle(attemptEvent);

  int memory = 1024;
  int priority = 1;

  List<ResourceRequest> ask = new ArrayList<ResourceRequest>();
  ResourceRequest nodeLocal = createResourceRequest(memory,
      node0.getHostName(), priority, 1);
  ResourceRequest rackLocal = createResourceRequest(memory,
      node0.getRackName(), priority, 1);
  ResourceRequest any = createResourceRequest(memory, ResourceRequest.ANY, priority,
      1);
  ask.add(nodeLocal);
  ask.add(rackLocal);
  ask.add(any);
  scheduler.allocate(appAttemptId, ask, new ArrayList<ContainerId>(), null, null);

  // Before the node update event, there are one local request
  Assert.assertEquals(1, nodeLocal.getNumContainers());

  NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
  // Now schedule.
  scheduler.handle(node0Update);

  // After the node update event, check no local request
  Assert.assertEquals(0, nodeLocal.getNumContainers());
  // Also check that one container was scheduled
  SchedulerAppReport info = scheduler.getSchedulerAppInfo(appAttemptId);
  Assert.assertEquals(1, info.getLiveContainers().size());
  // And check the default Queue now is full.
  queueInfo = scheduler.getQueueInfo(null, false, false);
  Assert.assertEquals(1.0f, queueInfo.getCurrentCapacity(), 0.0f);
}