Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNoMoreCpuOnNode() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  
  ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default",
      "user1", 2);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);
  scheduler.update();

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);
  assertEquals(1, app.getLiveContainers().size());
  scheduler.handle(updateEvent);
  assertEquals(1, app.getLiveContainers().size());
}
 
Example 2
Source Project: big-c   Source File: TestFairSchedulerPreemption.java    License: Apache License 2.0 6 votes vote down vote up
private void registerNodeAndSubmitApp(
    int memory, int vcores, int appContainers, int appMemory) {
  RMNode node1 = MockNodes.newNodeInfo(
      1, Resources.createResource(memory, vcores), 1, "node1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  assertEquals("Incorrect amount of resources in the cluster",
      memory, scheduler.rootMetrics.getAvailableMB());
  assertEquals("Incorrect amount of resources in the cluster",
      vcores, scheduler.rootMetrics.getAvailableVirtualCores());

  createSchedulingRequest(appMemory, "queueA", "user1", appContainers);
  scheduler.update();
  // Sufficient node check-ins to fully schedule containers
  for (int i = 0; i < 3; i++) {
    NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
    scheduler.handle(nodeUpdate1);
  }
  assertEquals("app1's request is not met",
      memory - appContainers * appMemory,
      scheduler.rootMetrics.getAvailableMB());
}
 
Example 3
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testNoMoreCpuOnNode() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1, 1),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  
  ApplicationAttemptId attId = createSchedulingRequest(1024, 1, 1, "default",
      "user1", 2);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);
  scheduler.update();

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(updateEvent);
  assertEquals(1, app.getLiveContainers().size());
  scheduler.handle(updateEvent);
  assertEquals(1, app.getLiveContainers().size());
}
 
Example 4
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAggregateCapacityTracking() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add a node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(1024), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  assertEquals(1024, scheduler.getClusterResource().getMemory());

  // Add another node
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(512), 2, "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  assertEquals(1536, scheduler.getClusterResource().getMemory());

  // Remove the first node
  NodeRemovedSchedulerEvent nodeEvent3 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(nodeEvent3);
  assertEquals(512, scheduler.getClusterResource().getMemory());
}
 
Example 5
Source Project: hadoop   Source File: TestFairSchedulerPreemption.java    License: Apache License 2.0 6 votes vote down vote up
private void registerNodeAndSubmitApp(
    int memory, int vcores, int gcores, int appContainers, int appMemory) {
  RMNode node1 = MockNodes.newNodeInfo(
      1, Resources.createResource(memory, vcores, gcores), 1, "node1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  assertEquals("Incorrect amount of resources in the cluster",
      memory, scheduler.rootMetrics.getAvailableMB());
  assertEquals("Incorrect amount of resources in the cluster",
      vcores, scheduler.rootMetrics.getAvailableVirtualCores());

  createSchedulingRequest(appMemory, "queueA", "user1", appContainers);
  scheduler.update();
  // Sufficient node check-ins to fully schedule containers
  for (int i = 0; i < 3; i++) {
    NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
    scheduler.handle(nodeUpdate1);
  }
  assertEquals("app1's request is not met",
      memory - appContainers * appMemory,
      scheduler.rootMetrics.getAvailableMB());
}
 
Example 6
Source Project: hadoop   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 
Example 7
Source Project: big-c   Source File: RMNodeImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public NodeState transition(RMNodeImpl rmNode, RMNodeEvent event) {
  RMNodeStatusEvent statusEvent = (RMNodeStatusEvent) event;

  // Switch the last heartbeatresponse.
  rmNode.latestNodeHeartBeatResponse = statusEvent.getLatestResponse();
  NodeHealthStatus remoteNodeHealthStatus = statusEvent.getNodeHealthStatus();
  rmNode.setHealthReport(remoteNodeHealthStatus.getHealthReport());
  rmNode.setLastHealthReportTime(
      remoteNodeHealthStatus.getLastHealthReportTime());
  if (remoteNodeHealthStatus.getIsNodeHealthy()) {
    rmNode.context.getDispatcher().getEventHandler().handle(
        new NodeAddedSchedulerEvent(rmNode));
    rmNode.context.getDispatcher().getEventHandler().handle(
            new NodesListManagerEvent(
                NodesListManagerEventType.NODE_USABLE, rmNode));
    // ??? how about updating metrics before notifying to ensure that
    // notifiers get update metadata because they will very likely query it
    // upon notification
    // Update metrics
    rmNode.updateMetricsForRejoinedNode(NodeState.UNHEALTHY);
    return NodeState.RUNNING;
  }

  return NodeState.UNHEALTHY;
}
 
Example 8
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAsyncScheduling() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();

  final int NODES = 100;
  
  // Register nodes
  for (int i=0; i < NODES; ++i) {
    String host = "192.168.1." + i;
    RMNode node =
        MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
    cs.handle(new NodeAddedSchedulerEvent(node));
  }
  
  // Now directly exercise the scheduling loop
  for (int i=0; i < NODES; ++i) {
    CapacityScheduler.schedule(cs);
  }
}
 
Example 9
Source Project: big-c   Source File: ReservationSystemTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static FairScheduler setupFairScheduler(
    ReservationSystemTestUtil testUtil,
    RMContext rmContext, Configuration conf, int numContainers) throws
    IOException {
  FairScheduler scheduler = new FairScheduler();
  scheduler.setRMContext(rmContext);

  when(rmContext.getScheduler()).thenReturn(scheduler);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rmContext);


  Resource resource = testUtil.calculateClusterResource(numContainers);
  RMNode node1 = MockNodes.newNodeInfo(1, resource, 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  return scheduler;
}
 
Example 10
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test (expected = YarnException.class)
public void testMoveWouldViolateMaxResourcesConstraints() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  QueueManager queueMgr = scheduler.getQueueManager();
  FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
  queueMgr.getLeafQueue("queue2", true);
  scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2",
      Resource.newInstance(1024, 1));

  ApplicationAttemptId appAttId =
      createSchedulingRequest(1024, 1, "queue1", "user1", 3);
  RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);
  scheduler.handle(updateEvent);
  scheduler.handle(updateEvent);
  
  assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
  scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}
 
Example 11
Source Project: hadoop   Source File: RMNodeImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
  // Inform the scheduler
  RMNodeStartedEvent startEvent = (RMNodeStartedEvent) event;
  List<NMContainerStatus> containers = null;

  String host = rmNode.nodeId.getHost();
  if (rmNode.context.getInactiveRMNodes().containsKey(host)) {
    // Old node rejoining
    RMNode previouRMNode = rmNode.context.getInactiveRMNodes().get(host);
    rmNode.context.getInactiveRMNodes().remove(host);
    rmNode.updateMetricsForRejoinedNode(previouRMNode.getState());
  } else {
    // Increment activeNodes explicitly because this is a new node.
    ClusterMetrics.getMetrics().incrNumActiveNodes();
    containers = startEvent.getNMContainerStatuses();
    if (containers != null && !containers.isEmpty()) {
      for (NMContainerStatus container : containers) {
        if (container.getContainerState() == ContainerState.RUNNING) {
          rmNode.launchedContainers.add(container.getContainerId());
        }
      }
    }
  }
  
  if (null != startEvent.getRunningApplications()) {
    for (ApplicationId appId : startEvent.getRunningApplications()) {
      handleRunningAppOnNode(rmNode, rmNode.context, appId, rmNode.nodeId);
    }
  }

  rmNode.context.getDispatcher().getEventHandler()
    .handle(new NodeAddedSchedulerEvent(rmNode, containers));
  rmNode.context.getDispatcher().getEventHandler().handle(
    new NodesListManagerEvent(
        NodesListManagerEventType.NODE_USABLE, rmNode));
}
 
Example 12
Source Project: hadoop   Source File: TestResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
    registerNode(String hostName, int containerManagerPort, int httpPort,
        String rackName, Resource capability) throws IOException,
        YarnException {
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm = 
      new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
          hostName, containerManagerPort, httpPort, rackName, capability,
          resourceManager);
  NodeAddedSchedulerEvent nodeAddEvent1 = 
      new NodeAddedSchedulerEvent(resourceManager.getRMContext()
          .getRMNodes().get(nm.getNodeId()));
  resourceManager.getResourceScheduler().handle(nodeAddEvent1);
  return nm;
}
 
Example 13
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 50000)
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  conf.setQueues("default", new String[] {"default"});
  conf.setCapacity("default", 100);
  FifoScheduler fs = new FifoScheduler();
  fs.init(conf);
  fs.start();
  // mock rmContext to avoid NPE.
  RMContext context = mock(RMContext.class);
  fs.reinitialize(conf, null);
  fs.setRMContext(context);

  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  RMNode n2 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3");

  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n2));
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());

  // reconnect n1 with downgraded memory
  n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2");
  fs.handle(new NodeRemovedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeUpdateSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
  fs.stop();
}
 
Example 14
Source Project: big-c   Source File: TestContinuousScheduling.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testSchedulingDelay() throws InterruptedException {
  // Add one node
  String host = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(
      1, Resources.createResource(4096, 4), 1, host);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(nodeUpdateEvent);

  // Create one application and submit one each of node-local, rack-local
  // and ANY requests
  ApplicationAttemptId appAttemptId =
      createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
  createMockRMApp(appAttemptId);

  scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false);
  scheduler.addApplicationAttempt(appAttemptId, false, false);
  List<ResourceRequest> ask = new ArrayList<>();
  ask.add(createResourceRequest(1024, 1, ResourceRequest.ANY, 1, 1, true));
  scheduler.allocate(
      appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
  FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);

  // Advance time and let continuous scheduling kick in
  mockClock.tick(1);
  while (1024 != app.getCurrentConsumption().getMemory()) {
    Thread.sleep(100);
  }
  assertEquals(1024, app.getCurrentConsumption().getMemory());
}
 
Example 15
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
    registerNode(String hostName, int containerManagerPort, int httpPort,
        String rackName, Resource capability)
        throws IOException, YarnException {
  org.apache.hadoop.yarn.server.resourcemanager.NodeManager nm =
      new org.apache.hadoop.yarn.server.resourcemanager.NodeManager(
          hostName, containerManagerPort, httpPort, rackName, capability,
          resourceManager);
  NodeAddedSchedulerEvent nodeAddEvent1 = 
      new NodeAddedSchedulerEvent(resourceManager.getRMContext()
          .getRMNodes().get(nm.getNodeId()));
  resourceManager.getResourceScheduler().handle(nodeAddEvent1);
  return nm;
}
 
Example 16
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleFairShareCalculation() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add one big node (only care about aggregate capacity)
  RMNode node1 =
      MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Have two queues which want entire cluster capacity
  createSchedulingRequest(10 * 1024, "queue1", "user1");
  createSchedulingRequest(10 * 1024, "queue2", "user1");
  createSchedulingRequest(10 * 1024, "root.default", "user1");

  scheduler.update();
  scheduler.getQueueManager().getRootQueue()
      .setSteadyFairShare(scheduler.getClusterResource());
  scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();

  Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
  assertEquals(3, queues.size());
  
  // Divided three ways - between the two queues and the default queue
  for (FSLeafQueue p : queues) {
    assertEquals(3414, p.getFairShare().getMemory());
    assertEquals(3414, p.getMetrics().getFairShareMB());
    assertEquals(3414, p.getSteadyFairShare().getMemory());
    assertEquals(3414, p.getMetrics().getSteadyFairShareMB());
  }
}
 
Example 17
Source Project: hadoop   Source File: TestFifoScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testBlackListNodes() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  String host = "127.0.0.1";
  RMNode node =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, host);
  fs.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);

  createMockRMApp(appAttemptId, rm.getRMContext());

  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId, "default",
        "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId, false);
  fs.handle(attemptEvent);

  // Verify the blacklist can be updated independent of requesting containers
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(),
      Collections.singletonList(host), null);
  Assert.assertTrue(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  fs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(),
      Collections.<ContainerId>emptyList(), null,
      Collections.singletonList(host));
  Assert.assertFalse(fs.getApplicationAttempt(appAttemptId).isBlacklisted(host));
  rm.stop();
}
 
Example 18
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
  public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());

    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
	assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4), 1,
        "127.0.0.1");
    NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(addEvent);
    
    assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    scheduler.update(); // update shouldn't change things
    assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    
    NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1);
    scheduler.handle(removeEvent);
    
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    scheduler.update(); // update shouldn't change things
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
}
 
Example 19
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSimpleFairShareCalculation() throws IOException {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add one big node (only care about aggregate capacity)
  RMNode node1 =
      MockNodes.newNodeInfo(1, Resources.createResource(10 * 1024), 1,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // Have two queues which want entire cluster capacity
  createSchedulingRequest(10 * 1024, "queue1", "user1");
  createSchedulingRequest(10 * 1024, "queue2", "user1");
  createSchedulingRequest(10 * 1024, "root.default", "user1");

  scheduler.update();
  scheduler.getQueueManager().getRootQueue()
      .setSteadyFairShare(scheduler.getClusterResource());
  scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();

  Collection<FSLeafQueue> queues = scheduler.getQueueManager().getLeafQueues();
  assertEquals(3, queues.size());
  
  // Divided three ways - between the two queues and the default queue
  for (FSLeafQueue p : queues) {
    assertEquals(3414, p.getFairShare().getMemory());
    assertEquals(3414, p.getMetrics().getFairShareMB());
    assertEquals(3414, p.getSteadyFairShare().getMemory());
    assertEquals(3414, p.getMetrics().getSteadyFairShareMB());
  }
}
 
Example 20
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testMoveRunnableApp() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  QueueManager queueMgr = scheduler.getQueueManager();
  FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
  FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);

  ApplicationAttemptId appAttId =
      createSchedulingRequest(1024, 1, "queue1", "user1", 3);
  ApplicationId appId = appAttId.getApplicationId();
  RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(1024));
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);
  scheduler.handle(updateEvent);
  
  assertEquals(Resource.newInstance(1024, 1), oldQueue.getResourceUsage());
  scheduler.update();
  assertEquals(Resource.newInstance(3072, 3), oldQueue.getDemand());
  
  scheduler.moveApplication(appId, "queue2");
  FSAppAttempt app = scheduler.getSchedulerApp(appAttId);
  assertSame(targetQueue, app.getQueue());
  assertFalse(oldQueue.isRunnableApp(app));
  assertTrue(targetQueue.isRunnableApp(app));
  assertEquals(Resource.newInstance(0, 0), oldQueue.getResourceUsage());
  assertEquals(Resource.newInstance(1024, 1), targetQueue.getResourceUsage());
  assertEquals(0, oldQueue.getNumRunnableApps());
  assertEquals(1, targetQueue.getNumRunnableApps());
  assertEquals(1, queueMgr.getRootQueue().getNumRunnableApps());
  
  scheduler.update();
  assertEquals(Resource.newInstance(0, 0), oldQueue.getDemand());
  assertEquals(Resource.newInstance(3072, 3), targetQueue.getDemand());
}
 
Example 21
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 3000)
public void testMaxAssign() throws Exception {
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node =
      MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId attId =
      createSchedulingRequest(1024, "root.default", "user", 8);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);

  // set maxAssign to 2: only 2 containers should be allocated
  scheduler.maxAssign = 2;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 2, app
      .getLiveContainers().size());

  // set maxAssign to -1: all remaining containers should be allocated
  scheduler.maxAssign = -1;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 8, app
      .getLiveContainers().size());
}
 
Example 22
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSteadyFairShareWithQueueCreatedRuntime() throws Exception {
  conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
      SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  // Add one node
  RMNode node1 =
      MockNodes
          .newNodeInfo(1, Resources.createResource(6144), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  assertEquals(6144, scheduler.getClusterResource().getMemory());
  assertEquals(6144, scheduler.getQueueManager().getRootQueue()
      .getSteadyFairShare().getMemory());
  assertEquals(6144, scheduler.getQueueManager()
      .getLeafQueue("default", false).getSteadyFairShare().getMemory());

  // Submit one application
  ApplicationAttemptId appAttemptId1 = createAppAttemptId(1, 1);
  createApplicationWithAMResource(appAttemptId1, "default", "user1", null);
  assertEquals(3072, scheduler.getQueueManager()
      .getLeafQueue("default", false).getSteadyFairShare().getMemory());
  assertEquals(3072, scheduler.getQueueManager()
      .getLeafQueue("user1", false).getSteadyFairShare().getMemory());
}
 
Example 23
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 3000)
public void testMaxAssignWithZeroMemoryContainers() throws Exception {
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node =
      MockNodes.newNodeInfo(1, Resources.createResource(16384, 16), 0,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId attId =
      createSchedulingRequest(0, 1, "root.default", "user", 8);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);

  // set maxAssign to 2: only 2 containers should be allocated
  scheduler.maxAssign = 2;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 2, app
      .getLiveContainers().size());

  // set maxAssign to -1: all remaining containers should be allocated
  scheduler.maxAssign = -1;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 8, app
      .getLiveContainers().size());
}
 
Example 24
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 3000)
public void testMaxAssignWithZeroMemoryContainers() throws Exception {
  conf.setBoolean(FairSchedulerConfiguration.ASSIGN_MULTIPLE, true);
  conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 0);
  
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node =
      MockNodes.newNodeInfo(1, Resources.createResource(16384, 16, 16), 0,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId attId =
      createSchedulingRequest(0, 1, 1, "root.default", "user", 8);
  FSAppAttempt app = scheduler.getSchedulerApp(attId);

  // set maxAssign to 2: only 2 containers should be allocated
  scheduler.maxAssign = 2;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 2, app
      .getLiveContainers().size());

  // set maxAssign to -1: all remaining containers should be allocated
  scheduler.maxAssign = -1;
  scheduler.update();
  scheduler.handle(updateEvent);
  assertEquals("Incorrect number of containers allocated", 8, app
      .getLiveContainers().size());
}
 
Example 25
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
  public void testRemoveNodeUpdatesRootQueueMetrics() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());

    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
	assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(1024, 4, 4), 1,
        "127.0.0.1");
    NodeAddedSchedulerEvent addEvent = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(addEvent);
    
    assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    scheduler.update(); // update shouldn't change things
    assertEquals(1024, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(4, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    
    NodeRemovedSchedulerEvent removeEvent = new NodeRemovedSchedulerEvent(node1);
    scheduler.handle(removeEvent);
    
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
    scheduler.update(); // update shouldn't change things
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableMB());
    assertEquals(0, scheduler.getRootQueueMetrics().getAvailableVirtualCores());
}
 
Example 26
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Two apps on one queue, one app on another
 */
@Test
public void testBasicDRFWithQueues() throws Exception {
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1, BuilderUtils.newResource(8192, 7, 7),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
  scheduler.handle(nodeEvent);

  ApplicationAttemptId appAttId1 = createSchedulingRequest(3072, 1, 1, "queue1",
      "user1", 2);
  FSAppAttempt app1 = scheduler.getSchedulerApp(appAttId1);
  ApplicationAttemptId appAttId2 = createSchedulingRequest(2048, 2, 2, "queue1",
      "user1", 2);
  FSAppAttempt app2 = scheduler.getSchedulerApp(appAttId2);
  ApplicationAttemptId appAttId3 = createSchedulingRequest(1024, 2, 2, "queue2",
      "user1", 2);
  FSAppAttempt app3 = scheduler.getSchedulerApp(appAttId3);
  
  DominantResourceFairnessPolicy drfPolicy = new DominantResourceFairnessPolicy();
  drfPolicy.initialize(scheduler.getClusterResource());
  scheduler.getQueueManager().getQueue("root").setPolicy(drfPolicy);
  scheduler.getQueueManager().getQueue("queue1").setPolicy(drfPolicy);
  scheduler.update();

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app1.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app3.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(2, app3.getLiveContainers().size());
  scheduler.handle(updateEvent);
  Assert.assertEquals(1, app2.getLiveContainers().size());
}
 
Example 27
Source Project: hadoop   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContinuousSchedulingWithNodeRemoved() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling once manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  // Add two nodes
  RMNode node1 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8, 8), 1,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8, 8), 2,
          "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  Assert.assertEquals("We should have two alive nodes.",
      2, scheduler.getNumClusterNodes());

  // Remove one node
  NodeRemovedSchedulerEvent removeNode1 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);
  Assert.assertEquals("We should only have one alive node.",
      1, scheduler.getNumClusterNodes());

  // Invoke the continuous scheduling once
  try {
    scheduler.continuousSchedulingAttempt();
  } catch (Exception e) {
    fail("Exception happened when doing continuous scheduling. " +
      e.toString());
  }
}
 
Example 28
Source Project: hadoop   Source File: TestContinuousScheduling.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testSchedulingDelay() throws InterruptedException {
  // Add one node
  String host = "127.0.0.1";
  RMNode node1 = MockNodes.newNodeInfo(
      1, Resources.createResource(4096, 4, 4), 1, host);
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
  scheduler.handle(nodeUpdateEvent);

  // Create one application and submit one each of node-local, rack-local
  // and ANY requests
  ApplicationAttemptId appAttemptId =
      createAppAttemptId(this.APP_ID++, this.ATTEMPT_ID++);
  createMockRMApp(appAttemptId);

  scheduler.addApplication(appAttemptId.getApplicationId(), "queue11", "user11", false);
  scheduler.addApplicationAttempt(appAttemptId, false, false);
  List<ResourceRequest> ask = new ArrayList<>();
  ask.add(createResourceRequest(1024, 1, 1, ResourceRequest.ANY, 1, 1, true));
  scheduler.allocate(
      appAttemptId, ask, new ArrayList<ContainerId>(), null, null);
  FSAppAttempt app = scheduler.getSchedulerApp(appAttemptId);

  // Advance time and let continuous scheduling kick in
  mockClock.tick(1);
  while (1024 != app.getCurrentConsumption().getMemory()) {
    Thread.sleep(100);
  }
  assertEquals(1024, app.getCurrentConsumption().getMemory());
}
 
Example 29
Source Project: hadoop   Source File: TestFairSchedulerFairShare.java    License: Apache License 2.0 5 votes vote down vote up
private void createClusterWithQueuesAndOneNode(int mem, int vCores, int gCores,
    String policy) throws IOException {
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"root\" >");
  out.println("   <queue name=\"parentA\" >");
  out.println("       <weight>8</weight>");
  out.println("       <queue name=\"childA1\" />");
  out.println("       <queue name=\"childA2\" />");
  out.println("       <queue name=\"childA3\" />");
  out.println("       <queue name=\"childA4\" />");
  out.println("   </queue>");
  out.println("   <queue name=\"parentB\" >");
  out.println("       <weight>1</weight>");
  out.println("       <queue name=\"childB1\" />");
  out.println("       <queue name=\"childB2\" />");
  out.println("   </queue>");
  out.println("</queue>");
  out.println("<defaultQueueSchedulingPolicy>" + policy
      + "</defaultQueueSchedulingPolicy>");
  out.println("</allocations>");
  out.close();

  resourceManager = new MockRM(conf);
  resourceManager.start();
  scheduler = (FairScheduler) resourceManager.getResourceScheduler();

  RMNode node1 = MockNodes.newNodeInfo(1,
      Resources.createResource(mem, vCores, gCores), 1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
}
 
Example 30
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testContinuousSchedulingWithNodeRemoved() throws Exception {
  // Disable continuous scheduling, will invoke continuous scheduling once manually
  scheduler.init(conf);
  scheduler.start();
  Assert.assertTrue("Continuous scheduling should be disabled.",
      !scheduler.isContinuousSchedulingEnabled());

  // Add two nodes
  RMNode node1 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1,
          "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);
  RMNode node2 =
      MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 2,
          "127.0.0.2");
  NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
  scheduler.handle(nodeEvent2);
  Assert.assertEquals("We should have two alive nodes.",
      2, scheduler.getNumClusterNodes());

  // Remove one node
  NodeRemovedSchedulerEvent removeNode1 = new NodeRemovedSchedulerEvent(node1);
  scheduler.handle(removeNode1);
  Assert.assertEquals("We should only have one alive node.",
      1, scheduler.getNumClusterNodes());

  // Invoke the continuous scheduling once
  try {
    scheduler.continuousSchedulingAttempt();
  } catch (Exception e) {
    fail("Exception happened when doing continuous scheduling. " +
      e.toString());
  }
}