Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler#handle()

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler#handle() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestFifoScheduler.java From hadoop with Apache License 2.0

5 votes

@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}

Example 2

Source File: TestFifoScheduler.java From hadoop with Apache License 2.0

5 votes

@Test (timeout = 50000)
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  conf.setQueues("default", new String[] {"default"});
  conf.setCapacity("default", 100);
  FifoScheduler fs = new FifoScheduler();
  fs.init(conf);
  fs.start();
  // mock rmContext to avoid NPE.
  RMContext context = mock(RMContext.class);
  fs.reinitialize(conf, null);
  fs.setRMContext(context);

  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  RMNode n2 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3");

  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n2));
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());

  // reconnect n1 with downgraded memory
  n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2");
  fs.handle(new NodeRemovedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeUpdateSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
  fs.stop();
}

Example 3

Source File: TestFifoScheduler.java From big-c with Apache License 2.0

5 votes

@Test
public void testNodeUpdateBeforeAppAttemptInit() throws Exception {
  FifoScheduler scheduler = new FifoScheduler();
  MockRM rm = new MockRM(conf);
  scheduler.setRMContext(rm.getRMContext());
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, rm.getRMContext());

  RMNode node = MockNodes.newNodeInfo(1,
          Resources.createResource(1024, 4), 1, "127.0.0.1");
  scheduler.handle(new NodeAddedSchedulerEvent(node));

  ApplicationId appId = ApplicationId.newInstance(0, 1);
  scheduler.addApplication(appId, "queue1", "user1", false);

  NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
  try {
    scheduler.handle(updateEvent);
  } catch (NullPointerException e) {
      Assert.fail();
  }

  ApplicationAttemptId attId = ApplicationAttemptId.newInstance(appId, 1);
  scheduler.addApplicationAttempt(attId, false, false);

  rm.stop();
}

Example 4

Source File: TestFifoScheduler.java From big-c with Apache License 2.0

5 votes

@Test (timeout = 50000)
public void testReconnectedNode() throws Exception {
  CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
  conf.setQueues("default", new String[] {"default"});
  conf.setCapacity("default", 100);
  FifoScheduler fs = new FifoScheduler();
  fs.init(conf);
  fs.start();
  // mock rmContext to avoid NPE.
  RMContext context = mock(RMContext.class);
  fs.reinitialize(conf, null);
  fs.setRMContext(context);

  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  RMNode n2 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2, "127.0.0.3");

  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n2));
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  Assert.assertEquals(6 * GB, fs.getRootQueueMetrics().getAvailableMB());

  // reconnect n1 with downgraded memory
  n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 1, "127.0.0.2");
  fs.handle(new NodeRemovedSchedulerEvent(n1));
  fs.handle(new NodeAddedSchedulerEvent(n1));
  fs.handle(new NodeUpdateSchedulerEvent(n1));

  Assert.assertEquals(4 * GB, fs.getRootQueueMetrics().getAvailableMB());
  fs.stop();
}

Example 5

Source File: TestFifoScheduler.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 50000)
public void testHeadroom() throws Exception {
  
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  // Add a node
  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  fs.handle(new NodeAddedSchedulerEvent(n1));
  
  // Add two applications
  ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
      appId1, 1);
  createMockRMApp(appAttemptId1, rm.getRMContext());
  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId1, "queue", "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
  fs.handle(attemptEvent);

  ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
  ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
      appId2, 1);
  createMockRMApp(appAttemptId2, rm.getRMContext());
  SchedulerEvent appEvent2 =
      new AppAddedSchedulerEvent(appId2, "queue", "user");
  fs.handle(appEvent2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
  fs.handle(attemptEvent2);

  List<ContainerId> emptyId = new ArrayList<ContainerId>();
  List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

  // Set up resource requests
  
  // Ask for a 1 GB container for app 1
  List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
  ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
  fs.allocate(appAttemptId1, ask1, emptyId, null, null);

  // Ask for a 2 GB container for app 2
  List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
  ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
  fs.allocate(appAttemptId2, ask2, emptyId, null, null);
  
  // Trigger container assignment
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  
  // Get the allocation for the applications and verify headroom
  Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation1.getResourceLimit().getMemory());

  Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation2.getResourceLimit().getMemory());

  rm.stop();
}

Example 6

Source File: TestFifoScheduler.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 50000)
public void testHeadroom() throws Exception {
  
  Configuration conf = new Configuration();
  conf.setClass(YarnConfiguration.RM_SCHEDULER, FifoScheduler.class,
      ResourceScheduler.class);
  MockRM rm = new MockRM(conf);
  rm.start();
  FifoScheduler fs = (FifoScheduler) rm.getResourceScheduler();

  // Add a node
  RMNode n1 =
      MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1, "127.0.0.2");
  fs.handle(new NodeAddedSchedulerEvent(n1));
  
  // Add two applications
  ApplicationId appId1 = BuilderUtils.newApplicationId(100, 1);
  ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(
      appId1, 1);
  createMockRMApp(appAttemptId1, rm.getRMContext());
  SchedulerEvent appEvent =
      new AppAddedSchedulerEvent(appId1, "queue", "user");
  fs.handle(appEvent);
  SchedulerEvent attemptEvent =
      new AppAttemptAddedSchedulerEvent(appAttemptId1, false);
  fs.handle(attemptEvent);

  ApplicationId appId2 = BuilderUtils.newApplicationId(200, 2);
  ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(
      appId2, 1);
  createMockRMApp(appAttemptId2, rm.getRMContext());
  SchedulerEvent appEvent2 =
      new AppAddedSchedulerEvent(appId2, "queue", "user");
  fs.handle(appEvent2);
  SchedulerEvent attemptEvent2 =
      new AppAttemptAddedSchedulerEvent(appAttemptId2, false);
  fs.handle(attemptEvent2);

  List<ContainerId> emptyId = new ArrayList<ContainerId>();
  List<ResourceRequest> emptyAsk = new ArrayList<ResourceRequest>();

  // Set up resource requests
  
  // Ask for a 1 GB container for app 1
  List<ResourceRequest> ask1 = new ArrayList<ResourceRequest>();
  ask1.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(GB, 1), 1));
  fs.allocate(appAttemptId1, ask1, emptyId, null, null);

  // Ask for a 2 GB container for app 2
  List<ResourceRequest> ask2 = new ArrayList<ResourceRequest>();
  ask2.add(BuilderUtils.newResourceRequest(BuilderUtils.newPriority(0),
      ResourceRequest.ANY, BuilderUtils.newResource(2 * GB, 1), 1));
  fs.allocate(appAttemptId2, ask2, emptyId, null, null);
  
  // Trigger container assignment
  fs.handle(new NodeUpdateSchedulerEvent(n1));
  
  // Get the allocation for the applications and verify headroom
  Allocation allocation1 = fs.allocate(appAttemptId1, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation1.getResourceLimit().getMemory());

  Allocation allocation2 = fs.allocate(appAttemptId2, emptyAsk, emptyId, null, null);
  Assert.assertEquals("Allocation headroom", 1 * GB,
      allocation2.getResourceLimit().getMemory());

  rm.stop();
}