Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState#NEW

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState#NEW . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAssignToQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
  
  FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix");
  FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix");
  
  // assert FSLeafQueue's name is the correct name is the one set in the RMApp
  assertEquals(rmApp1.getQueue(), queue1.getName());
  assertEquals("root.asterix", rmApp1.getQueue());
  assertEquals(rmApp2.getQueue(), queue2.getName());
  assertEquals("root.notdefault", rmApp2.getQueue());
}
Example 2
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
  scheduler.getQueueManager().getLeafQueue("root.child2", true);

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  // Trying to assign to non leaf queue would return null
  assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin"));
  assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy"));
}
Example 3
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAssignToQueue() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
  
  FSLeafQueue queue1 = scheduler.assignToQueue(rmApp1, "default", "asterix");
  FSLeafQueue queue2 = scheduler.assignToQueue(rmApp2, "notdefault", "obelix");
  
  // assert FSLeafQueue's name is the correct name is the one set in the RMApp
  assertEquals(rmApp1.getQueue(), queue1.getName());
  assertEquals("root.asterix", rmApp1.getQueue());
  assertEquals(rmApp2.getQueue(), queue2.getName());
  assertEquals("root.notdefault", rmApp2.getQueue());
}
Example 4
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
  scheduler.getQueueManager().getLeafQueue("root.child2", true);

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  // Trying to assign to non leaf queue would return null
  assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin"));
  assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy"));
}
Example 5
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}
Example 6
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNestedUserQueue() throws IOException {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
      SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"user1group\" type=\"parent\">");
  out.println("<minResources>1024mb,0vcores,0gcores</minResources>");
  out.println("</queue>");
  out.println("<queuePlacementPolicy>");
  out.println("<rule name=\"specified\" create=\"false\" />");
  out.println("<rule name=\"nestedUserQueue\">");
  out.println("     <rule name=\"primaryGroup\" create=\"false\" />");
  out.println("</rule>");
  out.println("<rule name=\"default\" />");
  out.println("</queuePlacementPolicy>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);

  FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default",
      "user1");

  assertEquals("root.user1group.user1", user1Leaf.getName());
}
Example 7
Source File: TestRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up 
private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}
Example 8
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up 
@Test
public void testNestedUserQueue() throws IOException {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
      SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"user1group\" type=\"parent\">");
  out.println("<minResources>1024mb,0vcores</minResources>");
  out.println("</queue>");
  out.println("<queuePlacementPolicy>");
  out.println("<rule name=\"specified\" create=\"false\" />");
  out.println("<rule name=\"nestedUserQueue\">");
  out.println("     <rule name=\"primaryGroup\" create=\"false\" />");
  out.println("</rule>");
  out.println("<rule name=\"default\" />");
  out.println("</queuePlacementPolicy>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);

  FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default",
      "user1");

  assertEquals("root.user1group.user1", user1Leaf.getName());
}
Example 9
Source File: MyriadFileSystemRMStateStoreTest.java    From incubator-myriad with Apache License 2.0 5 votes vote down vote up 
@Test
public void testStoreAndRemoveApplication() throws Exception {
  MyriadFileSystemRMStateStore store = getInitializedStore();
  store.start();
  RMApp appOne = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp appTwo = new MockRMApp(0, 0, RMAppState.NEW);

  store.storeNewApplication(appOne);
  store.storeNewApplication(appTwo);
  store.removeApplication(appOne);
  store.removeApplication(appTwo);
  store.close();
}
Example 10
Source File: TestAppManager.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRMAppRetireSomeDifferentStates() throws Exception {
  long now = System.currentTimeMillis();

  // these parameters don't matter, override applications below
  RMContext rmContext = mockRMContext(10, now - 20000);
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 2);
  conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2);

  TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);

  // clear out applications map
  rmContext.getRMApps().clear();
  Assert.assertEquals("map isn't empty", 0, rmContext.getRMApps().size());

  // 6 applications are in final state, 4 are not in final state.
  // / set with various finished states
  RMApp app = new MockRMApp(0, now - 20000, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(1, now - 200000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(2, now - 30000, RMAppState.FINISHED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(3, now - 20000, RMAppState.RUNNING);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(4, now - 20000, RMAppState.NEW);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  // make sure it doesn't expire these since still running
  app = new MockRMApp(5, now - 10001, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(6, now - 30000, RMAppState.ACCEPTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(7, now - 20000, RMAppState.SUBMITTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(8, now - 10001, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(9, now - 20000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  Assert.assertEquals("Number of apps incorrect before", 10, rmContext
      .getRMApps().size());

  // add them to completed apps list
  addToCompletedApps(appMonitor, rmContext);

  // shouldn't  have to many apps
  appMonitor.checkAppNumCompletedLimit();
  Assert.assertEquals("Number of apps incorrect after # completed check", 6,
      rmContext.getRMApps().size());
  Assert.assertEquals("Number of completed apps incorrect after check", 2,
      appMonitor.getCompletedAppsListSize());
  // 6 applications in final state, 4 of them are removed
  verify(rmContext.getStateStore(), times(4)).removeApplication(
    isA(RMApp.class));
}
Example 11
Source File: TestFairScheduler.java    From hadoop with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);

  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));

  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"parentq\" type=\"parent\">");
  out.println("<minResources>1024mb,0vcores,0gcores</minResources>");
  out.println("</queue>");
  out.println("<queuePlacementPolicy>");
  out.println("<rule name=\"nestedUserQueue\">");
  out.println("     <rule name=\"specified\" create=\"false\" />");
  out.println("</rule>");
  out.println("<rule name=\"default\" />");
  out.println("</queuePlacementPolicy>");
  out.println("</allocations>");
  out.close();

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  int capacity = 16 * 1024;
  // create node with 16 G
  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // user1,user2 submit their apps to parentq and create user queues
  createSchedulingRequest(10 * 1024, "root.parentq", "user1");
  createSchedulingRequest(10 * 1024, "root.parentq", "user2");
  // user3 submits app in default queue
  createSchedulingRequest(10 * 1024, "root.default", "user3");

  scheduler.update();
  scheduler.getQueueManager().getRootQueue()
      .setSteadyFairShare(scheduler.getClusterResource());
  scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();

  Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
      .getLeafQueues();

  for (FSLeafQueue leaf : leafQueues) {
    if (leaf.getName().equals("root.parentq.user1")
        || leaf.getName().equals("root.parentq.user2")) {
      // assert that the fair share is 1/4th node1's capacity
      assertEquals(capacity / 4, leaf.getFairShare().getMemory());
      // assert that the steady fair share is 1/4th node1's capacity
      assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemory());
      // assert weights are equal for both the user queues
      assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
    }
  }
}
Example 12
Source File: TestAppManager.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testRMAppRetireSomeDifferentStates() throws Exception {
  long now = System.currentTimeMillis();

  // these parameters don't matter, override applications below
  RMContext rmContext = mockRMContext(10, now - 20000);
  Configuration conf = new YarnConfiguration();
  conf.setInt(YarnConfiguration.RM_STATE_STORE_MAX_COMPLETED_APPLICATIONS, 2);
  conf.setInt(YarnConfiguration.RM_MAX_COMPLETED_APPLICATIONS, 2);

  TestRMAppManager appMonitor = new TestRMAppManager(rmContext, conf);

  // clear out applications map
  rmContext.getRMApps().clear();
  Assert.assertEquals("map isn't empty", 0, rmContext.getRMApps().size());

  // 6 applications are in final state, 4 are not in final state.
  // / set with various finished states
  RMApp app = new MockRMApp(0, now - 20000, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(1, now - 200000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(2, now - 30000, RMAppState.FINISHED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(3, now - 20000, RMAppState.RUNNING);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(4, now - 20000, RMAppState.NEW);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  // make sure it doesn't expire these since still running
  app = new MockRMApp(5, now - 10001, RMAppState.KILLED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(6, now - 30000, RMAppState.ACCEPTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(7, now - 20000, RMAppState.SUBMITTED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(8, now - 10001, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);
  app = new MockRMApp(9, now - 20000, RMAppState.FAILED);
  rmContext.getRMApps().put(app.getApplicationId(), app);

  Assert.assertEquals("Number of apps incorrect before", 10, rmContext
      .getRMApps().size());

  // add them to completed apps list
  addToCompletedApps(appMonitor, rmContext);

  // shouldn't  have to many apps
  appMonitor.checkAppNumCompletedLimit();
  Assert.assertEquals("Number of apps incorrect after # completed check", 6,
      rmContext.getRMApps().size());
  Assert.assertEquals("Number of completed apps incorrect after check", 2,
      appMonitor.getCompletedAppsListSize());
  // 6 applications in final state, 4 of them are removed
  verify(rmContext.getStateStore(), times(4)).removeApplication(
    isA(RMApp.class));
}
Example 13
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 4 votes vote down vote up 
@Test
public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);

  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));

  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"parentq\" type=\"parent\">");
  out.println("<minResources>1024mb,0vcores</minResources>");
  out.println("</queue>");
  out.println("<queuePlacementPolicy>");
  out.println("<rule name=\"nestedUserQueue\">");
  out.println("     <rule name=\"specified\" create=\"false\" />");
  out.println("</rule>");
  out.println("<rule name=\"default\" />");
  out.println("</queuePlacementPolicy>");
  out.println("</allocations>");
  out.close();

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  int capacity = 16 * 1024;
  // create node with 16 G
  RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(capacity),
      1, "127.0.0.1");
  NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
  scheduler.handle(nodeEvent1);

  // user1,user2 submit their apps to parentq and create user queues
  createSchedulingRequest(10 * 1024, "root.parentq", "user1");
  createSchedulingRequest(10 * 1024, "root.parentq", "user2");
  // user3 submits app in default queue
  createSchedulingRequest(10 * 1024, "root.default", "user3");

  scheduler.update();
  scheduler.getQueueManager().getRootQueue()
      .setSteadyFairShare(scheduler.getClusterResource());
  scheduler.getQueueManager().getRootQueue().recomputeSteadyShares();

  Collection<FSLeafQueue> leafQueues = scheduler.getQueueManager()
      .getLeafQueues();

  for (FSLeafQueue leaf : leafQueues) {
    if (leaf.getName().equals("root.parentq.user1")
        || leaf.getName().equals("root.parentq.user2")) {
      // assert that the fair share is 1/4th node1's capacity
      assertEquals(capacity / 4, leaf.getFairShare().getMemory());
      // assert that the steady fair share is 1/4th node1's capacity
      assertEquals(capacity / 4, leaf.getSteadyFairShare().getMemory());
      // assert weights are equal for both the user queues
      assertEquals(1.0, leaf.getWeights().getWeight(ResourceType.MEMORY), 0);
    }
  }
}