org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestRMWebServicesAppsModification.java    From hadoop with Apache License 2.0 6 votes vote down vote up
protected static void verifyAppStateJson(ClientResponse response,
    RMAppState... states) throws JSONException {

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  String responseState = json.getString("state");
  boolean valid = false;
  for (RMAppState state : states) {
    if (state.toString().equals(responseState)) {
      valid = true;
    }
  }
  String msg = "app state incorrect, got " + responseState;
  assertTrue(msg, valid);
}
 
Example #2
Source File: TestQueueMappings.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkAppQueue(MockRM resourceManager, String user,
    String submissionQueue, String expected)
    throws Exception {
  RMApp app = resourceManager.submitApp(200, "name", user,
      new HashMap<ApplicationAccessType, String>(), false, submissionQueue, -1,
      null, "MAPREDUCE", false);
  RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED
      : RMAppState.ACCEPTED;
  resourceManager.waitForState(app.getApplicationId(), expectedState);
  // get scheduler app
  CapacityScheduler cs = (CapacityScheduler)
      resourceManager.getResourceScheduler();
  SchedulerApplication schedulerApp =
      cs.getSchedulerApplications().get(app.getApplicationId());
  String queue = "";
  if (schedulerApp != null) {
    queue = schedulerApp.getQueue().getQueueName();
  }
  Assert.assertTrue("expected " + expected + " actual " + queue,
      expected.equals(queue));
  Assert.assertEquals(expected, app.getQueue());
}
 
Example #3
Source File: TestAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitDuplicateApplicationId() throws Exception {
  ApplicationId appId = MockApps.newAppID(0);
  asContext.setApplicationId(appId);
  RMApp appOrig = rmContext.getRMApps().get(appId);
  Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());

  // our testApp1 should be rejected and original app with same id should be left in place
  try {
    appMonitor.submitApplication(asContext, "test");
    Assert.fail("Exception is expected when applicationId is duplicate.");
  } catch (YarnException e) {
    Assert.assertTrue("The thrown exception is not the expectd one.",
        e.getMessage().contains("Cannot add a duplicate!"));
  }

  // make sure original app didn't get removed
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.FINISHED, app.getState());
}
 
Example #4
Source File: TestRMWebServicesAppsModification.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleAppState() throws Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  String[] mediaTypes =
      { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
  for (String mediaType : mediaTypes) {
    RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
    amNodeManager.nodeHeartbeat(true);
    ClientResponse response =
        this
          .constructWebResource("apps", app.getApplicationId().toString(),
            "state").accept(mediaType).get(ClientResponse.class);
    assertEquals(Status.OK, response.getClientResponseStatus());
    if (mediaType.equals(MediaType.APPLICATION_JSON)) {
      verifyAppStateJson(response, RMAppState.ACCEPTED);
    } else if (mediaType.equals(MediaType.APPLICATION_XML)) {
      verifyAppStateXML(response, RMAppState.ACCEPTED);
    }
  }
  rm.stop();
}
 
Example #5
Source File: TestSchedulerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example #6
Source File: TestKillApplicationWithRMHA.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtRunningState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // failover and kill application
  // The application is at RUNNING State when failOver happens.
  // Since RMStateStore has already saved ApplicationState, the active RM
  // will load the ApplicationState. After that, the application will be at
  // ACCEPTED State. Because the application is not at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return false.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.RUNNING,
      RMAppAttemptState.RUNNING, RMAppState.ACCEPTED);
}
 
Example #7
Source File: TestAppManager.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testRMAppSubmit() throws Exception {
  appMonitor.submitApplication(asContext, "test");
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.NEW, app.getState());
  verify(metricsPublisher).appACLsUpdated(
      any(RMApp.class), any(String.class), anyLong());

  // wait for event to be processed
  int timeoutSecs = 0;
  while ((getAppEventType() == RMAppEventType.KILL) && 
      timeoutSecs++ < 20) {
    Thread.sleep(1000);
  }
  Assert.assertEquals("app event type sent is wrong", RMAppEventType.START,
      getAppEventType());
}
 
Example #8
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
  scheduler.getQueueManager().getLeafQueue("root.child2", true);

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  // Trying to assign to non leaf queue would return null
  assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin"));
  assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy"));
}
 
Example #9
Source File: RMServerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example #10
Source File: TestRM.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test (timeout = 60000)
public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // a failed app
  RMApp app2 = rm1.submitApp(200);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1
    .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);

  // before new attempt is launched, the app report returns the invalid AM
  // host and port.
  GetApplicationReportRequest request1 =
      GetApplicationReportRequest.newInstance(app2.getApplicationId());
  ApplicationReport report1 =
      rm1.getClientRMService().getApplicationReport(request1)
        .getApplicationReport();
  Assert.assertEquals("N/A", report1.getHost());
  Assert.assertEquals(-1, report1.getRpcPort());
}
 
Example #11
Source File: CapacityScheduler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}
 
Example #12
Source File: TestRMWebServicesAppsModification.java    From big-c with Apache License 2.0 6 votes vote down vote up
protected static void verifyAppStateJson(ClientResponse response,
    RMAppState... states) throws JSONException {

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  String responseState = json.getString("state");
  boolean valid = false;
  for (RMAppState state : states) {
    if (state.toString().equals(responseState)) {
      valid = true;
    }
  }
  String msg = "app state incorrect, got " + responseState;
  assertTrue(msg, valid);
}
 
Example #13
Source File: TestRMWebServicesApps.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testMultipleAppAttempts() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 8192);
  RMApp app1 = rm.submitApp(CONTAINER_MB, "testwordcount", "user1");
  MockAM am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
  int maxAppAttempts = rm.getConfig().getInt(
      YarnConfiguration.RM_AM_MAX_ATTEMPTS,
      YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  assertTrue(maxAppAttempts > 1);
  int numAttempt = 1;
  while (true) {
    // fail the AM by sending CONTAINER_FINISHED event without registering.
    amNodeManager.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    am.waitForState(RMAppAttemptState.FAILED);
    if (numAttempt == maxAppAttempts) {
      rm.waitForState(app1.getApplicationId(), RMAppState.FAILED);
      break;
    }
    // wait for app to start a new attempt.
    rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
    am = MockRM.launchAndRegisterAM(app1, rm, amNodeManager);
    numAttempt++;
  }
  assertEquals("incorrect number of attempts", maxAppAttempts,
      app1.getAppAttempts().values().size());
  testAppAttemptsHelper(app1.getApplicationId().toString(), app1,
      MediaType.APPLICATION_JSON);
  rm.stop();
}
 
Example #14
Source File: TestKillApplicationWithRMHA.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void failOverAndKillApp(ApplicationId appId,
    RMAppState initialRMAppState) throws Exception {
  Assert.assertEquals(initialRMAppState,
      rm1.getRMContext().getRMApps().get(appId).getState());
  explicitFailover();
  Assert.assertTrue(rm2.getRMContext().getRMApps().get(appId) == null);
  killApplication(rm2, appId, null, initialRMAppState);
}
 
Example #15
Source File: RMHATestBase.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  rm.waitForState(app.getCurrentAppAttempt().getAppAttemptId(),
      RMAppAttemptState.RUNNING);
  return am;
}
 
Example #16
Source File: TestApplicationCleanup.java    From big-c with Apache License 2.0 5 votes vote down vote up
private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  return am;
}
 
Example #17
Source File: MockRM.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public void waitForState(ApplicationId appId, RMAppState finalState)
    throws Exception {
  RMApp app = getRMContext().getRMApps().get(appId);
  Assert.assertNotNull("app shouldn't be null", app);
  int timeoutSecs = 0;
  while (!finalState.equals(app.getState()) && timeoutSecs++ < 40) {
    System.out.println("App : " + appId + " State is : " + app.getState()
        + " Waiting for state : " + finalState);
    Thread.sleep(2000);
  }
  System.out.println("App State is : " + app.getState());
  Assert.assertEquals("App state is not correct (timedout)", finalState,
      app.getState());
}
 
Example #18
Source File: TestFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testNestedUserQueue() throws IOException {
  conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
  conf.setClass(CommonConfigurationKeys.HADOOP_SECURITY_GROUP_MAPPING,
      SimpleGroupsMapping.class, GroupMappingServiceProvider.class);
  PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
  out.println("<?xml version=\"1.0\"?>");
  out.println("<allocations>");
  out.println("<queue name=\"user1group\" type=\"parent\">");
  out.println("<minResources>1024mb,0vcores</minResources>");
  out.println("</queue>");
  out.println("<queuePlacementPolicy>");
  out.println("<rule name=\"specified\" create=\"false\" />");
  out.println("<rule name=\"nestedUserQueue\">");
  out.println("     <rule name=\"primaryGroup\" create=\"false\" />");
  out.println("</rule>");
  out.println("<rule name=\"default\" />");
  out.println("</queuePlacementPolicy>");
  out.println("</allocations>");
  out.close();

  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());
  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);

  FSLeafQueue user1Leaf = scheduler.assignToQueue(rmApp1, "root.default",
      "user1");

  assertEquals("root.user1group.user1", user1Leaf.getName());
}
 
Example #19
Source File: TestKillApplicationWithRMHA.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void killApplication(MockRM rm, ApplicationId appId,
    ApplicationAttemptId appAttemptId, RMAppState rmAppState)
    throws Exception {
  KillApplicationResponse response = rm.killApp(appId);
  Assert
      .assertTrue(response.getIsKillCompleted() == isFinalState(rmAppState));
  RMApp loadedApp0 =
      rm.getRMContext().getRMApps().get(appId);
  rm.waitForState(appId, RMAppState.KILLED);
  if (appAttemptId != null) {
    rm.waitForState(appAttemptId, RMAppAttemptState.KILLED);
  }
  // no new attempt is created.
  Assert.assertEquals(1, loadedApp0.getAppAttempts().size());
}
 
Example #20
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifySuperUserAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol superUserClient = getRMClientForUser(SUPER_USER);

    // View as the superUser
    superUserClient.getApplicationReport(appReportRequest);

    // List apps as superUser
    Assert.assertEquals("App view by super-user should list the apps!!", 2,
        superUserClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the superUser
    superUserClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #21
Source File: TestRMRestart.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void waitForTokensToBeRenewed(MockRM rm2) throws Exception {
  int waitCnt = 20;
  boolean atleastOneAppInNEWState = true;
  while (waitCnt-- > 0 && atleastOneAppInNEWState) {
    atleastOneAppInNEWState = false;
    for (RMApp rmApp : rm2.getRMContext().getRMApps().values()) {
      if (rmApp.getState() == RMAppState.NEW) {
        Thread.sleep(1000);
        atleastOneAppInNEWState = true;
        break;
      }
    }
  }
}
 
Example #22
Source File: SystemMetricsPublisher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new ApplicationFinishedEvent(
            app.getApplicationId(),
            app.getDiagnostics().toString(),
            app.getFinalApplicationStatus(),
            RMServerUtils.createApplicationState(state),
            app.getCurrentAppAttempt() == null ?
                null : app.getCurrentAppAttempt().getAppAttemptId(),
            finishedTime,
            app.getRMAppMetrics()));
  }
}
 
Example #23
Source File: TestKillApplicationWithRMHA.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void failOverAndKillApp(ApplicationId appId,
    ApplicationAttemptId appAttemptId, RMAppState initialRMAppState,
    RMAppAttemptState initialRMAppAttemptState,
    RMAppState expectedAppStateBeforeKillApp) throws Exception {
  Assert.assertEquals(initialRMAppState,
      rm1.getRMContext().getRMApps().get(appId).getState());
  Assert.assertEquals(initialRMAppAttemptState, rm1.getRMContext()
      .getRMApps().get(appId).getAppAttempts().get(appAttemptId).getState());
  explicitFailover();
  Assert.assertEquals(expectedAppStateBeforeKillApp,
      rm2.getRMContext().getRMApps().get(appId).getState());
  killApplication(rm2, appId, appAttemptId, initialRMAppState);
}
 
Example #24
Source File: TestRMApplicationHistoryWriter.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testWriteApplication() throws Exception {
  RMApp app = createRMApp(ApplicationId.newInstance(0, 1));

  writer.applicationStarted(app);
  ApplicationHistoryData appHD = null;
  for (int i = 0; i < MAX_RETRIES; ++i) {
    appHD = store.getApplication(ApplicationId.newInstance(0, 1));
    if (appHD != null) {
      break;
    } else {
      Thread.sleep(100);
    }
  }
  Assert.assertNotNull(appHD);
  Assert.assertEquals("test app", appHD.getApplicationName());
  Assert.assertEquals("test app type", appHD.getApplicationType());
  Assert.assertEquals("test user", appHD.getUser());
  Assert.assertEquals("test queue", appHD.getQueue());
  Assert.assertEquals(0L, appHD.getSubmitTime());
  Assert.assertEquals(1L, appHD.getStartTime());

  writer.applicationFinished(app, RMAppState.FINISHED);
  for (int i = 0; i < MAX_RETRIES; ++i) {
    appHD = store.getApplication(ApplicationId.newInstance(0, 1));
    if (appHD.getYarnApplicationState() != null) {
      break;
    } else {
      Thread.sleep(100);
    }
  }
  Assert.assertEquals(2L, appHD.getFinishTime());
  Assert.assertEquals("test diagnostics info", appHD.getDiagnosticsInfo());
  Assert.assertEquals(FinalApplicationStatus.UNDEFINED,
    appHD.getFinalApplicationStatus());
  Assert.assertEquals(YarnApplicationState.FINISHED,
    appHD.getYarnApplicationState());
}
 
Example #25
Source File: TestAppManager.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56), 10, 1, 16384, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example #26
Source File: TestQueueMetrics.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test public void testDefaultSingleQueueMetrics() {
  String queueName = "single";
  String user = "alice";

  QueueMetrics metrics = QueueMetrics.forQueue(ms, queueName, null, false,
				 conf);
  MetricsSource queueSource= queueSource(ms, queueName);
  AppSchedulingInfo app = mockApp(user);

  metrics.submitApp(user);
  MetricsSource userSource = userSource(ms, queueName, user);
  checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
  metrics.submitAppAttempt(user);
  checkApps(queueSource, 1, 1, 0, 0, 0, 0, true);

  metrics.setAvailableResourcesToQueue(Resources.createResource(100*GB, 100, 100));
  metrics.incrPendingResources(user, 5, Resources.createResource(3*GB, 3, 3));
  // Available resources is set externally, as it depends on dynamic
  // configurable cluster/queue resources
  checkResources(queueSource, 0, 0, 0, 0, 0, 0, 100*GB, 100, 100, 15*GB, 15, 15, 5, 0, 0, 0, 0);

  metrics.runAppAttempt(app.getApplicationId(), user);
  checkApps(queueSource, 1, 0, 1, 0, 0, 0, true);

  metrics.allocateResources(user, 3, Resources.createResource(2*GB, 2, 2), true);
  checkResources(queueSource, 6*GB, 6, 6, 3, 3, 0, 100*GB, 100, 100, 9*GB, 9, 9, 2, 0, 0, 0, 0);

  metrics.releaseResources(user, 1, Resources.createResource(2*GB, 2, 2));
  checkResources(queueSource, 4*GB, 4, 4, 2, 3, 1, 100*GB, 100, 100, 9*GB, 9, 9, 2, 0, 0, 0, 0);

  metrics.finishAppAttempt(
      app.getApplicationId(), app.isPending(), app.getUser());
  checkApps(queueSource, 1, 0, 0, 0, 0, 0, true);
  metrics.finishApp(user, RMAppState.FINISHED);
  checkApps(queueSource, 1, 0, 0, 1, 0, 0, true);
  assertNull(userSource);
}
 
Example #27
Source File: TestApplicationCleanup.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testAppCleanupWhenNMReconnects() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // start RM
  MockRM rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);
  nm1.nodeHeartbeat(am0.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(app0.getApplicationId(), RMAppState.FAILED);

  // wait for application cleanup message received
  waitForAppCleanupMessageRecved(nm1, app0.getApplicationId());

  // reconnect NM with application still active
  nm1.registerNode(Arrays.asList(app0.getApplicationId()));
  waitForAppCleanupMessageRecved(nm1, app0.getApplicationId());

  rm1.stop();
}
 
Example #28
Source File: TestRMWebAppFairScheduler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 
Example #29
Source File: TestWorkPreservingRMRestart.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainersNotRecoveredForCompletedApps() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus runningContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2,
        ContainerState.RUNNING);
  NMContainerStatus completedContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3,
        ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(runningContainer, completedContainer), null);
  RMApp recoveredApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  assertEquals(RMAppState.FINISHED, recoveredApp1.getState());

  // Wait for RM to settle down on recovering containers;
  Thread.sleep(3000);

  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();

  // scheduler should not recover containers for finished apps.
  assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
  assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
}
 
Example #30
Source File: TestRMWebAppFairScheduler.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testFairSchedulerWebAppPage() {
  List<RMAppState> appStates = Arrays.asList(RMAppState.NEW,
      RMAppState.NEW_SAVING, RMAppState.SUBMITTED);
  final RMContext rmContext = mockRMContext(appStates);
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      rmContext,
      new Module() {
        @Override
        public void configure(Binder binder) {
          try {
            ResourceManager mockRmWithFairScheduler =
                mockRm(rmContext);
            binder.bind(ResourceManager.class).toInstance
                (mockRmWithFairScheduler);
            binder.bind(ApplicationBaseProtocol.class).toInstance(
              mockRmWithFairScheduler.getClientRMService());
          } catch (IOException e) {
            throw new IllegalStateException(e);
          }
        }
      });
  FairSchedulerPage fsViewInstance = injector.getInstance(FairSchedulerPage
      .class);
  fsViewInstance.render();
  WebAppTests.flushOutput(injector);
}