Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtRunningState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // failover and kill application
  // The application is at RUNNING State when failOver happens.
  // Since RMStateStore has already saved ApplicationState, the active RM
  // will load the ApplicationState. After that, the application will be at
  // ACCEPTED State. Because the application is not at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return false.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.RUNNING,
      RMAppAttemptState.RUNNING, RMAppState.ACCEPTED);
}
 
Example 2
Source Project: hadoop   Source File: TestSchedulerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static SchedulerApplication<SchedulerApplicationAttempt>
    verifyAppAddedAndRemovedFromScheduler(
        Map<ApplicationId, SchedulerApplication<SchedulerApplicationAttempt>> applications,
        EventHandler<SchedulerEvent> handler, String queueName)
        throws Exception {
  ApplicationId appId =
      ApplicationId.newInstance(System.currentTimeMillis(), 1);
  AppAddedSchedulerEvent appAddedEvent =
      new AppAddedSchedulerEvent(appId, queueName, "user");
  handler.handle(appAddedEvent);
  SchedulerApplication<SchedulerApplicationAttempt> app =
      applications.get(appId);
  // verify application is added.
  Assert.assertNotNull(app);
  Assert.assertEquals("user", app.getUser());

  AppRemovedSchedulerEvent appRemoveEvent =
      new AppRemovedSchedulerEvent(appId, RMAppState.FINISHED);
  handler.handle(appRemoveEvent);
  Assert.assertNull(applications.get(appId));
  return app;
}
 
Example 3
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 30000)
public void testRMAppSubmitDuplicateApplicationId() throws Exception {
  ApplicationId appId = MockApps.newAppID(0);
  asContext.setApplicationId(appId);
  RMApp appOrig = rmContext.getRMApps().get(appId);
  Assert.assertTrue("app name matches but shouldn't", "testApp1" != appOrig.getName());

  // our testApp1 should be rejected and original app with same id should be left in place
  try {
    appMonitor.submitApplication(asContext, "test");
    Assert.fail("Exception is expected when applicationId is duplicate.");
  } catch (YarnException e) {
    Assert.assertTrue("The thrown exception is not the expectd one.",
        e.getMessage().contains("Cannot add a duplicate!"));
  }

  // make sure original app didn't get removed
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.FINISHED, app.getState());
}
 
Example 4
Source Project: big-c   Source File: TestQueueMappings.java    License: Apache License 2.0 6 votes vote down vote up
private void checkAppQueue(MockRM resourceManager, String user,
    String submissionQueue, String expected)
    throws Exception {
  RMApp app = resourceManager.submitApp(200, "name", user,
      new HashMap<ApplicationAccessType, String>(), false, submissionQueue, -1,
      null, "MAPREDUCE", false);
  RMAppState expectedState = expected.isEmpty() ? RMAppState.FAILED
      : RMAppState.ACCEPTED;
  resourceManager.waitForState(app.getApplicationId(), expectedState);
  // get scheduler app
  CapacityScheduler cs = (CapacityScheduler)
      resourceManager.getResourceScheduler();
  SchedulerApplication schedulerApp =
      cs.getSchedulerApplications().get(app.getApplicationId());
  String queue = "";
  if (schedulerApp != null) {
    queue = schedulerApp.getQueue().getQueueName();
  }
  Assert.assertTrue("expected " + expected + " actual " + queue,
      expected.equals(queue));
  Assert.assertEquals(expected, app.getQueue());
}
 
Example 5
Source Project: big-c   Source File: TestRM.java    License: Apache License 2.0 6 votes vote down vote up
@Test (timeout = 60000)
public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
  MockRM rm1 = new MockRM(conf);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // a failed app
  RMApp app2 = rm1.submitApp(200);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1
    .nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);

  // before new attempt is launched, the app report returns the invalid AM
  // host and port.
  GetApplicationReportRequest request1 =
      GetApplicationReportRequest.newInstance(app2.getApplicationId());
  ApplicationReport report1 =
      rm1.getClientRMService().getApplicationReport(request1)
        .getApplicationReport();
  Assert.assertEquals("N/A", report1.getHost());
  Assert.assertEquals(-1, report1.getRpcPort());
}
 
Example 6
Source Project: big-c   Source File: TestRMWebServicesAppsModification.java    License: Apache License 2.0 6 votes vote down vote up
protected static void verifyAppStateJson(ClientResponse response,
    RMAppState... states) throws JSONException {

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  String responseState = json.getString("state");
  boolean valid = false;
  for (RMAppState state : states) {
    if (state.toString().equals(responseState)) {
      valid = true;
    }
  }
  String msg = "app state incorrect, got " + responseState;
  assertTrue(msg, valid);
}
 
Example 7
Source Project: hadoop   Source File: TestRMWebServicesAppsModification.java    License: Apache License 2.0 6 votes vote down vote up
protected static void verifyAppStateJson(ClientResponse response,
    RMAppState... states) throws JSONException {

  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  String responseState = json.getString("state");
  boolean valid = false;
  for (RMAppState state : states) {
    if (state.toString().equals(responseState)) {
      valid = true;
    }
  }
  String msg = "app state incorrect, got " + responseState;
  assertTrue(msg, valid);
}
 
Example 8
Source Project: big-c   Source File: CapacityScheduler.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void doneApplication(ApplicationId applicationId,
    RMAppState finalState) {
  SchedulerApplication<FiCaSchedulerApp> application =
      applications.get(applicationId);
  if (application == null){
    // The AppRemovedSchedulerEvent maybe sent on recovery for completed apps,
    // ignore it.
    LOG.warn("Couldn't find application " + applicationId);
    return;
  }
  CSQueue queue = (CSQueue) application.getQueue();
  if (!(queue instanceof LeafQueue)) {
    LOG.error("Cannot finish application " + "from non-leaf queue: "
        + queue.getQueueName());
  } else {
    queue.finishApplication(applicationId, application.getUser());
  }
  application.stop(finalState);
  applications.remove(applicationId);
}
 
Example 9
Source Project: big-c   Source File: RMServerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example 10
Source Project: big-c   Source File: TestFairScheduler.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testAssignToNonLeafQueueReturnsNull() throws Exception {
  conf.set(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, "true");
  scheduler.init(conf);
  scheduler.start();
  scheduler.reinitialize(conf, resourceManager.getRMContext());

  scheduler.getQueueManager().getLeafQueue("root.child1.granchild", true);
  scheduler.getQueueManager().getLeafQueue("root.child2", true);

  RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
  RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);

  // Trying to assign to non leaf queue would return null
  assertNull(scheduler.assignToQueue(rmApp1, "root.child1", "tintin"));
  assertNotNull(scheduler.assignToQueue(rmApp2, "root.child2", "snowy"));
}
 
Example 11
Source Project: big-c   Source File: TestAppManager.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRMAppSubmit() throws Exception {
  appMonitor.submitApplication(asContext, "test");
  RMApp app = rmContext.getRMApps().get(appId);
  Assert.assertNotNull("app is null", app);
  Assert.assertEquals("app id doesn't match", appId, app.getApplicationId());
  Assert.assertEquals("app state doesn't match", RMAppState.NEW, app.getState());
  verify(metricsPublisher).appACLsUpdated(
      any(RMApp.class), any(String.class), anyLong());

  // wait for event to be processed
  int timeoutSecs = 0;
  while ((getAppEventType() == RMAppEventType.KILL) && 
      timeoutSecs++ < 20) {
    Thread.sleep(1000);
  }
  Assert.assertEquals("app event type sent is wrong", RMAppEventType.START,
      getAppEventType());
}
 
Example 12
Source Project: big-c   Source File: TestRMWebServicesAppsModification.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testSingleAppState() throws Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  String[] mediaTypes =
      { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
  for (String mediaType : mediaTypes) {
    RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
    amNodeManager.nodeHeartbeat(true);
    ClientResponse response =
        this
          .constructWebResource("apps", app.getApplicationId().toString(),
            "state").accept(mediaType).get(ClientResponse.class);
    assertEquals(Status.OK, response.getClientResponseStatus());
    if (mediaType.equals(MediaType.APPLICATION_JSON)) {
      verifyAppStateJson(response, RMAppState.ACCEPTED);
    } else if (mediaType.equals(MediaType.APPLICATION_XML)) {
      verifyAppStateXML(response, RMAppState.ACCEPTED);
    }
  }
  rm.stop();
}
 
Example 13
Source Project: hadoop   Source File: RMApplicationHistoryWriter.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
public void applicationFinished(RMApp app, RMAppState finalState) {
  if (historyServiceEnabled) {
    dispatcher.getEventHandler().handle(
      new WritingApplicationFinishEvent(app.getApplicationId(),
        ApplicationFinishData.newInstance(app.getApplicationId(),
          app.getFinishTime(), app.getDiagnostics().toString(),
          app.getFinalApplicationStatus(),
          RMServerUtils.createApplicationState(finalState))));
  }
}
 
Example 14
Source Project: big-c   Source File: TestRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  return am;
}
 
Example 15
Source Project: hadoop   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
private void killApplication(MockRM rm, ApplicationId appId,
    ApplicationAttemptId appAttemptId, RMAppState rmAppState)
    throws Exception {
  KillApplicationResponse response = rm.killApp(appId);
  Assert
      .assertTrue(response.getIsKillCompleted() == isFinalState(rmAppState));
  RMApp loadedApp0 =
      rm.getRMContext().getRMApps().get(appId);
  rm.waitForState(appId, RMAppState.KILLED);
  if (appAttemptId != null) {
    rm.waitForState(appAttemptId, RMAppAttemptState.KILLED);
  }
  // no new attempt is created.
  Assert.assertEquals(1, loadedApp0.getAppAttempts().size());
}
 
Example 16
Source Project: big-c   Source File: TestWorkPreservingRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 600000)
public void testAppReregisterOnRMWorkPreservingRestart() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);

  // start RM
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAM(app0, rm1, nm1);
  // Issuing registerAppAttempt() before and after RM restart to confirm
  // registerApplicationMaster() is idempotent.
  am0.registerAppAttempt();

  // start new RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  rm2.waitForState(app0.getApplicationId(), RMAppState.ACCEPTED);
  rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);

  am0.setAMRMProtocol(rm2.getApplicationMasterService(), rm2.getRMContext());
  // retry registerApplicationMaster() after RM restart.
  am0.registerAppAttempt(true);

  rm2.waitForState(app0.getApplicationId(), RMAppState.RUNNING);
  rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
}
 
Example 17
Source Project: hadoop   Source File: ApplicationStateData.java    License: Apache License 2.0 5 votes vote down vote up
public static ApplicationStateData newInstance(long submitTime,
    long startTime, String user,
    ApplicationSubmissionContext submissionContext,
    RMAppState state, String diagnostics, long finishTime) {
  ApplicationStateData appState = Records.newRecord(ApplicationStateData.class);
  appState.setSubmitTime(submitTime);
  appState.setStartTime(startTime);
  appState.setUser(user);
  appState.setApplicationSubmissionContext(submissionContext);
  appState.setState(state);
  appState.setDiagnostics(diagnostics);
  appState.setFinishTime(finishTime);
  return appState;
}
 
Example 18
Source Project: hadoop   Source File: ApplicationStateDataPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RMAppState getState() {
  ApplicationStateDataProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasApplicationState()) {
    return null;
  }
  return convertFromProtoFormat(p.getApplicationState());
}
 
Example 19
Source Project: hadoop   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtFinalState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // kill the app.
  rm1.killApp(app0.getApplicationId());
  rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED);
  rm1.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED);

  // failover and kill application
  // The application is at Killed State and RMStateStore has already
  // saved this applicationState. After failover happens, the current
  // active RM will load the ApplicationState whose RMAppState is killed.
  // Because this application is at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return true.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.KILLED,
      RMAppAttemptState.KILLED, RMAppState.KILLED);
}
 
Example 20
Source Project: hadoop   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
private void failOverAndKillApp(ApplicationId appId,
    RMAppState initialRMAppState) throws Exception {
  Assert.assertEquals(initialRMAppState,
      rm1.getRMContext().getRMApps().get(appId).getState());
  explicitFailover();
  Assert.assertTrue(rm2.getRMContext().getRMApps().get(appId) == null);
  killApplication(rm2, appId, null, initialRMAppState);
}
 
Example 21
Source Project: big-c   Source File: TestRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 60000)
public void testClientRetryOnKillingApplication() throws Exception {
  MemoryRMStateStore memStore = new TestMemoryRMStateStore();
  memStore.init(conf);

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  RMApp app1 =
      rm1.submitApp(200, "name", "user", null, false, "default", 1, null,
        "myType");
  MockAM am1 = launchAM(app1, rm1, nm1);

  KillApplicationResponse response;
  int count = 0;
  while (true) {
    response = rm1.killApp(app1.getApplicationId());
    if (response.getIsKillCompleted()) {
      break;
    }
    Thread.sleep(100);
    count++;
  }
  // we expect at least 2 calls for killApp as the first killApp always return
  // false.
  Assert.assertTrue(count >= 1);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.KILLED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
  Assert.assertEquals(1, ((TestMemoryRMStateStore) memStore).updateAttempt);
  Assert.assertEquals(2, ((TestMemoryRMStateStore) memStore).updateApp);
}
 
Example 22
Source Project: hadoop   Source File: QueueACLsTestBase.java    License: Apache License 2.0 5 votes vote down vote up
private ApplicationId submitAppAndGetAppId(String submitter,
    String queueName, boolean setupACLs) throws Exception {

  GetNewApplicationRequest newAppRequest =
      GetNewApplicationRequest.newInstance();

  ApplicationClientProtocol submitterClient = getRMClientForUser(submitter);
  ApplicationId applicationId =
      submitterClient.getNewApplication(newAppRequest).getApplicationId();

  Resource resource = BuilderUtils.newResource(1024, 1);
  Map<ApplicationAccessType, String> acls = createACLs(submitter, setupACLs);
  ContainerLaunchContext amContainerSpec =
      ContainerLaunchContext.newInstance(null, null, null, null, null, acls);

  ApplicationSubmissionContext appSubmissionContext =
      ApplicationSubmissionContext.newInstance(applicationId,
        "applicationName", queueName, null, amContainerSpec, false, true, 1,
        resource, "applicationType");
  appSubmissionContext.setApplicationId(applicationId);
  appSubmissionContext.setQueue(queueName);

  SubmitApplicationRequest submitRequest =
      SubmitApplicationRequest.newInstance(appSubmissionContext);
  submitterClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 23
Source Project: hadoop   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailOverHappensDuringApplicationKill()
    throws Exception {
  // create a customized ClientRMService
  // When receives the killApplicationRequest, simply return the response
  // and make sure the application will not be KILLED State
  startRMsWithCustomizedClientRMService();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // ensure that the app is in running state
  Assert.assertEquals(app0.getState(), RMAppState.RUNNING);

  // kill the app.
  rm1.killApp(app0.getApplicationId());

  // failover happens before this application goes to final state.
  // The RMAppState that will be loaded by the active rm
  // should be ACCEPTED.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.RUNNING,
      RMAppAttemptState.RUNNING, RMAppState.ACCEPTED);

}
 
Example 24
Source Project: big-c   Source File: TestApplicationACLs.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
    AccessControlList modifyACL) throws Exception {
  SubmitApplicationRequest submitRequest = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  ApplicationSubmissionContext context = recordFactory
      .newRecordInstance(ApplicationSubmissionContext.class);

  ApplicationId applicationId = rmClient.getNewApplication(
      recordFactory.newRecordInstance(GetNewApplicationRequest.class))
      .getApplicationId();
  context.setApplicationId(applicationId);

  Map<ApplicationAccessType, String> acls
      = new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
  acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());

  ContainerLaunchContext amContainer = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  Resource resource = BuilderUtils.newResource(1024, 1);
  context.setResource(resource);
  amContainer.setApplicationACLs(acls);
  context.setAMContainerSpec(amContainer);
  submitRequest.setApplicationSubmissionContext(context);
  rmClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 
Example 25
Source Project: big-c   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtNewState()
    throws Exception {
  // create a customized RMAppManager
  // During the process of Application submission,
  // the RMAppState will always be NEW.
  // The ApplicationState will not be saved in RMStateStore.
  startRMsWithCustomizedRMAppManager();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // Submit the application
  RMApp app0 =
      rm1.submitApp(200, "", UserGroupInformation
          .getCurrentUser().getShortUserName(), null, false, null,
          configuration.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
              YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null,
          false, false);

  // failover and kill application
  // When FailOver happens, the state of this application is NEW,
  // and ApplicationState is not saved in RMStateStore. The active RM
  // can not load the ApplicationState of this application.
  // Expected to get ApplicationNotFoundException
  // when receives the KillApplicationRequest
  try {
    failOverAndKillApp(app0.getApplicationId(), RMAppState.NEW);
    fail("Should get an exception here");
  } catch (ApplicationNotFoundException ex) {
    Assert.assertTrue(ex.getMessage().contains(
        "Trying to kill an absent application " + app0.getApplicationId()));
  }
}
 
Example 26
Source Project: big-c   Source File: TestKillApplicationWithRMHA.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 20000)
public void testKillAppWhenFailoverHappensAtFinalState()
    throws Exception {
  startRMs();
  MockNM nm1 = new MockNM("127.0.0.1:1234", 15120,
      rm1.getResourceTrackerService());
  nm1.registerNode();

  // create app and launch the AM
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // kill the app.
  rm1.killApp(app0.getApplicationId());
  rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED);
  rm1.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED);

  // failover and kill application
  // The application is at Killed State and RMStateStore has already
  // saved this applicationState. After failover happens, the current
  // active RM will load the ApplicationState whose RMAppState is killed.
  // Because this application is at Final State,
  // KillApplicationResponse.getIsKillCompleted is expected to return true.
  failOverAndKillApp(app0.getApplicationId(),
      am0.getApplicationAttemptId(), RMAppState.KILLED,
      RMAppAttemptState.KILLED, RMAppState.KILLED);
}
 
Example 27
Source Project: hadoop   Source File: TestAppManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 30000)
 public void testEscapeApplicationSummary() {
   RMApp app = mock(RMAppImpl.class);
   when(app.getApplicationId()).thenReturn(
       ApplicationId.newInstance(100L, 1));
   when(app.getName()).thenReturn("Multiline\n\n\r\rAppName");
   when(app.getUser()).thenReturn("Multiline\n\n\r\rUserName");
   when(app.getQueue()).thenReturn("Multiline\n\n\r\rQueueName");
   when(app.getState()).thenReturn(RMAppState.RUNNING);
   when(app.getApplicationType()).thenReturn("MAPREDUCE");
   RMAppMetrics metrics =
       new RMAppMetrics(Resource.newInstance(1234, 56, 56), 10, 1, 16384, 64, 64);
   when(app.getRMAppMetrics()).thenReturn(metrics);

   RMAppManager.ApplicationSummary.SummaryBuilder summary =
       new RMAppManager.ApplicationSummary().createAppSummary(app);
   String msg = summary.toString();
   LOG.info("summary: " + msg);
   Assert.assertFalse(msg.contains("\n"));
   Assert.assertFalse(msg.contains("\r"));

   String escaped = "\\n\\n\\r\\r";
   Assert.assertTrue(msg.contains("Multiline" + escaped +"AppName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"UserName"));
   Assert.assertTrue(msg.contains("Multiline" + escaped +"QueueName"));
   Assert.assertTrue(msg.contains("memorySeconds=16384"));
   Assert.assertTrue(msg.contains("vcoreSeconds=64"));
   Assert.assertTrue(msg.contains("gcoreSeconds=64"));
   Assert.assertTrue(msg.contains("preemptedAMContainers=1"));
   Assert.assertTrue(msg.contains("preemptedNonAMContainers=10"));
   Assert.assertTrue(msg.contains("preemptedResources=<memory:1234\\, vCores:56\\, gCores:56>"));
   Assert.assertTrue(msg.contains("applicationType=MAPREDUCE"));
}
 
Example 28
Source Project: big-c   Source File: TestCapacityScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  return am;
}
 
Example 29
Source Project: big-c   Source File: TestWorkPreservingRMRestart.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 20000)
public void testContainersNotRecoveredForCompletedApps() throws Exception {
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  NMContainerStatus runningContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2,
        ContainerState.RUNNING);
  NMContainerStatus completedContainer =
      TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3,
        ContainerState.COMPLETE);
  nm1.registerNode(Arrays.asList(runningContainer, completedContainer), null);
  RMApp recoveredApp1 =
      rm2.getRMContext().getRMApps().get(app1.getApplicationId());
  assertEquals(RMAppState.FINISHED, recoveredApp1.getState());

  // Wait for RM to settle down on recovering containers;
  Thread.sleep(3000);

  AbstractYarnScheduler scheduler =
      (AbstractYarnScheduler) rm2.getResourceScheduler();

  // scheduler should not recover containers for finished apps.
  assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
  assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
}
 
Example 30
Source Project: hadoop   Source File: TestRMWebAppFairScheduler.java    License: Apache License 2.0 5 votes vote down vote up
private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}