Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestRMRestart.java From hadoop with Apache License 2.0

5 votes

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}

Example 2

Source File: MockRM.java From hadoop with Apache License 2.0

5 votes

public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
}

Example 3

Source File: MockAM.java From hadoop with Apache License 2.0

5 votes

public void unregisterAppAttempt(boolean waitForStateRunning)
    throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  unregisterAppAttempt(req, waitForStateRunning);
}

Example 4

Source File: TestApplicationMasterServiceProtocolOnHA.java From hadoop with Apache License 2.0

5 votes

@Test(timeout = 15000)
public void testFinishApplicationMasterOnHA() throws YarnException,
    IOException {
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  FinishApplicationMasterResponse response =
      amClient.finishApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeFinishApplicationMasterResponse());
}

Example 5

Source File: TestRMRestart.java From big-c with Apache License 2.0

5 votes

private void finishApplicationMaster(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  finishApplicationMaster(rmApp, rm, nm, am, req);
}

Example 6

Source File: MockRM.java From big-c with Apache License 2.0

5 votes

public static void finishAMAndVerifyAppState(RMApp rmApp, MockRM rm, MockNM nm,
    MockAM am) throws Exception {
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am.unregisterAppAttempt(req,true);
  am.waitForState(RMAppAttemptState.FINISHING);
  nm.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am.waitForState(RMAppAttemptState.FINISHED);
  rm.waitForState(rmApp.getApplicationId(), RMAppState.FINISHED);
}

Example 7

Source File: MockAM.java From big-c with Apache License 2.0

5 votes

public void unregisterAppAttempt(boolean waitForStateRunning)
    throws Exception {
  final FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  unregisterAppAttempt(req, waitForStateRunning);
}

Example 8

Source File: TestApplicationMasterServiceProtocolOnHA.java From big-c with Apache License 2.0

5 votes

@Test(timeout = 15000)
public void testFinishApplicationMasterOnHA() throws YarnException,
    IOException {
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(
          FinalApplicationStatus.SUCCEEDED, "", "");
  FinishApplicationMasterResponse response =
      amClient.finishApplicationMaster(request);
  Assert.assertEquals(response,
      this.cluster.createFakeFinishApplicationMasterResponse());
}

Example 9

Source File: TestRM.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 30000)
public void testKillFinishingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am1.unregisterAppAttempt(req,true);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}

Example 10

Source File: TestRMRestart.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartWaitForPreviousSucceededAttempt() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  MemoryRMStateStore memStore = new MemoryRMStateStore() {
    int count = 0;

    @Override
    public void updateApplicationStateInternal(ApplicationId appId,
        ApplicationStateData appStateData) throws Exception {
      if (count == 0) {
        // do nothing; simulate app final state is not saved.
        LOG.info(appId + " final state is not saved.");
        count++;
      } else {
        super.updateApplicationStateInternal(appId, appStateData);
      }
    }
  };
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120);
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am0.unregisterAppAttempt(req, true);
  am0.waitForState(RMAppAttemptState.FINISHING);
  // app final state is not saved. This guarantees that RMApp cannot be
  // recovered via its own saved state, but only via the event notification
  // from the RMAppAttempt on recovery.
  Assert.assertNull(rmAppState.get(app0.getApplicationId()).getState());

  // start RM
  MockRM rm2 = createMockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.start();

  rm2.waitForState(app0.getCurrentAppAttempt().getAppAttemptId(),
    RMAppAttemptState.FINISHED);
  rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED);
  // app final state is saved via the finish event from attempt.
  Assert.assertEquals(RMAppState.FINISHED,
      rmAppState.get(app0.getApplicationId()).getState());
}

Example 11

Source File: TestRMRestart.java From hadoop with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartSucceededApp() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create an app and finish the app.
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // unregister am
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "diagnostics", "trackingUrl");
  finishApplicationMaster(app0, rm1, nm1, am0, req);
 
  // check the state store about the unregistered info.
  ApplicationStateData appState = rmAppState.get(app0.getApplicationId());
  ApplicationAttemptStateData attemptState0 =
    appState.getAttempt(am0.getApplicationAttemptId());
  Assert.assertEquals("diagnostics", attemptState0.getDiagnostics());
  Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
    attemptState0.getFinalApplicationStatus());
  Assert.assertEquals("trackingUrl", attemptState0.getFinalTrackingUrl());
  Assert.assertEquals(app0.getFinishTime(), appState.getFinishTime());

  // restart rm
  MockRM rm2 = createMockRM(conf, memStore);
  rm2.start();

  // verify application report returns the same app info as the app info
  // before RM restarts.
  ApplicationReport appReport = verifyAppReportAfterRMRestart(app0, rm2);
  Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
    appReport.getFinalApplicationStatus());
  Assert.assertEquals("trackingUrl", appReport.getOriginalTrackingUrl());
}

Example 12

Source File: RMCommunicator.java From hadoop with Apache License 2.0

4 votes

@VisibleForTesting
protected void doUnregistration()
    throws YarnException, IOException, InterruptedException {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  JobImpl jobImpl = (JobImpl)job;
  if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
    finishState = FinalApplicationStatus.SUCCEEDED;
  } else if (jobImpl.getInternalState() == JobStateInternal.KILLED
      || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
    finishState = FinalApplicationStatus.KILLED;
  } else if (jobImpl.getInternalState() == JobStateInternal.FAILED
      || jobImpl.getInternalState() == JobStateInternal.ERROR) {
    finishState = FinalApplicationStatus.FAILED;
  }
  StringBuffer sb = new StringBuffer();
  for (String s : job.getDiagnostics()) {
    sb.append(s).append("\n");
  }
  LOG.info("Setting job diagnostics to " + sb.toString());

  String historyUrl =
      MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
          context.getApplicationID());
  LOG.info("History url is " + historyUrl);
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(finishState,
        sb.toString(), historyUrl);
  try {
    while (true) {
      FinishApplicationMasterResponse response =
          scheduler.finishApplicationMaster(request);
      if (response.getIsUnregistered()) {
        // When excepting ClientService, other services are already stopped,
        // it is safe to let clients know the final states. ClientService
        // should wait for some time so clients have enough time to know the
        // final states.
        RunningAppContext raContext = (RunningAppContext) context;
        raContext.markSuccessfulUnregistration();
        break;
      }
      LOG.info("Waiting for application to be successfully unregistered.");
      Thread.sleep(rmPollInterval);
    }
  } catch (ApplicationMasterNotRegisteredException e) {
    // RM might have restarted or failed over and so lost the fact that AM had
    // registered before.
    register();
    doUnregistration();
  }
}

Example 13

Source File: TestRM.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 30000)
public void testKillFinishingApp() throws Exception{

  // this dispatcher ignores RMAppAttemptEventType.KILL event
  final Dispatcher dispatcher = new AsyncDispatcher() {
    @Override
    public EventHandler getEventHandler() {

      class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {
        @Override
        public boolean matches(Object argument) {
          if (argument instanceof RMAppAttemptEvent) {
            if (((RMAppAttemptEvent) argument).getType().equals(
              RMAppAttemptEventType.KILL)) {
              return true;
            }
          }
          return false;
        }
      }

      EventHandler handler = spy(super.getEventHandler());
      doNothing().when(handler).handle(argThat(new EventArgMatcher()));
      return handler;
    }
  };

  MockRM rm1 = new MockRM(conf){
    @Override
    protected Dispatcher createDispatcher() {
      return dispatcher;
    }
  };
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1 = rm1.submitApp(200);
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);

  rm1.killApp(app1.getApplicationId());

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am1.unregisterAppAttempt(req,true);

  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
  nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
  rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}

Example 14

Source File: TestRMRestart.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartWaitForPreviousSucceededAttempt() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 2);
  MemoryRMStateStore memStore = new MemoryRMStateStore() {
    int count = 0;

    @Override
    public void updateApplicationStateInternal(ApplicationId appId,
        ApplicationStateData appStateData) throws Exception {
      if (count == 0) {
        // do nothing; simulate app final state is not saved.
        LOG.info(appId + " final state is not saved.");
        count++;
      } else {
        super.updateApplicationStateInternal(appId, appStateData);
      }
    }
  };
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 15120);
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = MockRM.launchAndRegisterAM(app0, rm1, nm1);

  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "", "");
  am0.unregisterAppAttempt(req, true);
  am0.waitForState(RMAppAttemptState.FINISHING);
  // app final state is not saved. This guarantees that RMApp cannot be
  // recovered via its own saved state, but only via the event notification
  // from the RMAppAttempt on recovery.
  Assert.assertNull(rmAppState.get(app0.getApplicationId()).getState());

  // start RM
  MockRM rm2 = createMockRM(conf, memStore);
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());
  rm2.start();

  rm2.waitForState(app0.getCurrentAppAttempt().getAppAttemptId(),
    RMAppAttemptState.FINISHED);
  rm2.waitForState(app0.getApplicationId(), RMAppState.FINISHED);
  // app final state is saved via the finish event from attempt.
  Assert.assertEquals(RMAppState.FINISHED,
      rmAppState.get(app0.getApplicationId()).getState());
}

Example 15

Source File: TestRMRestart.java From big-c with Apache License 2.0

4 votes

@Test (timeout = 60000)
public void testRMRestartSucceededApp() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
    YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  RMState rmState = memStore.getState();
  Map<ApplicationId, ApplicationStateData> rmAppState =
      rmState.getApplicationState();

  // start RM
  MockRM rm1 = createMockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();

  // create an app and finish the app.
  RMApp app0 = rm1.submitApp(200);
  MockAM am0 = launchAM(app0, rm1, nm1);

  // unregister am
  FinishApplicationMasterRequest req =
      FinishApplicationMasterRequest.newInstance(
        FinalApplicationStatus.SUCCEEDED, "diagnostics", "trackingUrl");
  finishApplicationMaster(app0, rm1, nm1, am0, req);
 
  // check the state store about the unregistered info.
  ApplicationStateData appState = rmAppState.get(app0.getApplicationId());
  ApplicationAttemptStateData attemptState0 =
    appState.getAttempt(am0.getApplicationAttemptId());
  Assert.assertEquals("diagnostics", attemptState0.getDiagnostics());
  Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
    attemptState0.getFinalApplicationStatus());
  Assert.assertEquals("trackingUrl", attemptState0.getFinalTrackingUrl());
  Assert.assertEquals(app0.getFinishTime(), appState.getFinishTime());

  // restart rm
  MockRM rm2 = createMockRM(conf, memStore);
  rm2.start();

  // verify application report returns the same app info as the app info
  // before RM restarts.
  ApplicationReport appReport = verifyAppReportAfterRMRestart(app0, rm2);
  Assert.assertEquals(FinalApplicationStatus.SUCCEEDED,
    appReport.getFinalApplicationStatus());
  Assert.assertEquals("trackingUrl", appReport.getOriginalTrackingUrl());
}

Example 16

Source File: RMCommunicator.java From big-c with Apache License 2.0

4 votes

@VisibleForTesting
protected void doUnregistration()
    throws YarnException, IOException, InterruptedException {
  FinalApplicationStatus finishState = FinalApplicationStatus.UNDEFINED;
  JobImpl jobImpl = (JobImpl)job;
  if (jobImpl.getInternalState() == JobStateInternal.SUCCEEDED) {
    finishState = FinalApplicationStatus.SUCCEEDED;
  } else if (jobImpl.getInternalState() == JobStateInternal.KILLED
      || (jobImpl.getInternalState() == JobStateInternal.RUNNING && isSignalled)) {
    finishState = FinalApplicationStatus.KILLED;
  } else if (jobImpl.getInternalState() == JobStateInternal.FAILED
      || jobImpl.getInternalState() == JobStateInternal.ERROR) {
    finishState = FinalApplicationStatus.FAILED;
  }
  StringBuffer sb = new StringBuffer();
  for (String s : job.getDiagnostics()) {
    sb.append(s).append("\n");
  }
  LOG.info("Setting job diagnostics to " + sb.toString());

  String historyUrl =
      MRWebAppUtil.getApplicationWebURLOnJHSWithScheme(getConfig(),
          context.getApplicationID());
  LOG.info("History url is " + historyUrl);
  FinishApplicationMasterRequest request =
      FinishApplicationMasterRequest.newInstance(finishState,
        sb.toString(), historyUrl);
  try {
    while (true) {
      FinishApplicationMasterResponse response =
          scheduler.finishApplicationMaster(request);
      if (response.getIsUnregistered()) {
        // When excepting ClientService, other services are already stopped,
        // it is safe to let clients know the final states. ClientService
        // should wait for some time so clients have enough time to know the
        // final states.
        RunningAppContext raContext = (RunningAppContext) context;
        raContext.markSuccessfulUnregistration();
        break;
      }
      LOG.info("Waiting for application to be successfully unregistered.");
      Thread.sleep(rmPollInterval);
    }
  } catch (ApplicationMasterNotRegisteredException e) {
    // RM might have restarted or failed over and so lost the fact that AM had
    // registered before.
    register();
    doUnregistration();
  }
}