org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MiniYARNCluster.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example #2
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 6 votes vote down vote up
@Test
public void testFinalSavingToFinishedWithExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successssseeeful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Expire event comes before Attempt_saved event.
  applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
    .getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example #3
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@Test
public void testFinalSavingToFinishedWithExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successssseeeful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Expire event comes before Attempt_saved event.
  applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
    .getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example #4
Source File: MiniYARNCluster.java    From big-c with Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example #5
Source File: MiniYARNClusterSplice.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
    if (HAUtil.isHAEnabled(conf)) {
        conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
    }

    if (conf.get(YarnConfiguration.RM_HOSTNAME) == null) {
        conf.set(YarnConfiguration.RM_HOSTNAME, "0.0.0.0");
    }
    LOG.info("*** "+YarnConfiguration.RM_HOSTNAME+" is set to: "+conf.get(YarnConfiguration.RM_HOSTNAME));

    if (conf.get(YarnConfiguration.RM_ADDRESS) == null) {
        conf.set(YarnConfiguration.RM_ADDRESS,
                   YarnConfiguration.DEFAULT_RM_ADDRESS);
    }
    LOG.info("*** "+YarnConfiguration.RM_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_ADDRESS));

    if (conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS) == null) {
        WebAppUtils
            .setNMWebAppHostNameAndPort(conf,
                                        MiniYARNClusterSplice.getHostname(), 0);
    }
    LOG.info("*** "+YarnConfiguration.RM_WEBAPP_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));

    resourceManagers[index].init(conf);
    resourceManagers[index].getRMContext().getDispatcher().register(
        RMAppAttemptEventType.class,
        new EventHandler<RMAppAttemptEvent>() {
            public void handle(RMAppAttemptEvent event) {
                if (event instanceof RMAppAttemptRegistrationEvent) {
                    appMasters.put(event.getApplicationAttemptId(),
                                   event.getTimestamp());
                } else if (event instanceof RMAppAttemptUnregistrationEvent) {
                    appMasters.remove(event.getApplicationAttemptId());
                }
            }
        });
}
 
Example #6
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void
    testFinalSavingToFinishedWithContainerFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Container_finished event comes before Attempt_Saved event.
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example #7
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
}
 
Example #8
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void testUnmanagedAMSuccess(String url) {
  unmanagedAM = true;
  when(submissionContext.getUnmanagedAM()).thenReturn(true);
  // submit AM and check it goes to LAUNCHED state
  scheduleApplicationAttempt();
  testAppAttemptLaunchedState(null);
  verify(amLivelinessMonitor, times(1)).register(
      applicationAttempt.getAppAttemptId());

  // launch AM
  runApplicationAttempt(null, "host", 8042, url, true);

  // complete a container
  Container container = mock(Container.class);
  when(container.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
  application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(),
      container.getNodeId()));
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class),
      container.getNodeId()));
  // complete AM
  String diagnostics = "Successful";
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
      applicationAttempt.getAppAttemptId(), url, finalStatus,
      diagnostics));
  testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
      true);
  assertFalse(transferStateFromPreviousAttempt);
}
 
Example #9
Source File: TestRMAppAttemptTransitions.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void unregisterApplicationAttempt(Container container,
    FinalApplicationStatus finalStatus, String trackingUrl,
    String diagnostics) {
  applicationAttempt.handle(
      new RMAppAttemptUnregistrationEvent(
          applicationAttempt.getAppAttemptId(),
          trackingUrl, finalStatus, diagnostics));
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishingState(container, finalStatus,
      trackingUrl, diagnostics);
}
 
Example #10
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
  progress = 1.0f;
  RMAppAttemptUnregistrationEvent unregisterEvent =
      (RMAppAttemptUnregistrationEvent) event;
  diagnostics.append(unregisterEvent.getDiagnostics());
  originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
  finalStatus = unregisterEvent.getFinalApplicationStatus();
}
 
Example #11
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void
    testFinalSavingToFinishedWithContainerFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Container_finished event comes before Attempt_Saved event.
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example #12
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void testUnmanagedAMSuccess(String url) {
  unmanagedAM = true;
  when(submissionContext.getUnmanagedAM()).thenReturn(true);
  // submit AM and check it goes to LAUNCHED state
  scheduleApplicationAttempt();
  testAppAttemptLaunchedState(null);
  verify(amLivelinessMonitor, times(1)).register(
      applicationAttempt.getAppAttemptId());

  // launch AM
  runApplicationAttempt(null, "host", 8042, url, true);

  // complete a container
  Container container = mock(Container.class);
  when(container.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
  application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(),
      container.getNodeId()));
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class),
      container.getNodeId()));
  // complete AM
  String diagnostics = "Successful";
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
      applicationAttempt.getAppAttemptId(), url, finalStatus,
      diagnostics));
  testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
      true);
  assertFalse(transferStateFromPreviousAttempt);
}
 
Example #13
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void unregisterApplicationAttempt(Container container,
    FinalApplicationStatus finalStatus, String trackingUrl,
    String diagnostics) {
  applicationAttempt.handle(
      new RMAppAttemptUnregistrationEvent(
          applicationAttempt.getAppAttemptId(),
          trackingUrl, finalStatus, diagnostics));
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishingState(container, finalStatus,
      trackingUrl, diagnostics);
}
 
Example #14
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
  progress = 1.0f;
  RMAppAttemptUnregistrationEvent unregisterEvent =
      (RMAppAttemptUnregistrationEvent) event;
  diagnostics.append(unregisterEvent.getDiagnostics());
  originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
  finalStatus = unregisterEvent.getFinalApplicationStatus();
}
 
Example #15
Source File: RMAppAttemptImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
    Object transitionToDo, RMAppAttemptState targetFinalState,
    RMAppAttemptState stateToBeStored) {

  rememberTargetTransitions(event, transitionToDo, targetFinalState);
  stateBeforeFinalSaving = getState();

  // As of today, finalState, diagnostics, final-tracking-url and
  // finalAppStatus are the only things that we store into the StateStore
  // AFTER the initial saving on app-attempt-start
  // These fields can be visible from outside only after they are saved in
  // StateStore
  String diags = null;

  // don't leave the tracking URL pointing to a non-existent AM
  setTrackingUrlToRMAppPage(stateToBeStored);
  String finalTrackingUrl = getOriginalTrackingUrl();
  FinalApplicationStatus finalStatus = null;
  int exitStatus = ContainerExitStatus.INVALID;
  switch (event.getType()) {
  case LAUNCH_FAILED:
    RMAppAttemptLaunchFailedEvent launchFaileEvent =
        (RMAppAttemptLaunchFailedEvent) event;
    diags = launchFaileEvent.getMessage();
    break;
  case REGISTERED:
    diags = getUnexpectedAMRegisteredDiagnostics();
    break;
  case UNREGISTERED:
    RMAppAttemptUnregistrationEvent unregisterEvent =
        (RMAppAttemptUnregistrationEvent) event;
    diags = unregisterEvent.getDiagnostics();
    // reset finalTrackingUrl to url sent by am
    finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
    finalStatus = unregisterEvent.getFinalApplicationStatus();
    break;
  case CONTAINER_FINISHED:
    RMAppAttemptContainerFinishedEvent finishEvent =
        (RMAppAttemptContainerFinishedEvent) event;
    diags = getAMContainerCrashedDiagnostics(finishEvent);
    exitStatus = finishEvent.getContainerStatus().getExitStatus();
    break;
  case KILL:
    break;
  case EXPIRE:
    diags = getAMExpiredDiagnostics(event);
    break;
  default:
    break;
  }
  AggregateAppResourceUsage resUsage =
      this.attemptMetrics.getAggregateAppResourceUsage();
  RMStateStore rmStore = rmContext.getStateStore();
  setFinishTime(System.currentTimeMillis());

  ApplicationAttemptStateData attemptState =
      ApplicationAttemptStateData.newInstance(
          applicationAttemptId,  getMasterContainer(),
          rmStore.getCredentialsFromAppAttempt(this),
          startTime, stateToBeStored, finalTrackingUrl, diags,
          finalStatus, exitStatus,
        getFinishTime(), resUsage.getMemorySeconds(),
        resUsage.getVcoreSeconds());
  LOG.info("Updating application attempt " + applicationAttemptId
      + " with final state: " + targetedFinalState + ", and exit status: "
      + exitStatus);
  rmStore.updateApplicationAttemptState(attemptState);
}
 
Example #16
Source File: ApplicationMasterService.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}
 
Example #17
Source File: TestRMAppAttemptTransitions.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(appResUsgRpt.getGcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());
  Assert.assertEquals(55544L, report.getGcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  when(appResUsgRpt.getGcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
  Assert.assertEquals(75544, report.getGcoreSeconds());
}
 
Example #18
Source File: ApplicationMasterService.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}
 
Example #19
Source File: RMAppAttemptImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
    Object transitionToDo, RMAppAttemptState targetFinalState,
    RMAppAttemptState stateToBeStored) {

  rememberTargetTransitions(event, transitionToDo, targetFinalState);
  stateBeforeFinalSaving = getState();

  // As of today, finalState, diagnostics, final-tracking-url and
  // finalAppStatus are the only things that we store into the StateStore
  // AFTER the initial saving on app-attempt-start
  // These fields can be visible from outside only after they are saved in
  // StateStore
  String diags = null;

  // don't leave the tracking URL pointing to a non-existent AM
  setTrackingUrlToRMAppPage(stateToBeStored);
  String finalTrackingUrl = getOriginalTrackingUrl();
  FinalApplicationStatus finalStatus = null;
  int exitStatus = ContainerExitStatus.INVALID;
  switch (event.getType()) {
  case LAUNCH_FAILED:
    RMAppAttemptLaunchFailedEvent launchFaileEvent =
        (RMAppAttemptLaunchFailedEvent) event;
    diags = launchFaileEvent.getMessage();
    break;
  case REGISTERED:
    diags = getUnexpectedAMRegisteredDiagnostics();
    break;
  case UNREGISTERED:
    RMAppAttemptUnregistrationEvent unregisterEvent =
        (RMAppAttemptUnregistrationEvent) event;
    diags = unregisterEvent.getDiagnostics();
    // reset finalTrackingUrl to url sent by am
    finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
    finalStatus = unregisterEvent.getFinalApplicationStatus();
    break;
  case CONTAINER_FINISHED:
    RMAppAttemptContainerFinishedEvent finishEvent =
        (RMAppAttemptContainerFinishedEvent) event;
    diags = getAMContainerCrashedDiagnostics(finishEvent);
    exitStatus = finishEvent.getContainerStatus().getExitStatus();
    break;
  case KILL:
    break;
  case EXPIRE:
    diags = getAMExpiredDiagnostics(event);
    break;
  default:
    break;
  }
  AggregateAppResourceUsage resUsage =
      this.attemptMetrics.getAggregateAppResourceUsage();
  RMStateStore rmStore = rmContext.getStateStore();
  setFinishTime(System.currentTimeMillis());

  ApplicationAttemptStateData attemptState =
      ApplicationAttemptStateData.newInstance(
          applicationAttemptId,  getMasterContainer(),
          rmStore.getCredentialsFromAppAttempt(this),
          startTime, stateToBeStored, finalTrackingUrl, diags,
          finalStatus, exitStatus,
        getFinishTime(), resUsage.getMemorySeconds(),
        resUsage.getVcoreSeconds(), resUsage.getGcoreSeconds());
  LOG.info("Updating application attempt " + applicationAttemptId
      + " with final state: " + targetedFinalState + ", and exit status: "
      + exitStatus);
  rmStore.updateApplicationAttemptState(attemptState);
}