Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent

The following examples show how to use org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptUnregistrationEvent. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example 2
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFinalSavingToFinishedWithExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successssseeeful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Expire event comes before Attempt_saved event.
  applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
    .getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example 3
Source Project: big-c   Source File: MiniYARNCluster.java    License: Apache License 2.0 6 votes vote down vote up
private synchronized void initResourceManager(int index, Configuration conf) {
  if (HAUtil.isHAEnabled(conf)) {
    conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
  }
  resourceManagers[index].init(conf);
  resourceManagers[index].getRMContext().getDispatcher().register(
      RMAppAttemptEventType.class,
      new EventHandler<RMAppAttemptEvent>() {
        public void handle(RMAppAttemptEvent event) {
          if (event instanceof RMAppAttemptRegistrationEvent) {
            appMasters.put(event.getApplicationAttemptId(),
                event.getTimestamp());
          } else if (event instanceof RMAppAttemptUnregistrationEvent) {
            appMasters.remove(event.getApplicationAttemptId());
          }
        }
      });
}
 
Example 4
Source Project: big-c   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testFinalSavingToFinishedWithExpire() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successssseeeful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Expire event comes before Attempt_saved event.
  applicationAttempt.handle(new RMAppAttemptEvent(applicationAttempt
    .getAppAttemptId(), RMAppAttemptEventType.EXPIRE));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example 5
Source Project: hadoop   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
  progress = 1.0f;
  RMAppAttemptUnregistrationEvent unregisterEvent =
      (RMAppAttemptUnregistrationEvent) event;
  diagnostics.append(unregisterEvent.getDiagnostics());
  originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
  finalStatus = unregisterEvent.getFinalApplicationStatus();
}
 
Example 6
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
private void unregisterApplicationAttempt(Container container,
    FinalApplicationStatus finalStatus, String trackingUrl,
    String diagnostics) {
  applicationAttempt.handle(
      new RMAppAttemptUnregistrationEvent(
          applicationAttempt.getAppAttemptId(),
          trackingUrl, finalStatus, diagnostics));
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishingState(container, finalStatus,
      trackingUrl, diagnostics);
}
 
Example 7
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
private void testUnmanagedAMSuccess(String url) {
  unmanagedAM = true;
  when(submissionContext.getUnmanagedAM()).thenReturn(true);
  // submit AM and check it goes to LAUNCHED state
  scheduleApplicationAttempt();
  testAppAttemptLaunchedState(null);
  verify(amLivelinessMonitor, times(1)).register(
      applicationAttempt.getAppAttemptId());

  // launch AM
  runApplicationAttempt(null, "host", 8042, url, true);

  // complete a container
  Container container = mock(Container.class);
  when(container.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
  application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(),
      container.getNodeId()));
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class),
      container.getNodeId()));
  // complete AM
  String diagnostics = "Successful";
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
      applicationAttempt.getAppAttemptId(), url, finalStatus,
      diagnostics));
  testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
      true);
  assertFalse(transferStateFromPreviousAttempt);
}
 
Example 8
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void
    testFinalSavingToFinishedWithContainerFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Container_finished event comes before Attempt_Saved event.
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example 9
Source Project: big-c   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void updateInfoOnAMUnregister(RMAppAttemptEvent event) {
  progress = 1.0f;
  RMAppAttemptUnregistrationEvent unregisterEvent =
      (RMAppAttemptUnregistrationEvent) event;
  diagnostics.append(unregisterEvent.getDiagnostics());
  originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
  finalStatus = unregisterEvent.getFinalApplicationStatus();
}
 
Example 10
Source Project: big-c   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
private void unregisterApplicationAttempt(Container container,
    FinalApplicationStatus finalStatus, String trackingUrl,
    String diagnostics) {
  applicationAttempt.handle(
      new RMAppAttemptUnregistrationEvent(
          applicationAttempt.getAppAttemptId(),
          trackingUrl, finalStatus, diagnostics));
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishingState(container, finalStatus,
      trackingUrl, diagnostics);
}
 
Example 11
Source Project: big-c   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
private void testUnmanagedAMSuccess(String url) {
  unmanagedAM = true;
  when(submissionContext.getUnmanagedAM()).thenReturn(true);
  // submit AM and check it goes to LAUNCHED state
  scheduleApplicationAttempt();
  testAppAttemptLaunchedState(null);
  verify(amLivelinessMonitor, times(1)).register(
      applicationAttempt.getAppAttemptId());

  // launch AM
  runApplicationAttempt(null, "host", 8042, url, true);

  // complete a container
  Container container = mock(Container.class);
  when(container.getNodeId()).thenReturn(NodeId.newInstance("host", 1234));
  application.handle(new RMAppRunningOnNodeEvent(application.getApplicationId(),
      container.getNodeId()));
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class),
      container.getNodeId()));
  // complete AM
  String diagnostics = "Successful";
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
      applicationAttempt.getAppAttemptId(), url, finalStatus,
      diagnostics));
  testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
      true);
  assertFalse(transferStateFromPreviousAttempt);
}
 
Example 12
Source Project: big-c   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
}
 
Example 13
Source Project: big-c   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void
    testFinalSavingToFinishedWithContainerFinished() {
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
  String trackingUrl = "mytrackingurl";
  String diagnostics = "Successful";
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
    applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
    diagnostics));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  assertEquals(YarnApplicationAttemptState.RUNNING,
      applicationAttempt.createApplicationAttemptState());
  // Container_finished event comes before Attempt_Saved event.
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
    applicationAttempt.getAppAttemptId(), BuilderUtils.newContainerStatus(
      amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));
  assertEquals(RMAppAttemptState.FINAL_SAVING,
    applicationAttempt.getAppAttemptState());
  // send attempt_saved
  sendAttemptUpdateSavedEvent(applicationAttempt);
  testAppAttemptFinishedState(amContainer, finalStatus, trackingUrl,
    diagnostics, 0, false);
}
 
Example 14
private synchronized void initResourceManager(int index, Configuration conf) {
    if (HAUtil.isHAEnabled(conf)) {
        conf.set(YarnConfiguration.RM_HA_ID, rmIds[index]);
    }

    if (conf.get(YarnConfiguration.RM_HOSTNAME) == null) {
        conf.set(YarnConfiguration.RM_HOSTNAME, "0.0.0.0");
    }
    LOG.info("*** "+YarnConfiguration.RM_HOSTNAME+" is set to: "+conf.get(YarnConfiguration.RM_HOSTNAME));

    if (conf.get(YarnConfiguration.RM_ADDRESS) == null) {
        conf.set(YarnConfiguration.RM_ADDRESS,
                   YarnConfiguration.DEFAULT_RM_ADDRESS);
    }
    LOG.info("*** "+YarnConfiguration.RM_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_ADDRESS));

    if (conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS) == null) {
        WebAppUtils
            .setNMWebAppHostNameAndPort(conf,
                                        MiniYARNClusterSplice.getHostname(), 0);
    }
    LOG.info("*** "+YarnConfiguration.RM_WEBAPP_ADDRESS+" is set to: "+conf.get(YarnConfiguration.RM_WEBAPP_ADDRESS));

    resourceManagers[index].init(conf);
    resourceManagers[index].getRMContext().getDispatcher().register(
        RMAppAttemptEventType.class,
        new EventHandler<RMAppAttemptEvent>() {
            public void handle(RMAppAttemptEvent event) {
                if (event instanceof RMAppAttemptRegistrationEvent) {
                    appMasters.put(event.getApplicationAttemptId(),
                                   event.getTimestamp());
                } else if (event instanceof RMAppAttemptUnregistrationEvent) {
                    appMasters.remove(event.getApplicationAttemptId());
                }
            }
        });
}
 
Example 15
Source Project: hadoop   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 4 votes vote down vote up
private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
    Object transitionToDo, RMAppAttemptState targetFinalState,
    RMAppAttemptState stateToBeStored) {

  rememberTargetTransitions(event, transitionToDo, targetFinalState);
  stateBeforeFinalSaving = getState();

  // As of today, finalState, diagnostics, final-tracking-url and
  // finalAppStatus are the only things that we store into the StateStore
  // AFTER the initial saving on app-attempt-start
  // These fields can be visible from outside only after they are saved in
  // StateStore
  String diags = null;

  // don't leave the tracking URL pointing to a non-existent AM
  setTrackingUrlToRMAppPage(stateToBeStored);
  String finalTrackingUrl = getOriginalTrackingUrl();
  FinalApplicationStatus finalStatus = null;
  int exitStatus = ContainerExitStatus.INVALID;
  switch (event.getType()) {
  case LAUNCH_FAILED:
    RMAppAttemptLaunchFailedEvent launchFaileEvent =
        (RMAppAttemptLaunchFailedEvent) event;
    diags = launchFaileEvent.getMessage();
    break;
  case REGISTERED:
    diags = getUnexpectedAMRegisteredDiagnostics();
    break;
  case UNREGISTERED:
    RMAppAttemptUnregistrationEvent unregisterEvent =
        (RMAppAttemptUnregistrationEvent) event;
    diags = unregisterEvent.getDiagnostics();
    // reset finalTrackingUrl to url sent by am
    finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
    finalStatus = unregisterEvent.getFinalApplicationStatus();
    break;
  case CONTAINER_FINISHED:
    RMAppAttemptContainerFinishedEvent finishEvent =
        (RMAppAttemptContainerFinishedEvent) event;
    diags = getAMContainerCrashedDiagnostics(finishEvent);
    exitStatus = finishEvent.getContainerStatus().getExitStatus();
    break;
  case KILL:
    break;
  case EXPIRE:
    diags = getAMExpiredDiagnostics(event);
    break;
  default:
    break;
  }
  AggregateAppResourceUsage resUsage =
      this.attemptMetrics.getAggregateAppResourceUsage();
  RMStateStore rmStore = rmContext.getStateStore();
  setFinishTime(System.currentTimeMillis());

  ApplicationAttemptStateData attemptState =
      ApplicationAttemptStateData.newInstance(
          applicationAttemptId,  getMasterContainer(),
          rmStore.getCredentialsFromAppAttempt(this),
          startTime, stateToBeStored, finalTrackingUrl, diags,
          finalStatus, exitStatus,
        getFinishTime(), resUsage.getMemorySeconds(),
        resUsage.getVcoreSeconds(), resUsage.getGcoreSeconds());
  LOG.info("Updating application attempt " + applicationAttemptId
      + " with final state: " + targetedFinalState + ", and exit status: "
      + exitStatus);
  rmStore.updateApplicationAttemptState(attemptState);
}
 
Example 16
Source Project: hadoop   Source File: ApplicationMasterService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}
 
Example 17
Source Project: hadoop   Source File: TestRMAppAttemptTransitions.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testUsageReport() {
  // scheduler has info on running apps
  ApplicationAttemptId attemptId = applicationAttempt.getAppAttemptId();
  ApplicationResourceUsageReport appResUsgRpt =
          mock(ApplicationResourceUsageReport.class);
  when(appResUsgRpt.getMemorySeconds()).thenReturn(123456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(55544L);
  when(appResUsgRpt.getGcoreSeconds()).thenReturn(55544L);
  when(scheduler.getAppResourceUsageReport(any(ApplicationAttemptId.class)))
  .thenReturn(appResUsgRpt);

  // start and finish the attempt
  Container amContainer = allocateApplicationAttempt();
  launchApplicationAttempt(amContainer);
  runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
  applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(attemptId,
      "", FinalApplicationStatus.SUCCEEDED, ""));

  // expect usage stats to come from the scheduler report
  ApplicationResourceUsageReport report = 
      applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(123456L, report.getMemorySeconds());
  Assert.assertEquals(55544L, report.getVcoreSeconds());
  Assert.assertEquals(55544L, report.getGcoreSeconds());

  // finish app attempt and remove it from scheduler 
  when(appResUsgRpt.getMemorySeconds()).thenReturn(223456L);
  when(appResUsgRpt.getVcoreSeconds()).thenReturn(75544L);
  when(appResUsgRpt.getGcoreSeconds()).thenReturn(75544L);
  sendAttemptUpdateSavedEvent(applicationAttempt);
  NodeId anyNodeId = NodeId.newInstance("host", 1234);
  applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
      attemptId, 
      ContainerStatus.newInstance(
          amContainer.getId(), ContainerState.COMPLETE, "", 0), anyNodeId));

  when(scheduler.getSchedulerAppInfo(eq(attemptId))).thenReturn(null);

  report = applicationAttempt.getApplicationResourceUsageReport();
  Assert.assertEquals(223456, report.getMemorySeconds());
  Assert.assertEquals(75544, report.getVcoreSeconds());
  Assert.assertEquals(75544, report.getGcoreSeconds());
}
 
Example 18
Source Project: big-c   Source File: RMAppAttemptImpl.java    License: Apache License 2.0 4 votes vote down vote up
private void rememberTargetTransitionsAndStoreState(RMAppAttemptEvent event,
    Object transitionToDo, RMAppAttemptState targetFinalState,
    RMAppAttemptState stateToBeStored) {

  rememberTargetTransitions(event, transitionToDo, targetFinalState);
  stateBeforeFinalSaving = getState();

  // As of today, finalState, diagnostics, final-tracking-url and
  // finalAppStatus are the only things that we store into the StateStore
  // AFTER the initial saving on app-attempt-start
  // These fields can be visible from outside only after they are saved in
  // StateStore
  String diags = null;

  // don't leave the tracking URL pointing to a non-existent AM
  setTrackingUrlToRMAppPage(stateToBeStored);
  String finalTrackingUrl = getOriginalTrackingUrl();
  FinalApplicationStatus finalStatus = null;
  int exitStatus = ContainerExitStatus.INVALID;
  switch (event.getType()) {
  case LAUNCH_FAILED:
    RMAppAttemptLaunchFailedEvent launchFaileEvent =
        (RMAppAttemptLaunchFailedEvent) event;
    diags = launchFaileEvent.getMessage();
    break;
  case REGISTERED:
    diags = getUnexpectedAMRegisteredDiagnostics();
    break;
  case UNREGISTERED:
    RMAppAttemptUnregistrationEvent unregisterEvent =
        (RMAppAttemptUnregistrationEvent) event;
    diags = unregisterEvent.getDiagnostics();
    // reset finalTrackingUrl to url sent by am
    finalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
    finalStatus = unregisterEvent.getFinalApplicationStatus();
    break;
  case CONTAINER_FINISHED:
    RMAppAttemptContainerFinishedEvent finishEvent =
        (RMAppAttemptContainerFinishedEvent) event;
    diags = getAMContainerCrashedDiagnostics(finishEvent);
    exitStatus = finishEvent.getContainerStatus().getExitStatus();
    break;
  case KILL:
    break;
  case EXPIRE:
    diags = getAMExpiredDiagnostics(event);
    break;
  default:
    break;
  }
  AggregateAppResourceUsage resUsage =
      this.attemptMetrics.getAggregateAppResourceUsage();
  RMStateStore rmStore = rmContext.getStateStore();
  setFinishTime(System.currentTimeMillis());

  ApplicationAttemptStateData attemptState =
      ApplicationAttemptStateData.newInstance(
          applicationAttemptId,  getMasterContainer(),
          rmStore.getCredentialsFromAppAttempt(this),
          startTime, stateToBeStored, finalTrackingUrl, diags,
          finalStatus, exitStatus,
        getFinishTime(), resUsage.getMemorySeconds(),
        resUsage.getVcoreSeconds());
  LOG.info("Updating application attempt " + applicationAttemptId
      + " with final state: " + targetedFinalState + ", and exit status: "
      + exitStatus);
  rmStore.updateApplicationAttemptState(attemptState);
}
 
Example 19
Source Project: big-c   Source File: ApplicationMasterService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public FinishApplicationMasterResponse finishApplicationMaster(
    FinishApplicationMasterRequest request) throws YarnException,
    IOException {

  ApplicationAttemptId applicationAttemptId =
      authorizeRequest().getApplicationAttemptId();
  ApplicationId appId = applicationAttemptId.getApplicationId();

  RMApp rmApp =
      rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
  // checking whether the app exits in RMStateStore at first not to throw
  // ApplicationDoesNotExistInCacheException before and after
  // RM work-preserving restart.
  if (rmApp.isAppFinalStateStored()) {
    LOG.info(rmApp.getApplicationId() + " unregistered successfully. ");
    return FinishApplicationMasterResponse.newInstance(true);
  }

  AllocateResponseLock lock = responseMap.get(applicationAttemptId);
  if (lock == null) {
    throwApplicationDoesNotExistInCacheException(applicationAttemptId);
  }

  // Allow only one thread in AM to do finishApp at a time.
  synchronized (lock) {
    if (!hasApplicationMasterRegistered(applicationAttemptId)) {
      String message =
          "Application Master is trying to unregister before registering for: "
              + appId;
      LOG.error(message);
      RMAuditLogger.logFailure(
          this.rmContext.getRMApps()
              .get(appId).getUser(),
          AuditConstants.UNREGISTER_AM, "", "ApplicationMasterService",
          message, appId,
          applicationAttemptId);
      throw new ApplicationMasterNotRegisteredException(message);
    }

    this.amLivelinessMonitor.receivedPing(applicationAttemptId);

    rmContext.getDispatcher().getEventHandler().handle(
        new RMAppAttemptUnregistrationEvent(applicationAttemptId, request
            .getTrackingUrl(), request.getFinalApplicationStatus(), request
            .getDiagnostics()));

    // For UnmanagedAMs, return true so they don't retry
    return FinishApplicationMasterResponse.newInstance(
        rmApp.getApplicationSubmissionContext().getUnmanagedAM());
  }
}