Java Code Examples for org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt

private static void updateAttemptMetrics(RMContainerImpl container) {
  // If this is a preempted container, update preemption metrics
  Resource resource = container.getContainer().getResource();
  RMAppAttempt rmAttempt = container.rmContext.getRMApps()
      .get(container.getApplicationAttemptId().getApplicationId())
      .getCurrentAppAttempt();
  if (ContainerExitStatus.PREEMPTED == container.finishedStatus
    .getExitStatus()) {
    rmAttempt.getRMAppAttemptMetrics().updatePreemptionInfo(resource,
      container);
  }

  if (rmAttempt != null) {
    long usedMillis = container.finishTime - container.creationTime;
    long memorySeconds = resource.getMemory()
                          * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long vcoreSeconds = resource.getVirtualCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    long gcoreSeconds = resource.getGpuCores()
                         * usedMillis / DateUtils.MILLIS_PER_SECOND;
    rmAttempt.getRMAppAttemptMetrics()
              .updateAggregateAppResourceUsage(memorySeconds,vcoreSeconds, gcoreSeconds);
  }
}
 

protected ApplicationAttemptId createSchedulingRequest(String queueId,
    String userId, List<ResourceRequest> ask) {
  ApplicationAttemptId id = createAppAttemptId(this.APP_ID++,
      this.ATTEMPT_ID++);
  scheduler.addApplication(id.getApplicationId(), queueId, userId, false);
  // This conditional is for testAclSubmitApplication where app is rejected
  // and no app is added.
  if (scheduler.getSchedulerApplications().containsKey(id.getApplicationId())) {
    scheduler.addApplicationAttempt(id, false, false);
  }

  RMApp rmApp = mock(RMApp.class);
  RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
  when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
  when(rmAppAttempt.getRMAppAttemptMetrics()).thenReturn(
      new RMAppAttemptMetrics(id,resourceManager.getRMContext()));
  resourceManager.getRMContext().getRMApps()
      .put(id.getApplicationId(), rmApp);

  scheduler.allocate(id, ask, new ArrayList<ContainerId>(), null, null);
  return id;
}
 

@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 

@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
    RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptFinishedEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getDiagnostics(),
            // app will get the final status from app attempt, or create one
            // based on app state if it doesn't exist
            app.getFinalApplicationStatus(),
            RMServerUtils.createApplicationAttemptState(appAttemtpState),
            finishedTime));
  }
}
 

public void verifyAppAttemptInfoGeneric(RMAppAttempt appAttempt, int id,
    long startTime, String containerId, String nodeHttpAddress, String nodeId,
    String logsLink, String user)
        throws JSONException, Exception {

  assertEquals("id doesn't match", appAttempt.getAppAttemptId()
      .getAttemptId(), id);
  assertEquals("startedTime doesn't match", appAttempt.getStartTime(),
      startTime);
  WebServicesTestUtils.checkStringMatch("containerId", appAttempt
      .getMasterContainer().getId().toString(), containerId);
  WebServicesTestUtils.checkStringMatch("nodeHttpAddress", appAttempt
      .getMasterContainer().getNodeHttpAddress(), nodeHttpAddress);
  WebServicesTestUtils.checkStringMatch("nodeId", appAttempt
      .getMasterContainer().getNodeId().toString(), nodeId);
  assertTrue("logsLink doesn't match", logsLink.startsWith("//"));
  assertTrue(
      "logsLink doesn't contain user info", logsLink.endsWith("/"
      + user));
}
 

protected ContainerId storeAttempt(RMStateStore store,
    ApplicationAttemptId attemptId,
    String containerIdStr, Token<AMRMTokenIdentifier> appToken,
    SecretKey clientTokenMasterKey, TestDispatcher dispatcher)
    throws Exception {

  RMAppAttemptMetrics mockRmAppAttemptMetrics = 
      mock(RMAppAttemptMetrics.class);
  Container container = new ContainerPBImpl();
  container.setId(ConverterUtils.toContainerId(containerIdStr));
  RMAppAttempt mockAttempt = mock(RMAppAttempt.class);
  when(mockAttempt.getAppAttemptId()).thenReturn(attemptId);
  when(mockAttempt.getMasterContainer()).thenReturn(container);
  when(mockAttempt.getAMRMToken()).thenReturn(appToken);
  when(mockAttempt.getClientTokenMasterKey())
      .thenReturn(clientTokenMasterKey);
  when(mockAttempt.getRMAppAttemptMetrics())
      .thenReturn(mockRmAppAttemptMetrics);
  when(mockRmAppAttemptMetrics.getAggregateAppResourceUsage())
      .thenReturn(new AggregateAppResourceUsage(0, 0, 0));
  dispatcher.attemptId = attemptId;
  store.storeNewApplicationAttempt(mockAttempt);
  waitNotify(dispatcher);
  return container.getId();
}
 

public AppAttemptInfo(RMAppAttempt attempt, String user) {
  this.startTime = 0;
  this.containerId = "";
  this.nodeHttpAddress = "";
  this.nodeId = "";
  this.logsLink = "";
  if (attempt != null) {
    this.id = attempt.getAppAttemptId().getAttemptId();
    this.startTime = attempt.getStartTime();
    Container masterContainer = attempt.getMasterContainer();
    if (masterContainer != null) {
      this.containerId = masterContainer.getId().toString();
      this.nodeHttpAddress = masterContainer.getNodeHttpAddress();
      this.nodeId = masterContainer.getNodeId().toString();
      this.logsLink =
          WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
              ConverterUtils.toString(masterContainer.getId()), user);
    }
  }
}
 

public void verifyAppAttemptsXML(NodeList nodes, RMAppAttempt appAttempt,
    String user)
    throws JSONException, Exception {

  for (int i = 0; i < nodes.getLength(); i++) {
    Element element = (Element) nodes.item(i);

    verifyAppAttemptInfoGeneric(appAttempt,
        WebServicesTestUtils.getXmlInt(element, "id"),
        WebServicesTestUtils.getXmlLong(element, "startTime"),
        WebServicesTestUtils.getXmlString(element, "containerId"),
        WebServicesTestUtils.getXmlString(element, "nodeHttpAddress"),
        WebServicesTestUtils.getXmlString(element, "nodeId"),
        WebServicesTestUtils.getXmlString(element, "logsLink"), user);
  }
}
 

@Override
public void handle(RMAppAttemptEvent event) {
  ApplicationAttemptId appAttemptID = event.getApplicationAttemptId();
  ApplicationId appAttemptId = appAttemptID.getApplicationId();
  RMApp rmApp = this.rmContext.getRMApps().get(appAttemptId);
  if (rmApp != null) {
    RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptID);
    if (rmAppAttempt != null) {
      try {
        rmAppAttempt.handle(event);
      } catch (Throwable t) {
        LOG.error("Error in handling event type " + event.getType()
            + " for applicationAttempt " + appAttemptId, t);
      }
    }
  }
}
 

@Override
public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() {
  this.readLock.lock();

  try {
    return Collections.unmodifiableMap(this.attempts);
  } finally {
    this.readLock.unlock();
  }
}
 

private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  return am;
}
 

@Override
public Map<ApplicationAttemptId, RMAppAttempt> getAppAttempts() {
  Map<ApplicationAttemptId, RMAppAttempt> attempts =
    new LinkedHashMap<ApplicationAttemptId, RMAppAttempt>();
  if(attempt != null) {
    attempts.put(attempt.getAppAttemptId(), attempt);
  }
  return attempts;
}
 

private RMAppAttempt getRMAppAttempt() {
  ApplicationId appId = this.appAttemptId.getApplicationId();
  RMAppAttempt attempt = null;
  RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
  if (rmApp != null) { 
    attempt = rmApp.getAppAttempts().get(appAttemptId);
  }
  return attempt;
}
 

private void waitForLaunchedState(RMAppAttempt attempt)
    throws InterruptedException {
  int waitCount = 0;
  while (attempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED
      && waitCount++ < 20) {
    LOG.info("Waiting for AppAttempt to reach LAUNCHED state. "
        + "Current state is " + attempt.getAppAttemptState());
    Thread.sleep(1000);
  }
  Assert.assertEquals(attempt.getAppAttemptState(),
      RMAppAttemptState.LAUNCHED);
}
 

@SuppressWarnings("unchecked")
public void appAttemptRegistered(RMAppAttempt appAttempt,
    long registeredTime) {
  if (publishSystemMetrics) {
    dispatcher.getEventHandler().handle(
        new AppAttemptRegisteredEvent(
            appAttempt.getAppAttemptId(),
            appAttempt.getHost(),
            appAttempt.getRpcPort(),
            appAttempt.getTrackingUrl(),
            appAttempt.getOriginalTrackingUrl(),
            appAttempt.getMasterContainer().getId(),
            registeredTime));
  }
}
 

protected MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
    throws Exception {
  RMAppAttempt attempt = app.getCurrentAppAttempt();
  nm.nodeHeartbeat(true);
  MockAM am = rm.sendAMLaunched(attempt.getAppAttemptId());
  am.registerAppAttempt();
  rm.waitForState(app.getApplicationId(), RMAppState.RUNNING);
  rm.waitForState(app.getCurrentAppAttempt().getAppAttemptId(),
      RMAppAttemptState.RUNNING);
  return am;
}
 

@Test (timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {  
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1_1 = rm1.submitApp(1024);
  MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
  
  RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
  AbstractYarnScheduler scheduler =
      ((AbstractYarnScheduler) rm1.getResourceScheduler());
  
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());

  // Re-start RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  List<NMContainerStatus> am1_1Containers =
      createNMContainerStatusForApp(am1_1);
  nm1.registerNode(am1_1Containers, null);

  // Wait for RM to settle down on recovering containers;
  waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

  scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());
}
 

/**
 * Non-blocking API
 * ResourceManager services call this to remove an application from the state
 * store
 * This does not block the dispatcher threads
 * There is no notification of completion for this operation.
 */
@SuppressWarnings("unchecked")
public synchronized void removeApplication(RMApp app) {
  ApplicationStateData appState =
      ApplicationStateData.newInstance(
          app.getSubmitTime(), app.getStartTime(),
          app.getApplicationSubmissionContext(), app.getUser());
  for(RMAppAttempt appAttempt : app.getAppAttempts().values()) {
    appState.attempts.put(appAttempt.getAppAttemptId(), null);
  }
  
  dispatcher.getEventHandler().handle(new RMStateStoreRemoveAppEvent(appState));
}
 

public AMLauncher(RMContext rmContext, RMAppAttempt application,
    AMLauncherEventType eventType, Configuration conf) {
  this.application = application;
  this.conf = conf;
  this.eventType = eventType;
  this.rmContext = rmContext;
  this.handler = rmContext.getDispatcher().getEventHandler();
  this.masterContainer = application.getMasterContainer();
}
 

private RMAppAttempt getRMAppAttempt() {
  ApplicationId appId = this.appAttemptId.getApplicationId();
  RMAppAttempt attempt = null;
  RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
  if (rmApp != null) { 
    attempt = rmApp.getAppAttempts().get(appAttemptId);
  }
  return attempt;
}
 

@Override
public synchronized void  handle(AMLauncherEvent appEvent) {
  AMLauncherEventType event = appEvent.getType();
  RMAppAttempt application = appEvent.getAppAttempt();
  switch (event) {
  case LAUNCH:
    launch(application);
    break;
  case CLEANUP:
    cleanup(application);
    break;
  default:
    break;
  }
}
 

@Override
public float getProgress() {
  RMAppAttempt attempt = this.currentAttempt;
  if (attempt != null) {
    return attempt.getProgress();
  }
  return 0;
}
 

@Override
public String getTrackingUrl() {
  RMAppAttempt attempt = this.currentAttempt;
  if (attempt != null) {
    return attempt.getTrackingUrl();
  }
  return null;
}
 

private void createNewAttempt() {
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
  RMAppAttempt attempt =
      new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
        submissionContext, conf,
        // The newly created attempt maybe last attempt if (number of
        // previously failed attempts(which should not include Preempted,
        // hardware error and NM resync) + 1) equal to the max-attempt
        // limit.
        maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq);
  attempts.put(appAttemptId, attempt);
  currentAttempt = attempt;
}
 

private void createNewAttempt() {
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(applicationId, attempts.size() + 1);
  RMAppAttempt attempt =
      new RMAppAttemptImpl(appAttemptId, rmContext, scheduler, masterService,
        submissionContext, conf,
        // The newly created attempt maybe last attempt if (number of
        // previously failed attempts(which should not include Preempted,
        // hardware error and NM resync) + 1) equal to the max-attempt
        // limit.
        maxAppAttempts == (getNumFailedAppAttempts() + 1), amReq);
  attempts.put(appAttemptId, attempt);
  currentAttempt = attempt;
}
 

/**
 * Helper method to handle received ContainerStatus. If this corresponds to
 * the completion of a master-container of a managed AM,
 * we call the handler for RMAppAttemptContainerFinishedEvent.
 */
@SuppressWarnings("unchecked")
@VisibleForTesting
void handleNMContainerStatus(NMContainerStatus containerStatus, NodeId nodeId) {
  ApplicationAttemptId appAttemptId =
      containerStatus.getContainerId().getApplicationAttemptId();
  RMApp rmApp =
      rmContext.getRMApps().get(appAttemptId.getApplicationId());
  if (rmApp == null) {
    LOG.error("Received finished container : "
        + containerStatus.getContainerId()
        + " for unknown application " + appAttemptId.getApplicationId()
        + " Skipping.");
    return;
  }

  if (rmApp.getApplicationSubmissionContext().getUnmanagedAM()) {
    if (LOG.isDebugEnabled()) {
      LOG.debug("Ignoring container completion status for unmanaged AM "
          + rmApp.getApplicationId());
    }
    return;
  }

  RMAppAttempt rmAppAttempt = rmApp.getRMAppAttempt(appAttemptId);
  Container masterContainer = rmAppAttempt.getMasterContainer();
  if (masterContainer.getId().equals(containerStatus.getContainerId())
      && containerStatus.getContainerState() == ContainerState.COMPLETE) {
    ContainerStatus status =
        ContainerStatus.newInstance(containerStatus.getContainerId(),
          containerStatus.getContainerState(), containerStatus.getDiagnostics(),
          containerStatus.getContainerExitStatus());
    // sending master container finished event.
    RMAppAttemptContainerFinishedEvent evt =
        new RMAppAttemptContainerFinishedEvent(appAttemptId, status,
            nodeId);
    rmContext.getDispatcher().getEventHandler().handle(evt);
  }
}
 

@Override
public float getProgress() {
  RMAppAttempt attempt = this.currentAttempt;
  if (attempt != null) {
    return attempt.getProgress();
  }
  return 0;
}
 

@Override
public GetApplicationAttemptsResponse getApplicationAttempts(
    GetApplicationAttemptsRequest request) throws YarnException, IOException {
  ApplicationId appId = request.getApplicationId();
  UserGroupInformation callerUGI;
  try {
    callerUGI = UserGroupInformation.getCurrentUser();
  } catch (IOException ie) {
    LOG.info("Error getting UGI ", ie);
    throw RPCUtil.getRemoteException(ie);
  }
  RMApp application = this.rmContext.getRMApps().get(appId);
  if (application == null) {
    // If the RM doesn't have the application, throw
    // ApplicationNotFoundException and let client to handle.
    throw new ApplicationNotFoundException("Application with id '" + appId
        + "' doesn't exist in RM.");
  }
  boolean allowAccess = checkAccess(callerUGI, application.getUser(),
      ApplicationAccessType.VIEW_APP, application);
  GetApplicationAttemptsResponse response = null;
  if (allowAccess) {
    Map<ApplicationAttemptId, RMAppAttempt> attempts = application
        .getAppAttempts();
    List<ApplicationAttemptReport> listAttempts = 
      new ArrayList<ApplicationAttemptReport>();
    Iterator<Map.Entry<ApplicationAttemptId, RMAppAttempt>> iter = attempts
        .entrySet().iterator();
    while (iter.hasNext()) {
      listAttempts.add(iter.next().getValue()
          .createApplicationAttemptReport());
    }
    response = GetApplicationAttemptsResponse.newInstance(listAttempts);
  } else {
    throw new YarnException("User " + callerUGI.getShortUserName()
        + " does not have privilage to see this aplication " + appId);
  }
  return response;
}
 

@Override
public RMAppMetrics getRMAppMetrics() {
  Resource resourcePreempted = Resource.newInstance(0, 0);
  int numAMContainerPreempted = 0;
  int numNonAMContainerPreempted = 0;
  long memorySeconds = 0;
  long vcoreSeconds = 0;
  for (RMAppAttempt attempt : attempts.values()) {
    if (null != attempt) {
      RMAppAttemptMetrics attemptMetrics =
          attempt.getRMAppAttemptMetrics();
      Resources.addTo(resourcePreempted,
          attemptMetrics.getResourcePreempted());
      numAMContainerPreempted += attemptMetrics.getIsPreempted() ? 1 : 0;
      numNonAMContainerPreempted +=
          attemptMetrics.getNumNonAMContainersPreempted();
      // getAggregateAppResourceUsage() will calculate resource usage stats
      // for both running and finished containers.
      AggregateAppResourceUsage resUsage =
          attempt.getRMAppAttemptMetrics().getAggregateAppResourceUsage();
      memorySeconds += resUsage.getMemorySeconds();
      vcoreSeconds += resUsage.getVcoreSeconds();
    }
  }

  return new RMAppMetrics(resourcePreempted,
      numNonAMContainerPreempted, numAMContainerPreempted,
      memorySeconds, vcoreSeconds);
}
 

@Test (timeout = 30000)
public void testAMContainerStatusWithRMRestart() throws Exception {  
  MemoryRMStateStore memStore = new MemoryRMStateStore();
  memStore.init(conf);
  rm1 = new MockRM(conf, memStore);
  rm1.start();
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
  nm1.registerNode();
  RMApp app1_1 = rm1.submitApp(1024);
  MockAM am1_1 = MockRM.launchAndRegisterAM(app1_1, rm1, nm1);
  
  RMAppAttempt attempt0 = app1_1.getCurrentAppAttempt();
  AbstractYarnScheduler scheduler =
      ((AbstractYarnScheduler) rm1.getResourceScheduler());
  
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());

  // Re-start RM
  rm2 = new MockRM(conf, memStore);
  rm2.start();
  nm1.setResourceTrackerService(rm2.getResourceTrackerService());

  List<NMContainerStatus> am1_1Containers =
      createNMContainerStatusForApp(am1_1);
  nm1.registerNode(am1_1Containers, null);

  // Wait for RM to settle down on recovering containers;
  waitForNumContainersToRecover(2, rm2, am1_1.getApplicationAttemptId());

  scheduler = ((AbstractYarnScheduler) rm2.getResourceScheduler());
  Assert.assertTrue(scheduler.getRMContainer(
      attempt0.getMasterContainer().getId()).isAMContainer());
}