Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationReport#getDiagnostics()

public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 

public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 

private ApplicationReport poll() throws IOException, YarnException {
  ApplicationReport report;
  report = yarnClient.getApplicationReport(appId);
  YarnApplicationState appState = report.getYarnApplicationState();
  LOG.debug("Application State: {}", appState);

  switch (appState) {
    case FAILED:
    case FINISHED:
      //TODO: the finished state may be valid in flip-6
    case KILLED:
      throw new IOException("The YARN application unexpectedly switched to state "
          + appState + " during deployment. \n"
          + "Diagnostics from YARN: " + report.getDiagnostics() + "\n"
          + "If log aggregation is enabled on your cluster, use this command to further investigate the issue:\n"
          + "yarn logs -applicationId " + appId);
      //break ..
    case RUNNING:
      LOG.info("YARN application has been deployed successfully.");
      break;
    default:
      if (appState != lastAppState) {
        LOG.info("Deploying cluster, current state " + appState);
      }
      lastAppState = appState;
      if (System.currentTimeMillis() - startTime > DEPLOY_TIMEOUT_MS) {
        throw new RuntimeException(String.format("Deployment took more than %d seconds. "
            + "Please check if the requested resources are available in the YARN cluster", DEPLOY_TIMEOUT_MS));
      }
      return null;
  }
  return report;
}
 

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
  
  addHistoryToken(ts);
  
  // Construct necessary information to start the MR AM
  ApplicationSubmissionContext appContext =
    createApplicationSubmissionContext(conf, jobSubmitDir, ts);

  // Submit to ResourceManager
  try {
    ApplicationId applicationId =
        resMgrDelegate.submitApplication(appContext);

    ApplicationReport appMaster = resMgrDelegate
        .getApplicationReport(applicationId);
    String diagnostics =
        (appMaster == null ?
            "application report is null" : appMaster.getDiagnostics());
    if (appMaster == null
        || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
        || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
      throw new IOException("Failed to run job : " +
          diagnostics);
    }
    return clientCache.getClient(jobId).getJobStatus(jobId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 

@Override
public JobStatus submitJob(JobID jobId, String jobSubmitDir, Credentials ts)
throws IOException, InterruptedException {
  
  addHistoryToken(ts);
  
  // Construct necessary information to start the MR AM
  ApplicationSubmissionContext appContext =
    createApplicationSubmissionContext(conf, jobSubmitDir, ts);

  // Submit to ResourceManager
  try {
    ApplicationId applicationId =
        resMgrDelegate.submitApplication(appContext);

    ApplicationReport appMaster = resMgrDelegate
        .getApplicationReport(applicationId);
    String diagnostics =
        (appMaster == null ?
            "application report is null" : appMaster.getDiagnostics());
    if (appMaster == null
        || appMaster.getYarnApplicationState() == YarnApplicationState.FAILED
        || appMaster.getYarnApplicationState() == YarnApplicationState.KILLED) {
      throw new IOException("Failed to run job : " +
          diagnostics);
    }
    return clientCache.getClient(jobId).getJobStatus(jobId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
}
 

@Override
 public ApplicationId submitApplication(ApplicationSubmissionContext appSubmissionContext)
     throws YarnException, IOException, TezException {
ApplicationId appId= yarnClient.submitApplication(appSubmissionContext);
   ApplicationReport appReport = getApplicationReport(appId);
   if (appReport.getYarnApplicationState() == YarnApplicationState.FAILED){
     throw new TezException("Failed to submit application to YARN"
         + ", applicationId=" + appId
         + ", diagnostics=" + appReport.getDiagnostics());
   }
   return appId;
 }
 

@Override
public ApplicationId
    submitApplication(ApplicationSubmissionContext appContext)
        throws YarnException, IOException {
  ApplicationId applicationId = appContext.getApplicationId();
  if (applicationId == null) {
    throw new ApplicationIdNotProvidedException(
        "ApplicationId is not provided in ApplicationSubmissionContext");
  }
  SubmitApplicationRequest request =
      Records.newRecord(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(appContext);

  // Automatically add the timeline DT into the CLC
  // Only when the security and the timeline service are both enabled
  if (isSecurityEnabled() && timelineServiceEnabled) {
    addTimelineDelegationToken(appContext.getAMContainerSpec());
  }

  //TODO: YARN-1763:Handle RM failovers during the submitApplication call.
  rmClient.submitApplication(request);

  int pollCount = 0;
  long startTime = System.currentTimeMillis();
  EnumSet<YarnApplicationState> waitingStates = 
                               EnumSet.of(YarnApplicationState.NEW,
                               YarnApplicationState.NEW_SAVING,
                               YarnApplicationState.SUBMITTED);
  EnumSet<YarnApplicationState> failToSubmitStates = 
                                EnumSet.of(YarnApplicationState.FAILED,
                                YarnApplicationState.KILLED);		
  while (true) {
    try {
      ApplicationReport appReport = getApplicationReport(applicationId);
      YarnApplicationState state = appReport.getYarnApplicationState();
      if (!waitingStates.contains(state)) {
        if(failToSubmitStates.contains(state)) {
          throw new YarnException("Failed to submit " + applicationId + 
              " to YARN : " + appReport.getDiagnostics());
        }
        LOG.info("Submitted application " + applicationId);
        break;
      }

      long elapsedMillis = System.currentTimeMillis() - startTime;
      if (enforceAsyncAPITimeout() &&
          elapsedMillis >= asyncApiPollTimeoutMillis) {
        throw new YarnException("Timed out while waiting for application " +
            applicationId + " to be submitted successfully");
      }

      // Notify the client through the log every 10 poll, in case the client
      // is blocked here too long.
      if (++pollCount % 10 == 0) {
        LOG.info("Application submission is not finished, " +
            "submitted application " + applicationId +
            " is still in " + state);
      }
      try {
        Thread.sleep(submitPollIntervalMillis);
      } catch (InterruptedException ie) {
        LOG.error("Interrupted while waiting for application "
            + applicationId
            + " to be successfully submitted.");
      }
    } catch (ApplicationNotFoundException ex) {
      // FailOver or RM restart happens before RMStateStore saves
      // ApplicationState
      LOG.info("Re-submit application " + applicationId + "with the " +
          "same ApplicationSubmissionContext");
      rmClient.submitApplication(request);
    }
  }

  return applicationId;
}
 

@Override
public ApplicationId
    submitApplication(ApplicationSubmissionContext appContext)
        throws YarnException, IOException {
  ApplicationId applicationId = appContext.getApplicationId();
  if (applicationId == null) {
    throw new ApplicationIdNotProvidedException(
        "ApplicationId is not provided in ApplicationSubmissionContext");
  }
  SubmitApplicationRequest request =
      Records.newRecord(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(appContext);

  // Automatically add the timeline DT into the CLC
  // Only when the security and the timeline service are both enabled
  if (isSecurityEnabled() && timelineServiceEnabled) {
    addTimelineDelegationToken(appContext.getAMContainerSpec());
  }

  //TODO: YARN-1763:Handle RM failovers during the submitApplication call.
  rmClient.submitApplication(request);

  int pollCount = 0;
  long startTime = System.currentTimeMillis();
  EnumSet<YarnApplicationState> waitingStates = 
                               EnumSet.of(YarnApplicationState.NEW,
                               YarnApplicationState.NEW_SAVING,
                               YarnApplicationState.SUBMITTED);
  EnumSet<YarnApplicationState> failToSubmitStates = 
                                EnumSet.of(YarnApplicationState.FAILED,
                                YarnApplicationState.KILLED);		
  while (true) {
    try {
      ApplicationReport appReport = getApplicationReport(applicationId);
      YarnApplicationState state = appReport.getYarnApplicationState();
      if (!waitingStates.contains(state)) {
        if(failToSubmitStates.contains(state)) {
          throw new YarnException("Failed to submit " + applicationId + 
              " to YARN : " + appReport.getDiagnostics());
        }
        LOG.info("Submitted application " + applicationId);
        break;
      }

      long elapsedMillis = System.currentTimeMillis() - startTime;
      if (enforceAsyncAPITimeout() &&
          elapsedMillis >= asyncApiPollTimeoutMillis) {
        throw new YarnException("Timed out while waiting for application " +
            applicationId + " to be submitted successfully");
      }

      // Notify the client through the log every 10 poll, in case the client
      // is blocked here too long.
      if (++pollCount % 10 == 0) {
        LOG.info("Application submission is not finished, " +
            "submitted application " + applicationId +
            " is still in " + state);
      }
      try {
        Thread.sleep(submitPollIntervalMillis);
      } catch (InterruptedException ie) {
        LOG.error("Interrupted while waiting for application "
            + applicationId
            + " to be successfully submitted.");
      }
    } catch (ApplicationNotFoundException ex) {
      // FailOver or RM restart happens before RMStateStore saves
      // ApplicationState
      LOG.info("Re-submit application " + applicationId + "with the " +
          "same ApplicationSubmissionContext");
      rmClient.submitApplication(request);
    }
  }

  return applicationId;
}
 

@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}
 

DAGStatus getDAGStatusViaRM() throws TezException, IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId);
  }
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new TezException(e);
  }

  if(appReport == null) {
    throw new TezException("Unknown/Invalid appId: " + appId);
  }

  DAGStatusProto.Builder builder = DAGStatusProto.newBuilder();
  DAGStatus dagStatus = new DAGStatus(builder);
  DAGStatusStateProto dagState;
  switch (appReport.getYarnApplicationState()) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
    dagState = DAGStatusStateProto.DAG_SUBMITTED;
    break;
  case RUNNING:
    dagState = DAGStatusStateProto.DAG_RUNNING;
    break;
  case FAILED:
    dagState = DAGStatusStateProto.DAG_FAILED;
    break;
  case KILLED:
    dagState = DAGStatusStateProto.DAG_KILLED;
    break;
  case FINISHED:
    switch(appReport.getFinalApplicationStatus()) {
    case UNDEFINED:
    case FAILED:
      dagState = DAGStatusStateProto.DAG_FAILED;
      break;
    case KILLED:
      dagState = DAGStatusStateProto.DAG_KILLED;
      break;
    case SUCCEEDED:
      dagState = DAGStatusStateProto.DAG_SUCCEEDED;
      break;
    default:
      throw new TezUncheckedException("Encountered unknown final application"
        + " status from YARN"
        + ", appState=" + appReport.getYarnApplicationState()
        + ", finalStatus=" + appReport.getFinalApplicationStatus());
    }
    break;
  default:
    throw new TezUncheckedException("Encountered unknown application state"
        + " from YARN, appState=" + appReport.getYarnApplicationState());
  }

  builder.setState(dagState);
  if(appReport.getDiagnostics() != null) {
    builder.addAllDiagnostics(Collections.singleton(appReport.getDiagnostics()));
  }

  return dagStatus;
}