Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationReport#getFinalApplicationStatus()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationReport#getFinalApplicationStatus() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YarnAppLauncherImpl.java    From Bats with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isFinished()
{
  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport != null) {
      if (appReport.getFinalApplicationStatus() == null
          || appReport.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) {
        return false;
      }
    }
    return true;
  } catch (YarnException | IOException e) {
    throw Throwables.propagate(e);
  }
}
 
Example 2
Source File: AppInfo.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example 3
Source File: AppInfo.java    From big-c with Apache License 2.0 6 votes vote down vote up
public AppInfo(ApplicationReport app) {
  appId = app.getApplicationId().toString();
  if (app.getCurrentApplicationAttemptId() != null) {
    currentAppAttemptId = app.getCurrentApplicationAttemptId().toString();
  }
  user = app.getUser();
  queue = app.getQueue();
  name = app.getName();
  type = app.getApplicationType();
  host = app.getHost();
  rpcPort = app.getRpcPort();
  appState = app.getYarnApplicationState();
  diagnosticsInfo = app.getDiagnostics();
  trackingUrl = app.getTrackingUrl();
  originalTrackingUrl = app.getOriginalTrackingUrl();
  submittedTime = app.getStartTime();
  startedTime = app.getStartTime();
  finishedTime = app.getFinishTime();
  elapsedTime = Times.elapsed(startedTime, finishedTime);
  finalAppStatus = app.getFinalApplicationStatus();
  progress = app.getProgress() * 100; // in percent
  if (app.getApplicationTags() != null && !app.getApplicationTags().isEmpty()) {
    this.applicationTags = CSV_JOINER.join(app.getApplicationTags());
  }
}
 
Example 4
Source File: YarnAppLauncherImpl.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isFinished()
{
  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport != null) {
      if (appReport.getFinalApplicationStatus() == null
          || appReport.getFinalApplicationStatus() == FinalApplicationStatus.UNDEFINED) {
        return false;
      }
    }
    return true;
  } catch (YarnException | IOException e) {
    throw Throwables.propagate(e);
  }
}
 
Example 5
Source File: Client.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
  boolean r = false;
  while (true) {
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      r = false;
      break;
    }

    ApplicationReport report = yarnClient.getApplicationReport(appId);
    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus status = report.getFinalApplicationStatus();

    if (state == YarnApplicationState.FINISHED) {
      if (status == FinalApplicationStatus.SUCCEEDED) {
        LOG.info("Completed sucessfully.");
        r = true;
        break;
      } else {
        LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue="
            + status.toString());
        r = false;
        break;
      }
    } else if (state == YarnApplicationState.KILLED || state == YarnApplicationState.FAILED) {
      LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue="
          + status.toString());
      r = false;
      break;
    }
  }// while
  return r;
}
 
Example 6
Source File: Client.java    From Hi-WAY with Apache License 2.0 5 votes vote down vote up
/**
 * Monitor the submitted application for completion. Kill application if time expires.
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
	while (true) {
		// Check app status every 1 second.
		try {
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			System.out.println("Thread sleep in monitoring loop interrupted");
		}

		// Get application report for the appId we are interested in
		ApplicationReport report = yarnClient.getApplicationReport(appId);

		YarnApplicationState state = report.getYarnApplicationState();
		FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
		if (YarnApplicationState.FINISHED == state) {
			if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
				System.out.println("Application has completed successfully. Breaking monitoring loop");
				System.out.println(report.getDiagnostics());
				return true;
			}
			System.out.println("Application finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
					+ ". Breaking monitoring loop");

			return false;
		} else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
			System.out.println("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
					+ ". Breaking monitoring loop");
			return false;
		}
		if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
			System.out.println("Reached client specified timeout for application. Killing application");
			forceKillApplication(appId);
			return false;
		}
	}
}
 
Example 7
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
@Subscribe
public void handleApplicationReportArrivalEvent(ApplicationReportArrivalEvent applicationReportArrivalEvent) {
  ApplicationReport applicationReport = applicationReportArrivalEvent.getApplicationReport();

  YarnApplicationState appState = applicationReport.getYarnApplicationState();
  LOGGER.info("Gobblin Yarn application state: " + appState.toString());

  // Reset the count on failures to get the ApplicationReport when there's one success
  this.getApplicationReportFailureCount.set(0);

  if (appState == YarnApplicationState.FINISHED ||
      appState == YarnApplicationState.FAILED ||
      appState == YarnApplicationState.KILLED) {

    applicationCompleted = true;

    LOGGER.info("Gobblin Yarn application finished with final status: " +
        applicationReport.getFinalApplicationStatus().toString());
    if (applicationReport.getFinalApplicationStatus() == FinalApplicationStatus.FAILED) {
      LOGGER.error("Gobblin Yarn application failed for the following reason: " + applicationReport.getDiagnostics());
    }

    try {
      GobblinYarnAppLauncher.this.stop();
    } catch (IOException ioe) {
      LOGGER.error("Failed to close the " + GobblinYarnAppLauncher.class.getSimpleName(), ioe);
    } catch (TimeoutException te) {
      LOGGER.error("Timeout in stopping the service manager", te);
    } finally {
      if (this.emailNotificationOnShutdown) {
        sendEmailOnShutdown(Optional.of(applicationReport));
      }
    }
  }
}
 
Example 8
Source File: StramClientUtils.java    From Bats with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion. Kill application if time expires.
 *
 * @param appId         Application Id of application to be monitored
 * @param callback
 * @param timeoutMillis
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings("SleepWhileInLoop")
public boolean waitForCompletion(ApplicationId appId, AppStatusCallback callback, long timeoutMillis) throws YarnException, IOException
{
  long startMillis = System.currentTimeMillis();
  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    ApplicationReport report = clientRM.getApplicationReport(appId);
    if (callback.exitLoop(report) == true) {
      return true;
    }

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      } else {
        LOG.info("Application finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }
    } else if (YarnApplicationState.KILLED == state
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() - startMillis > timeoutMillis) {
      LOG.info("Reached specified timeout. Killing application");
      clientRM.killApplication(appId);
      return false;
    }
  }
}
 
Example 9
Source File: Client.java    From metron with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
        throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
            + ", appId=" + appId.getId()
            + ", clientToAMToken=" + report.getClientToAMToken()
            + ", appDiagnostics=" + report.getDiagnostics()
            + ", appMasterHost=" + report.getHost()
            + ", appQueue=" + report.getQueue()
            + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime()
            + ", yarnAppState=" + report.getYarnApplicationState().toString()
            + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
            + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if(YarnApplicationState.RUNNING == state) {
      LOG.info("Application is running...");
      return true;
    }
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      }
      else {
        LOG.info("Application did finished unsuccessfully."
                + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
                + ". Breaking monitoring loop");
        return false;
      }
    }
    else if (YarnApplicationState.KILLED == state
            || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
              + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
              + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;
    }
  }

}
 
Example 10
Source File: EmrClusterJob.java    From datacollector with Apache License 2.0 4 votes vote down vote up
@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}
 
Example 11
Source File: InlineAM.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
public boolean run() throws Exception
{
  LOG.info("Starting Client");

  // Connect to ResourceManager
  rmClient.start();
  try {
    // Get a new application id
    YarnClientApplication newApp = rmClient.createApplication();
    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = Records
        .newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);
    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    // Monitor the application to wait for launch state
    ApplicationReport appReport = monitorApplication(appId,
        EnumSet.of(YarnApplicationState.ACCEPTED));
    ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
    LOG.info("Launching application with id: " + attemptId);

    // launch AM
    runAM(attemptId);

    // Monitor the application for end state
    appReport = monitorApplication(appId, EnumSet.of(
        YarnApplicationState.KILLED, YarnApplicationState.FAILED,
        YarnApplicationState.FINISHED));
    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);

    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }

    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 12
Source File: YarnScribeConsumerManager.java    From Scribengin with GNU Affero General Public License v3.0 4 votes vote down vote up
@Override 
public void monitorConsumers(){
  LinkedList<ScribeConsumerConfig> toAdd = new LinkedList<ScribeConsumerConfig>();
  synchronized(yarnApps){
    Iterator<YarnInfo> it = yarnApps.iterator();
    
    while (it.hasNext()) {
      YarnInfo yarnInfo = it.next();
      ApplicationReport report = null;
      Client c = null;
      try {
        c = yarnInfo.client;
        report = c.getYarnClient().getApplicationReport(c.getAppId());
      } catch (YarnException | IOException e) {
        e.printStackTrace();
      };
      YarnApplicationState state = report.getYarnApplicationState();
      FinalApplicationStatus status = report.getFinalApplicationStatus();

      if (state == YarnApplicationState.FINISHED || state == YarnApplicationState.KILLED ) {
        if (status == FinalApplicationStatus.SUCCEEDED) {
          LOG.info("Application completed successfully: "+c.getAppId().toString());
          it.remove();
          break;
        } 
        else if(state == YarnApplicationState.ACCEPTED){
          //Do nothing
        }
        else {
          LOG.info("Application finished but errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString() + ", AppId: "+c.getAppId().toString());
          yarnInfo.conf.cleanStart = false;
          toAdd.add(new ScribeConsumerConfig(yarnInfo.conf));
          it.remove();
          break;
        }
      } else if (state == YarnApplicationState.FAILED) {
        LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue=" + status.toString() + ", AppId: "+c.getAppId().toString());
        it.remove();
        break;
      }
    }
  }
  
  
  for(ScribeConsumerConfig sci: toAdd){
    startNewConsumer(sci);
  }
}
 
Example 13
Source File: StramClientUtils.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion. Kill application if time expires.
 *
 * @param appId         Application Id of application to be monitored
 * @param callback
 * @param timeoutMillis
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
@SuppressWarnings("SleepWhileInLoop")
public boolean waitForCompletion(ApplicationId appId, AppStatusCallback callback, long timeoutMillis) throws YarnException, IOException
{
  long startMillis = System.currentTimeMillis();
  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    ApplicationReport report = clientRM.getApplicationReport(appId);
    if (callback.exitLoop(report) == true) {
      return true;
    }

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      } else {
        LOG.info("Application finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }
    } else if (YarnApplicationState.KILLED == state
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() - startMillis > timeoutMillis) {
      LOG.info("Reached specified timeout. Killing application");
      clientRM.killApplication(appId);
      return false;
    }
  }
}
 
Example 14
Source File: UnmanagedAMLauncher.java    From big-c with Apache License 2.0 4 votes vote down vote up
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 15
Source File: DAGClientRPCImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
DAGStatus getDAGStatusViaRM() throws TezException, IOException {
  if(LOG.isDebugEnabled()) {
    LOG.debug("GetDAGStatus via AM for app: " + appId + " dag:" + dagId);
  }
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new TezException(e);
  }

  if(appReport == null) {
    throw new TezException("Unknown/Invalid appId: " + appId);
  }

  DAGStatusProto.Builder builder = DAGStatusProto.newBuilder();
  DAGStatus dagStatus = new DAGStatus(builder);
  DAGStatusStateProto dagState;
  switch (appReport.getYarnApplicationState()) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
    dagState = DAGStatusStateProto.DAG_SUBMITTED;
    break;
  case RUNNING:
    dagState = DAGStatusStateProto.DAG_RUNNING;
    break;
  case FAILED:
    dagState = DAGStatusStateProto.DAG_FAILED;
    break;
  case KILLED:
    dagState = DAGStatusStateProto.DAG_KILLED;
    break;
  case FINISHED:
    switch(appReport.getFinalApplicationStatus()) {
    case UNDEFINED:
    case FAILED:
      dagState = DAGStatusStateProto.DAG_FAILED;
      break;
    case KILLED:
      dagState = DAGStatusStateProto.DAG_KILLED;
      break;
    case SUCCEEDED:
      dagState = DAGStatusStateProto.DAG_SUCCEEDED;
      break;
    default:
      throw new TezUncheckedException("Encountered unknown final application"
        + " status from YARN"
        + ", appState=" + appReport.getYarnApplicationState()
        + ", finalStatus=" + appReport.getFinalApplicationStatus());
    }
    break;
  default:
    throw new TezUncheckedException("Encountered unknown application state"
        + " from YARN, appState=" + appReport.getYarnApplicationState());
  }

  builder.setState(dagState);
  if(appReport.getDiagnostics() != null) {
    builder.addAllDiagnostics(Collections.singleton(appReport.getDiagnostics()));
  }

  return dagStatus;
}
 
Example 16
Source File: UnmanagedAMLauncher.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 17
Source File: Client.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. 
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
    throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in 
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
        + ", appId=" + appId.getId()
        + ", clientToAMToken=" + report.getClientToAMToken()
        + ", appDiagnostics=" + report.getDiagnostics()
        + ", appMasterHost=" + report.getHost()
        + ", appQueue=" + report.getQueue()
        + ", appMasterRpcPort=" + report.getRpcPort()
        + ", appStartTime=" + report.getStartTime()
        + ", yarnAppState=" + report.getYarnApplicationState().toString()
        + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
        + ", appTrackingUrl=" + report.getTrackingUrl()
        + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;        
      }
      else {
        LOG.info("Application did finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }			  
    }
    else if (YarnApplicationState.KILLED == state	
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }			

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;				
    }
  }			

}
 
Example 18
Source File: AbstractYarnClusterDescriptor.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterClient<ApplicationId> retrieve(ApplicationId applicationId) throws ClusterRetrieveException {

	try {
		// check if required Hadoop environment variables are set. If not, warn user
		if (System.getenv("HADOOP_CONF_DIR") == null &&
			System.getenv("YARN_CONF_DIR") == null) {
			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." +
				"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
				"configuration for accessing YARN.");
		}

		final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

		if (appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
			// Flink cluster is not running anymore
			LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}",
				applicationId, appReport.getFinalApplicationStatus());
			throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore.");
		}

		final String host = appReport.getHost();
		final int rpcPort = appReport.getRpcPort();

		LOG.info("Found application JobManager host name '{}' and port '{}' from supplied application id '{}'",
			host, rpcPort, applicationId);

		flinkConfiguration.setString(JobManagerOptions.ADDRESS, host);
		flinkConfiguration.setInteger(JobManagerOptions.PORT, rpcPort);

		flinkConfiguration.setString(RestOptions.ADDRESS, host);
		flinkConfiguration.setInteger(RestOptions.PORT, rpcPort);

		return createYarnClusterClient(
			this,
			-1, // we don't know the number of task managers of a started Flink cluster
			-1, // we don't know how many slots each task manager has for a started Flink cluster
			appReport,
			flinkConfiguration,
			false);
	} catch (Exception e) {
		throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e);
	}
}
 
Example 19
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
@Override
public ClusterClient<ApplicationId> retrieve(ApplicationId applicationId) throws ClusterRetrieveException {

	try {
		// check if required Hadoop environment variables are set. If not, warn user
		if (System.getenv("HADOOP_CONF_DIR") == null &&
			System.getenv("YARN_CONF_DIR") == null) {
			LOG.warn("Neither the HADOOP_CONF_DIR nor the YARN_CONF_DIR environment variable is set." +
				"The Flink YARN Client needs one of these to be set to properly load the Hadoop " +
				"configuration for accessing YARN.");
		}

		final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

		if (appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
			// Flink cluster is not running anymore
			LOG.error("The application {} doesn't run anymore. It has previously completed with final status: {}",
				applicationId, appReport.getFinalApplicationStatus());
			throw new RuntimeException("The Yarn application " + applicationId + " doesn't run anymore.");
		}

		final String host = appReport.getHost();
		final int rpcPort = appReport.getRpcPort();

		LOG.info("Found application JobManager host name '{}' and port '{}' from supplied application id '{}'",
			host, rpcPort, applicationId);

		flinkConfiguration.setString(JobManagerOptions.ADDRESS, host);
		flinkConfiguration.setInteger(JobManagerOptions.PORT, rpcPort);

		flinkConfiguration.setString(RestOptions.ADDRESS, host);
		flinkConfiguration.setInteger(RestOptions.PORT, rpcPort);

		return createYarnClusterClient(
			this,
			-1, // we don't know the number of task managers of a started Flink cluster
			-1, // we don't know how many slots each task manager has for a started Flink cluster
			appReport,
			flinkConfiguration,
			false);
	} catch (Exception e) {
		throw new ClusterRetrieveException("Couldn't retrieve Yarn cluster", e);
	}
}
 
Example 20
Source File: UnmanagedAmTest.java    From reef with Apache License 2.0 2 votes vote down vote up
@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}