Java Code Examples for org.apache.hadoop.yarn.api.records.YarnApplicationState#FINISHED

The following examples show how to use org.apache.hadoop.yarn.api.records.YarnApplicationState#FINISHED . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YARNRunner.java    From hadoop with Apache License 2.0 7 votes vote down vote up
private void killUnFinishedApplication(ApplicationId appId)
    throws IOException {
  ApplicationReport application = null;
  try {
    application = resMgrDelegate.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
  if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
      || application.getYarnApplicationState() == YarnApplicationState.FAILED
      || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
    return;
  }
  killApplication(appId);
}
 
Example 2
Source File: ApexCli.java    From Bats with Apache License 2.0 6 votes vote down vote up
protected ApplicationReport getApplicationByName(String appName)
{
  if (appName == null) {
    throw new CliException("Invalid application name provided by user");
  }
  List<ApplicationReport> appList = getApplicationList();
  for (ApplicationReport ar : appList) {
    if ((ar.getName().equals(appName)) &&
        (ar.getYarnApplicationState() != YarnApplicationState.KILLED) &&
        (ar.getYarnApplicationState() != YarnApplicationState.FINISHED)) {
      LOG.debug("Application Name: {} Application ID: {} Application State: {}",
          ar.getName(), ar.getApplicationId().toString(), YarnApplicationState.FINISHED);
      return ar;
    }
  }
  return null;
}
 
Example 3
Source File: ApexCli.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
protected ApplicationReport getApplicationByName(String appName)
{
  if (appName == null) {
    throw new CliException("Invalid application name provided by user");
  }
  List<ApplicationReport> appList = getApplicationList();
  for (ApplicationReport ar : appList) {
    if ((ar.getName().equals(appName)) &&
        (ar.getYarnApplicationState() != YarnApplicationState.KILLED) &&
        (ar.getYarnApplicationState() != YarnApplicationState.FINISHED)) {
      LOG.debug("Application Name: {} Application ID: {} Application State: {}",
          ar.getName(), ar.getApplicationId().toString(), YarnApplicationState.FINISHED);
      return ar;
    }
  }
  return null;
}
 
Example 4
Source File: ApplicationCLI.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 
Example 5
Source File: RMServerUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example 6
Source File: ApplicationCLI.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 
Example 7
Source File: RMServerUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example 8
Source File: LocalClient.java    From tez with Apache License 2.0 6 votes vote down vote up
protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) {
  switch (dagAppMasterState) {
  case NEW:
    return YarnApplicationState.NEW;
  case INITED:
  case RECOVERING:
  case IDLE:
  case RUNNING:
    return YarnApplicationState.RUNNING;
  case SUCCEEDED:
    return YarnApplicationState.FINISHED;
  case FAILED:
    return YarnApplicationState.FAILED;
  case KILLED:
    return YarnApplicationState.KILLED;
  case ERROR:
    return YarnApplicationState.FAILED;
  default:
    return YarnApplicationState.SUBMITTED;
  }
}
 
Example 9
Source File: YARNITCase.java    From flink with Apache License 2.0 6 votes vote down vote up
private void waitApplicationFinishedElseKillIt(
		ApplicationId applicationId,
		Duration timeout,
		YarnClusterDescriptor yarnClusterDescriptor) throws Exception {
	Deadline deadline = Deadline.now().plus(timeout);
	YarnApplicationState state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();

	while (state != YarnApplicationState.FINISHED) {
		if (state == YarnApplicationState.FAILED || state == YarnApplicationState.KILLED) {
			Assert.fail("Application became FAILED or KILLED while expecting FINISHED");
		}

		if (deadline.isOverdue()) {
			yarnClusterDescriptor.killCluster(applicationId);
			Assert.fail("Application didn't finish before timeout");
		}

		sleep(sleepIntervalInMS);
		state = getYarnClient().getApplicationReport(applicationId).getYarnApplicationState();
	}
}
 
Example 10
Source File: Client.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
  boolean r = false;
  while (true) {
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      r = false;
      break;
    }

    ApplicationReport report = yarnClient.getApplicationReport(appId);
    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus status = report.getFinalApplicationStatus();

    if (state == YarnApplicationState.FINISHED) {
      if (status == FinalApplicationStatus.SUCCEEDED) {
        LOG.info("Completed sucessfully.");
        r = true;
        break;
      } else {
        LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue="
            + status.toString());
        r = false;
        break;
      }
    } else if (state == YarnApplicationState.KILLED || state == YarnApplicationState.FAILED) {
      LOG.info("Application errored out. YarnState=" + state.toString() + ", finalStatue="
          + status.toString());
      r = false;
      break;
    }
  }// while
  return r;
}
 
Example 11
Source File: Client.java    From Hi-WAY with Apache License 2.0 5 votes vote down vote up
/**
 * Monitor the submitted application for completion. Kill application if time expires.
 * 
 * @param appId
 *            Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId) throws YarnException, IOException {
	while (true) {
		// Check app status every 1 second.
		try {
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			System.out.println("Thread sleep in monitoring loop interrupted");
		}

		// Get application report for the appId we are interested in
		ApplicationReport report = yarnClient.getApplicationReport(appId);

		YarnApplicationState state = report.getYarnApplicationState();
		FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
		if (YarnApplicationState.FINISHED == state) {
			if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
				System.out.println("Application has completed successfully. Breaking monitoring loop");
				System.out.println(report.getDiagnostics());
				return true;
			}
			System.out.println("Application finished unsuccessfully." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
					+ ". Breaking monitoring loop");

			return false;
		} else if (YarnApplicationState.KILLED == state || YarnApplicationState.FAILED == state) {
			System.out.println("Application did not finish." + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
					+ ". Breaking monitoring loop");
			return false;
		}
		if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
			System.out.println("Reached client specified timeout for application. Killing application");
			forceKillApplication(appId);
			return false;
		}
	}
}
 
Example 12
Source File: DAGClientImpl.java    From tez with Apache License 2.0 5 votes vote down vote up
private void checkAndSetDagCompletionStatus() {
  ApplicationReport appReport = realClient.getApplicationReportInternal();
  if (appReport != null) {
    final YarnApplicationState appState = appReport.getYarnApplicationState();
    if (appState == YarnApplicationState.FINISHED || appState == YarnApplicationState.FAILED ||
        appState == YarnApplicationState.KILLED) {
      dagCompleted = true;
    }
  }
}
 
Example 13
Source File: Client.java    From hadoop-mini-clusters with Apache License 2.0 4 votes vote down vote up
public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 
Example 14
Source File: Client.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion. 
 * Kill application if time expires. 
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
    throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in 
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
        + ", appId=" + appId.getId()
        + ", clientToAMToken=" + report.getClientToAMToken()
        + ", appDiagnostics=" + report.getDiagnostics()
        + ", appMasterHost=" + report.getHost()
        + ", appQueue=" + report.getQueue()
        + ", appMasterRpcPort=" + report.getRpcPort()
        + ", appStartTime=" + report.getStartTime()
        + ", yarnAppState=" + report.getYarnApplicationState().toString()
        + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
        + ", appTrackingUrl=" + report.getTrackingUrl()
        + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;        
      }
      else {
        LOG.info("Application did finished unsuccessfully."
            + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
            + ". Breaking monitoring loop");
        return false;
      }			  
    }
    else if (YarnApplicationState.KILLED == state	
        || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
          + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
          + ". Breaking monitoring loop");
      return false;
    }			

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;				
    }
  }			

}
 
Example 15
Source File: UnmanagedAMLauncher.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 16
Source File: TezClient.java    From tez with Apache License 2.0 4 votes vote down vote up
private boolean isJobInTerminalState(YarnApplicationState yarnApplicationState) {
  return (yarnApplicationState == YarnApplicationState.FINISHED
      || yarnApplicationState == YarnApplicationState.FAILED
      || yarnApplicationState == YarnApplicationState.KILLED);
}
 
Example 17
Source File: YarnTestBase.java    From Flink-CEPplus with Apache License 2.0 4 votes vote down vote up
private static boolean isApplicationRunning(ApplicationReport app) {
	final YarnApplicationState yarnApplicationState = app.getYarnApplicationState();
	return yarnApplicationState != YarnApplicationState.FINISHED
		&& app.getYarnApplicationState() != YarnApplicationState.KILLED
		&& app.getYarnApplicationState() != YarnApplicationState.FAILED;
}
 
Example 18
Source File: UnmanagedAMLauncher.java    From big-c with Apache License 2.0 4 votes vote down vote up
public boolean run() throws IOException, YarnException {
  LOG.info("Starting Client");
  
  // Connect to ResourceManager
  rmClient.start();
  try {  
    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = rmClient.createApplication()
        .getApplicationSubmissionContext();
    ApplicationId appId = appContext.getApplicationId();

    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    ApplicationReport appReport =
        monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED,
          YarnApplicationState.KILLED, YarnApplicationState.FAILED,
          YarnApplicationState.FINISHED));

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      // Monitor the application attempt to wait for launch state
      ApplicationAttemptReport attemptReport =
          monitorCurrentAppAttempt(appId,
            YarnApplicationAttemptState.LAUNCHED);
      ApplicationAttemptId attemptId =
          attemptReport.getApplicationAttemptId();
      LOG.info("Launching AM with application attempt id " + attemptId);
      // launch AM
      launchAM(attemptId);
      // Monitor the application for end state
      appReport =
          monitorApplication(appId, EnumSet.of(YarnApplicationState.KILLED,
            YarnApplicationState.FAILED, YarnApplicationState.FINISHED));
    }

    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);
    
    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }
    
    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 19
Source File: InlineAM.java    From attic-apex-core with Apache License 2.0 4 votes vote down vote up
public boolean run() throws Exception
{
  LOG.info("Starting Client");

  // Connect to ResourceManager
  rmClient.start();
  try {
    // Get a new application id
    YarnClientApplication newApp = rmClient.createApplication();
    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    LOG.info("Setting up application submission context for ASM");
    ApplicationSubmissionContext appContext = Records
        .newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);
    // set the application name
    appContext.setApplicationName(appName);

    // Set the priority for the application master
    Priority pri = Records.newRecord(Priority.class);
    pri.setPriority(amPriority);
    appContext.setPriority(pri);

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue(amQueue);

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer = Records
        .newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);

    // unmanaged AM
    appContext.setUnmanagedAM(true);
    LOG.info("Setting unmanaged AM");

    // Submit the application to the applications manager
    LOG.info("Submitting application to ASM");
    rmClient.submitApplication(appContext);

    // Monitor the application to wait for launch state
    ApplicationReport appReport = monitorApplication(appId,
        EnumSet.of(YarnApplicationState.ACCEPTED));
    ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId();
    LOG.info("Launching application with id: " + attemptId);

    // launch AM
    runAM(attemptId);

    // Monitor the application for end state
    appReport = monitorApplication(appId, EnumSet.of(
        YarnApplicationState.KILLED, YarnApplicationState.FAILED,
        YarnApplicationState.FINISHED));
    YarnApplicationState appState = appReport.getYarnApplicationState();
    FinalApplicationStatus appStatus = appReport.getFinalApplicationStatus();

    LOG.info("App ended with state: " + appReport.getYarnApplicationState()
        + " and status: " + appStatus);

    boolean success;
    if (YarnApplicationState.FINISHED == appState
        && FinalApplicationStatus.SUCCEEDED == appStatus) {
      LOG.info("Application has completed successfully.");
      success = true;
    } else {
      LOG.info("Application did finished unsuccessfully." + " YarnState="
          + appState.toString() + ", FinalStatus=" + appStatus.toString());
      success = false;
    }

    return success;
  } finally {
    rmClient.stop();
  }
}
 
Example 20
Source File: Client.java    From metron with Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
        throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
            + ", appId=" + appId.getId()
            + ", clientToAMToken=" + report.getClientToAMToken()
            + ", appDiagnostics=" + report.getDiagnostics()
            + ", appMasterHost=" + report.getHost()
            + ", appQueue=" + report.getQueue()
            + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime()
            + ", yarnAppState=" + report.getYarnApplicationState().toString()
            + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
            + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if(YarnApplicationState.RUNNING == state) {
      LOG.info("Application is running...");
      return true;
    }
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      }
      else {
        LOG.info("Application did finished unsuccessfully."
                + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
                + ". Breaking monitoring loop");
        return false;
      }
    }
    else if (YarnApplicationState.KILLED == state
            || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
              + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
              + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;
    }
  }

}