Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#getApplicationReport()

The following examples show how to use org.apache.hadoop.yarn.client.api.YarnClient#getApplicationReport() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestYarnClient.java From big-c with Apache License 2.0

6 votes

private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
  throws Exception {
  try {
    long start = System.currentTimeMillis();
    ApplicationReport report = rmClient.getApplicationReport(appId);
    while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
      if (System.currentTimeMillis() - start > 20 * 1000) {
        throw new Exception("App '" + appId + 
          "' time out, failed to reach ACCEPTED state");
      }
      Thread.sleep(200);
      report = rmClient.getApplicationReport(appId);
    }
  } catch (Exception ex) {
    throw new Exception(ex);
  }
}

Example 2

Source File: LogsCLI.java From big-c with Apache License 2.0

6 votes

private int verifyApplicationState(ApplicationId appId) throws IOException,
    YarnException {
  YarnClient yarnClient = createYarnClient();

  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
      return -1;
    case ACCEPTED:
    case RUNNING:
    case FAILED:
    case FINISHED:
    case KILLED:
    default:
      break;

    }
  } finally {
    yarnClient.close();
  }
  return 0;
}

Example 3

Source File: LogsCLI.java From hadoop with Apache License 2.0

6 votes

private int verifyApplicationState(ApplicationId appId) throws IOException,
    YarnException {
  YarnClient yarnClient = createYarnClient();

  try {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    switch (appReport.getYarnApplicationState()) {
    case NEW:
    case NEW_SAVING:
    case SUBMITTED:
      return -1;
    case ACCEPTED:
    case RUNNING:
    case FAILED:
    case FINISHED:
    case KILLED:
    default:
      break;

    }
  } finally {
    yarnClient.close();
  }
  return 0;
}

Example 4

Source File: TestYarnClient.java From hadoop with Apache License 2.0

6 votes

private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
  throws Exception {
  try {
    long start = System.currentTimeMillis();
    ApplicationReport report = rmClient.getApplicationReport(appId);
    while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
      if (System.currentTimeMillis() - start > 20 * 1000) {
        throw new Exception("App '" + appId + 
          "' time out, failed to reach ACCEPTED state");
      }
      Thread.sleep(200);
      report = rmClient.getApplicationReport(appId);
    }
  } catch (Exception ex) {
    throw new Exception(ex);
  }
}

Example 5

Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0

5 votes

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}

Example 6

Source File: TezClientUtils.java From incubator-tez with Apache License 2.0

5 votes

static DAGClientAMProtocolBlockingPB getSessionAMProxy(YarnClient yarnClient,
    Configuration conf,
    ApplicationId applicationId) throws TezException, IOException {
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(
        applicationId);

    if(appReport == null) {
      throw new TezUncheckedException("Could not retrieve application report"
          + " from YARN, applicationId=" + applicationId);
    }
    YarnApplicationState appState = appReport.getYarnApplicationState();
    if(appState != YarnApplicationState.RUNNING) {
      if (appState == YarnApplicationState.FINISHED
          || appState == YarnApplicationState.KILLED
          || appState == YarnApplicationState.FAILED) {
        throw new SessionNotRunning("Application not running"
            + ", applicationId=" + applicationId
            + ", yarnApplicationState=" + appReport.getYarnApplicationState()
            + ", finalApplicationStatus="
            + appReport.getFinalApplicationStatus()
            + ", trackingUrl=" + appReport.getTrackingUrl());
      }
      return null;
    }
  } catch (YarnException e) {
    throw new TezException(e);
  }
  return getAMProxy(conf, appReport.getHost(),
      appReport.getRpcPort(), appReport.getClientToAMToken());
}

Example 7

Source File: FlinkYarnSessionCli.java From flink with Apache License 2.0

5 votes

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}

Example 8

Source File: Hadoop21YarnAppClient.java From twill with Apache License 2.0

5 votes

@Override
public YarnApplicationReport getReport() {
  YarnClient yarnClient = createYarnClient();
  try {
    return new Hadoop21YarnApplicationReport(yarnClient.getApplicationReport(appId));
  } catch (YarnException | IOException e) {
    throw new RuntimeException("Failed to get application report for " + appId, e);
  } finally {
    yarnClient.stop();
  }
}

Example 9

Source File: Client.java From TensorFlowOnYARN with Apache License 2.0

5 votes

static ClusterSpec getClusterSpec(YarnClient client, ApplicationId appId) throws Exception {
  ClusterSpec clusterSpec = ClusterSpec.empty();
  ApplicationReport report = client.getApplicationReport(appId);
  YarnApplicationState state = report.getYarnApplicationState();
  if (state.equals(YarnApplicationState.RUNNING)) {
    String hostname = report.getHost();
    int port = report.getRpcPort();
    TFApplicationRpc rpc = TFApplicationRpcClient.getInstance(hostname, port);
    String spec = rpc.getClusterSpec();
    if (spec != null) {
      clusterSpec = ClusterSpec.fromJsonString(spec);
    }
  }
  return clusterSpec;
}

Example 10

Source File: MiniAthenaXCluster.java From AthenaX with Apache License 2.0

5 votes

public static YarnApplicationState pollFinishedApplicationState(YarnClient client, ApplicationId appId)
    throws IOException, YarnException, InterruptedException {
  EnumSet<YarnApplicationState> finishedState = EnumSet.of(FINISHED, KILLED, FAILED);

  while (true) {
    ApplicationReport report = client.getApplicationReport(appId);
    YarnApplicationState state = report.getYarnApplicationState();
    if (finishedState.contains(state)) {
      return state;
    } else {
      Thread.sleep(250);
    }
  }
}

Example 11

Source File: FlinkYarnSessionCli.java From Flink-CEPplus with Apache License 2.0

5 votes

private void tryRetrieveAndLogApplicationReport(YarnClient yarnClient, ApplicationId yarnApplicationId) {
	ApplicationReport applicationReport;

	try {
		applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
	} catch (YarnException | IOException e) {
		LOG.info("Could not log the final application report.", e);
		applicationReport = null;
	}

	if (applicationReport != null) {
		logApplicationReport(applicationReport);
	}
}

Example 12

Source File: Client.java From XLearning with Apache License 2.0

4 votes

private static ApplicationReport getApplicationReport(ApplicationId appId, YarnClient yarnClient)
    throws YarnException, IOException {
  return yarnClient.getApplicationReport(appId);
}

Example 13

Source File: Client.java From hadoop-mini-clusters with Apache License 2.0

4 votes

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}

Example 14

Source File: EmrClusterJob.java From datacollector with Apache License 2.0

4 votes

@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}

Example 15

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
    tezClient.start();
    tezClient.cancelAMKeepAlive(true);

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}

Example 16

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
    tezClient.start();

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Session timed out"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}

Example 17

Source File: TestExceptionPropagation.java From tez with Apache License 2.0

4 votes

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}
 * 
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
  try {
    startMiniTezCluster();
    startNonSessionClient();

    ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
    LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
    DAG dag = createDAG(exLocation);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletion();
    String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
    LOG.info("Diagnostics:" + diagnostics);
    assertTrue(diagnostics.contains(exLocation.name()));

    // wait for app complete (unregisterApplicationMaster is done)
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(tezConf);
    yarnClient.start();
    Set<YarnApplicationState> FINAL_APPLICATION_STATES =
        EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED,
            YarnApplicationState.FINISHED);
    ApplicationReport appReport = null;
    while (true) {
      appReport = yarnClient.getApplicationReport(appId);
      Thread.sleep(1000);
      LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
      LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
      if (FINAL_APPLICATION_STATES.contains(appReport
          .getYarnApplicationState())) {
        break;
      }
    }
    // wait for 1 second and call getApplicationReport again to ensure get the
    // diagnostics
    // TODO remove it after YARN-2560
    Thread.sleep(1000);
    appReport = yarnClient.getApplicationReport(appId);

    LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
    LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
    assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
    // use "\n" as separator, because we also use it in Tez internally when
    // assembling the application diagnostics.
    assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(),
        appReport.getDiagnostics().trim());
  } finally {
    stopNonSessionClient();
    stopTezMiniCluster();
  }
}

Example 18

Source File: UnmanagedAmTest.java From reef with Apache License 2.0

2 votes

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}