Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#getApplications()

The following examples show how to use org.apache.hadoop.yarn.client.api.YarnClient#getApplications() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: StramClientUtils.java From Bats with Apache License 2.0

6 votes

public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException
{
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING,
      YarnApplicationState.ACCEPTED,
      YarnApplicationState.NEW,
      YarnApplicationState.NEW_SAVING,
      YarnApplicationState.SUBMITTED));
  // see whether there is an app with the app name and user name running
  for (ApplicationReport app : applications) {
    if (!app.getApplicationId().toString().equals(excludeAppId)
        && app.getName().equals(appName)
        && app.getUser().equals(user)) {
      return app;
    }
  }
  return null;
}

Example 2

Source File: StramClientUtils.java From Bats with Apache License 2.0

6 votes

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}

Example 3

Source File: TestRMFailover.java From hadoop with Apache License 2.0

6 votes

private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}

Example 4

Source File: ProtocolHATestBase.java From hadoop with Apache License 2.0

6 votes

protected void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = createAndStartYarnClient(conf);
    try {
      Thread.sleep(100);
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e.getMessage());
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}

Example 5

Source File: TestRMFailover.java From big-c with Apache License 2.0

6 votes

private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}

Example 6

Source File: ProtocolHATestBase.java From big-c with Apache License 2.0

6 votes

protected void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = createAndStartYarnClient(conf);
    try {
      Thread.sleep(100);
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e.getMessage());
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}

Example 7

Source File: StramClientUtils.java From attic-apex-core with Apache License 2.0

6 votes

public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException
{
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING,
      YarnApplicationState.ACCEPTED,
      YarnApplicationState.NEW,
      YarnApplicationState.NEW_SAVING,
      YarnApplicationState.SUBMITTED));
  // see whether there is an app with the app name and user name running
  for (ApplicationReport app : applications) {
    if (!app.getApplicationId().toString().equals(excludeAppId)
        && app.getName().equals(appName)
        && app.getUser().equals(user)) {
      return app;
    }
  }
  return null;
}

Example 8

Source File: StramClientUtils.java From attic-apex-core with Apache License 2.0

6 votes

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}

Example 9

Source File: YARNSessionCapacitySchedulerITCase.java From Flink-CEPplus with Apache License 2.0

5 votes

private ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}

Example 10

Source File: YARNSessionCapacitySchedulerITCase.java From flink with Apache License 2.0

5 votes

private ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}

Example 11

Source File: YarnTestBase.java From flink with Apache License 2.0

5 votes

protected ApplicationReport getOnlyApplicationReport() throws IOException, YarnException {
	final YarnClient yarnClient = getYarnClient();
	checkState(yarnClient != null);

	final List<ApplicationReport> apps = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING));
	assertEquals(1, apps.size()); // Only one running
	return apps.get(0);
}

Example 12

Source File: OlapServerSubmitter.java From spliceengine with GNU Affero General Public License v3.0

5 votes

private ApplicationId findApplication(YarnClient yarnClient) throws IOException, YarnException {
    List<ApplicationReport> reports = yarnClient.getApplications(Collections.singleton("YARN"),
            EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.SUBMITTED, YarnApplicationState.ACCEPTED, YarnApplicationState.NEW, YarnApplicationState.NEW_SAVING));
    for (ApplicationReport ar : reports) {
        if (ar.getName().equals("OlapServer-"+queueName) && ar.getUser().equals(UserGroupInformation.getCurrentUser().getUserName())) {
            LOG.info("Found existing application: " + ar);
            return ar.getApplicationId();
        }
    }
    return null;
}

Example 13

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testSessionTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/sessiontimeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_SESSION_AM_DAG_SUBMIT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testSessionTimeout", tezConf, true);
    tezClient.start();

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Session timed out"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}

Example 14

Source File: StramUtils.java From Bats with Apache License 2.0

4 votes

public static List<ApplicationReport> getApexApplicationList(YarnClient yarnClient) throws IOException, YarnException
{
  return yarnClient.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED));
}

Example 15

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testAMClientHeartbeatTimeout() throws Exception {
  Path stagingDirPath = new Path("/tmp/timeout-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  YarnClient yarnClient = YarnClient.createYarnClient();

  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
    tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
    tezConf.setInt(TezConfiguration.TEZ_AM_CLIENT_HEARTBEAT_TIMEOUT_SECS, 5);
    TezClient tezClient = TezClient.create("testAMClientHeartbeatTimeout", tezConf, true);
    tezClient.start();
    tezClient.cancelAMKeepAlive(true);

    ApplicationId appId = tezClient.getAppMasterApplicationId();

    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;

    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);

    ApplicationReport report;
    while (true) {
      report = yarnClient.getApplicationReport(appId);
      if (report.getYarnApplicationState() == YarnApplicationState.FINISHED
          || report.getYarnApplicationState() == YarnApplicationState.FAILED
          || report.getYarnApplicationState() == YarnApplicationState.KILLED) {
        break;
      }
      Thread.sleep(1000);
    }
    // Add a sleep because YARN is not consistent in terms of reporting uptodate diagnostics
    Thread.sleep(2000);
    report = yarnClient.getApplicationReport(appId);
    LOG.info("App Report for appId=" + appId
        + ", report=" + report);
    Assert.assertTrue("Actual diagnostics: " + report.getDiagnostics(),
        report.getDiagnostics().contains("Client-to-AM Heartbeat timeout interval expired"));

  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}

Example 16

Source File: TestTezJobs.java From tez with Apache License 2.0

4 votes

@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  int numIterations = 2;
  String[] inputPaths = new String[numIterations];
  String[] outputPaths = new String[numIterations];
  Path[] outputDirs = new Path[numIterations];
  for (int i=0; i<numIterations; ++i) {
    String inputDirStr = "/tmp/owc-input-" + i + "/";
    inputPaths[i] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    generateOrderedWordCountInput(inputDir, remoteFs);
    String outputDirStr = "/tmp/owc-output-" + i + "/"; 
    outputPaths[i] = outputDirStr;
    Path outputDir = new Path(outputDirStr);
    outputDirs[i] = outputDir;
  }


  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  YarnClient yarnClient = YarnClient.createYarnClient();

  try {
    
    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();
    
    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    Assert.assertTrue(
        "SimpleSessionExample failed",
        job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
            StringUtils.join(",", outputPaths), "2" }, null) == 0);

    for (int i=0; i<numIterations; ++i) {
      verifyOutput(outputDirs[i], remoteFs);
    }
    
    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;
    
    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }

}

Example 17

Source File: EmrClusterJob.java From datacollector with Apache License 2.0

4 votes

@Override
public Properties getJobStatus(Properties jobProps) throws IOException {
  EMRJobConfig emrJobConfig = new EMRJobConfig(jobProps);
  Utils.checkNotNull(emrJobConfig.getClusterId(), "EMR Cluster Id");
  String state;
  String message = null;
  DescribeStepResult res = getEmrClient(emrClusterConfig).describeStep(new DescribeStepRequest().withClusterId(
      emrJobConfig.getClusterId()).withStepId(emrJobConfig.getStepId()));
  StepStatus status = res.getStep().getStatus();
  ApplicationId appId = null;
  LOG.debug(Utils.format("Status of step: {} is {}", emrJobConfig.getStepId(), status.getState()));
  if ("PENDING".equals(status.getState())) {
    state = status.getState();
    if (status.getStateChangeReason() != null) {
      message = status.getStateChangeReason().getMessage();
    }
  } else if (!"COMPLETED".equals(status.getState()) && !"RUNNING".equals(status.getState())) {
    state = status.getState();
    if (status.getFailureDetails() != null) {
      message = status.getFailureDetails().getReason();
    }
  } else {
    YarnClient yarnClient = getYarnClient(emrJobConfig.getClusterId(), emrClusterConfig);
    ApplicationReport report = null;
    try {
      for (ApplicationReport applicationReport : yarnClient.getApplications()) {
        if (applicationReport.getName().contains(emrJobConfig.getUniquePrefix())) {
          appId = applicationReport.getApplicationId();
          break;
        }
      }
      if (appId != null) {
        report = yarnClient.getApplicationReport(appId);
      }
    } catch (YarnException ex) {
      throw new IOException("Failed to fetch yarn app report " + ex);
    }
    if (report != null) {
      YarnApplicationState yarnState = report.getYarnApplicationState();
      FinalApplicationStatus finalApplicationStatus = report.getFinalApplicationStatus();
      LOG.info(Utils.format("Application state for app id: {} is {} ", appId, yarnState));
      state = yarnState.name();
      if (YarnApplicationState.FINISHED == yarnState) {
        // override with final application state if yarnState is FINISHED
        state = finalApplicationStatus.name();
      }
      message = report.getDiagnostics();
    } else {
      state = "STARTING"; // a situation where step was in RUNNING but yarn application not yet created.
      message = "Yarn application not yet created";
    }
  }
  EmrState emrJobState = new EmrState();
  emrJobState.setState(state);
  emrJobState.setMessage(message);
  emrJobState.setAppId(appId != null ? appId.toString() : null);
  return emrJobState.toProperties();
}

Example 18

Source File: StramUtils.java From attic-apex-core with Apache License 2.0

4 votes

public static List<ApplicationReport> getApexApplicationList(YarnClient yarnClient) throws IOException, YarnException
{
  return yarnClient.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED));
}