Java Code Examples for org.apache.hadoop.yarn.api.records.YarnApplicationState#RUNNING

The following examples show how to use org.apache.hadoop.yarn.api.records.YarnApplicationState#RUNNING . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   File: RMServerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example 2
Source Project: big-c   File: RMServerUtils.java    License: Apache License 2.0 6 votes vote down vote up
public static YarnApplicationState createApplicationState(
    RMAppState rmAppState) {
  switch (rmAppState) {
    case NEW:
      return YarnApplicationState.NEW;
    case NEW_SAVING:
      return YarnApplicationState.NEW_SAVING;
    case SUBMITTED:
      return YarnApplicationState.SUBMITTED;
    case ACCEPTED:
      return YarnApplicationState.ACCEPTED;
    case RUNNING:
      return YarnApplicationState.RUNNING;
    case FINISHING:
    case FINISHED:
      return YarnApplicationState.FINISHED;
    case KILLED:
      return YarnApplicationState.KILLED;
    case FAILED:
      return YarnApplicationState.FAILED;
    default:
      throw new YarnRuntimeException("Unknown state passed!");
    }
}
 
Example 3
Source Project: tez   File: LocalClient.java    License: Apache License 2.0 6 votes vote down vote up
protected YarnApplicationState convertDAGAppMasterState(DAGAppMasterState dagAppMasterState) {
  switch (dagAppMasterState) {
  case NEW:
    return YarnApplicationState.NEW;
  case INITED:
  case RECOVERING:
  case IDLE:
  case RUNNING:
    return YarnApplicationState.RUNNING;
  case SUCCEEDED:
    return YarnApplicationState.FINISHED;
  case FAILED:
    return YarnApplicationState.FAILED;
  case KILLED:
    return YarnApplicationState.KILLED;
  case ERROR:
    return YarnApplicationState.FAILED;
  default:
    return YarnApplicationState.SUBMITTED;
  }
}
 
Example 4
Source Project: hadoop   File: TestTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example 5
Source Project: twill   File: YarnTwillController.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void doStartUp() {
  super.doStartUp();

  // Submit and poll the status of the yarn application
  try {
    processController = startUp.call();

    YarnApplicationReport report = processController.getReport();
    ApplicationId appId = report.getApplicationId();
    LOG.info("Application {} with id {} submitted", appName, appId);

    YarnApplicationState state = report.getYarnApplicationState();
    Stopwatch stopWatch = new Stopwatch().start();

    LOG.debug("Checking yarn application status for {} {}", appName, appId);
    while (!hasRun(state) && stopWatch.elapsedTime(startTimeoutUnit) < startTimeout) {
      report = processController.getReport();
      state = report.getYarnApplicationState();
      LOG.debug("Yarn application status for {} {}: {}", appName, appId, state);
      TimeUnit.SECONDS.sleep(1);
    }
    LOG.info("Yarn application {} {} is in state {}", appName, appId, state);
    if (state != YarnApplicationState.RUNNING) {
      LOG.info("Yarn application {} {} is not in running state. Shutting down controller.", appName, appId);
      forceShutDown();
    }

    currentAttemptId = report.getCurrentApplicationAttemptId();
  } catch (Exception e) {
    throw Throwables.propagate(e);
  }
}
 
Example 6
Source Project: big-c   File: TestTypeConverter.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testFromYarn() throws Exception {
  int appStartTime = 612354;
  int appFinishTime = 612355;
  YarnApplicationState state = YarnApplicationState.RUNNING;
  ApplicationId applicationId = ApplicationId.newInstance(0, 0);
  ApplicationReport applicationReport = Records
      .newRecord(ApplicationReport.class);
  applicationReport.setApplicationId(applicationId);
  applicationReport.setYarnApplicationState(state);
  applicationReport.setStartTime(appStartTime);
  applicationReport.setFinishTime(appFinishTime);
  applicationReport.setUser("TestTypeConverter-user");
  ApplicationResourceUsageReport appUsageRpt = Records
      .newRecord(ApplicationResourceUsageReport.class);
  Resource r = Records.newRecord(Resource.class);
  r.setMemory(2048);
  appUsageRpt.setNeededResources(r);
  appUsageRpt.setNumReservedContainers(1);
  appUsageRpt.setNumUsedContainers(3);
  appUsageRpt.setReservedResources(r);
  appUsageRpt.setUsedResources(r);
  applicationReport.setApplicationResourceUsageReport(appUsageRpt);
  JobStatus jobStatus = TypeConverter.fromYarn(applicationReport, "dummy-jobfile");
  Assert.assertEquals(appStartTime, jobStatus.getStartTime());
  Assert.assertEquals(appFinishTime, jobStatus.getFinishTime());    
  Assert.assertEquals(state.toString(), jobStatus.getState().toString());
}
 
Example 7
/**
 * For some yet unknown reason, hostname resolution for the ResourceManager in {@link MiniYARNCluster}
 * has some issue that causes the {@link YarnClient} not be able to connect and submit the Gobblin Yarn
 * application successfully. This works fine on local machine though. So disabling this and the test
 * below that depends on it on Travis-CI.
 */
@Test(enabled=false, groups = { "disabledOnTravis" }, dependsOnMethods = "testCreateHelixCluster")
public void testSetupAndSubmitApplication() throws Exception {
  HelixUtils.createGobblinHelixCluster(
      this.config.getString(GobblinClusterConfigurationKeys.ZK_CONNECTION_STRING_KEY),
      this.config.getString(GobblinClusterConfigurationKeys.HELIX_CLUSTER_NAME_KEY));

  this.gobblinYarnAppLauncher.startYarnClient();
  this.applicationId = this.gobblinYarnAppLauncher.setupAndSubmitApplication();

  int i;

  // wait for application to come up
  for (i = 0; i < 120; i++) {
    if (yarnClient.getApplicationReport(applicationId).getYarnApplicationState() ==
        YarnApplicationState.RUNNING) {
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for RUNNING state");

  // wait another 10 seconds and check state again to make sure that the application stays up
  Thread.sleep(10000);

  Assert.assertEquals(yarnClient.getApplicationReport(applicationId).getYarnApplicationState(),
      YarnApplicationState.RUNNING, "Application may have aborted");
}
 
Example 8
Source Project: incubator-tez   File: TezClientUtils.java    License: Apache License 2.0 5 votes vote down vote up
static DAGClientAMProtocolBlockingPB getSessionAMProxy(YarnClient yarnClient,
    Configuration conf,
    ApplicationId applicationId) throws TezException, IOException {
  ApplicationReport appReport;
  try {
    appReport = yarnClient.getApplicationReport(
        applicationId);

    if(appReport == null) {
      throw new TezUncheckedException("Could not retrieve application report"
          + " from YARN, applicationId=" + applicationId);
    }
    YarnApplicationState appState = appReport.getYarnApplicationState();
    if(appState != YarnApplicationState.RUNNING) {
      if (appState == YarnApplicationState.FINISHED
          || appState == YarnApplicationState.KILLED
          || appState == YarnApplicationState.FAILED) {
        throw new SessionNotRunning("Application not running"
            + ", applicationId=" + applicationId
            + ", yarnApplicationState=" + appReport.getYarnApplicationState()
            + ", finalApplicationStatus="
            + appReport.getFinalApplicationStatus()
            + ", trackingUrl=" + appReport.getTrackingUrl());
      }
      return null;
    }
  } catch (YarnException e) {
    throw new TezException(e);
  }
  return getAMProxy(conf, appReport.getHost(),
      appReport.getRpcPort(), appReport.getClientToAMToken());
}
 
Example 9
Source Project: incubator-tez   File: DAGClientRPCImpl.java    License: Apache License 2.0 5 votes vote down vote up
boolean createAMProxyIfNeeded() throws IOException, TezException {
  if(proxy != null) {
    // if proxy exist optimistically use it assuming there is no retry
    return true;
  }
  appReport = getAppReport();

  if(appReport == null) {
    return false;
  }
  YarnApplicationState appState = appReport.getYarnApplicationState();
  if(appState != YarnApplicationState.RUNNING) {
    return false;
  }

  // YARN-808. Cannot ascertain if AM is ready until we connect to it.
  // workaround check the default string set by YARN
  if(appReport.getHost() == null || appReport.getHost().equals("N/A") ||
      appReport.getRpcPort() == 0){
    // attempt not running
    return false;
  }

  proxy = TezClientUtils.getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(),
      appReport.getClientToAMToken());
  return true;
}
 
Example 10
Source Project: tez   File: DAGClientRPCImpl.java    License: Apache License 2.0 5 votes vote down vote up
boolean createAMProxyIfNeeded() throws IOException, TezException,
    ApplicationNotFoundException {
  if(proxy != null) {
    // if proxy exist optimistically use it assuming there is no retry
    return true;
  }
  appReport = null;
  appReport = getAppReport();

  if(appReport == null) {
    return false;
  }
  YarnApplicationState appState = appReport.getYarnApplicationState();
  if(appState != YarnApplicationState.RUNNING) {
    return false;
  }

  // YARN-808. Cannot ascertain if AM is ready until we connect to it.
  // workaround check the default string set by YARN
  if(appReport.getHost() == null || appReport.getHost().equals("N/A") ||
      appReport.getRpcPort() == 0){
    // attempt not running
    return false;
  }

  proxy = TezClientUtils.getAMProxy(conf, appReport.getHost(), appReport.getRpcPort(),
      appReport.getClientToAMToken(), ugi);
  return true;
}
 
Example 11
Source Project: Bats   File: ApexCli.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void execute(String[] args, ConsoleReader reader) throws Exception
{
  try {
    JSONArray jsonArray = new JSONArray();
    List<ApplicationReport> appList = getApplicationList();
    Collections.sort(appList, new Comparator<ApplicationReport>()
    {
      @Override
      public int compare(ApplicationReport o1, ApplicationReport o2)
      {
        return o1.getApplicationId().getId() - o2.getApplicationId().getId();
      }

    });
    int totalCnt = 0;
    int runningCnt = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");

    for (ApplicationReport ar : appList) {
      /*
       * This is inefficient, but what the heck, if this can be passed through the command line, can anyone notice slowness.
       */
      JSONObject jsonObj = new JSONObject();
      jsonObj.put("startTime", sdf.format(new java.util.Date(ar.getStartTime())));
      jsonObj.put("id", ar.getApplicationId().getId());
      jsonObj.put("name", ar.getName());
      jsonObj.put("state", ar.getYarnApplicationState().name());
      jsonObj.put("trackingUrl", ar.getTrackingUrl());
      jsonObj.put("finalStatus", ar.getFinalApplicationStatus());
      JSONArray tags = new JSONArray();
      for (String tag : ar.getApplicationTags()) {
        tags.put(tag);
      }
      jsonObj.put("tags", tags);

      totalCnt++;
      if (ar.getYarnApplicationState() == YarnApplicationState.RUNNING) {
        runningCnt++;
      }

      if (args.length > 1) {
        if (StringUtils.isNumeric(args[1])) {
          if (jsonObj.getString("id").equals(args[1])) {
            jsonArray.put(jsonObj);
            break;
          }
        } else {
          @SuppressWarnings("unchecked")
          Iterator<String> keys = jsonObj.keys();
          while (keys.hasNext()) {
            if (jsonObj.get(keys.next()).toString().toLowerCase().contains(args[1].toLowerCase())) {
              jsonArray.put(jsonObj);
              break;
            }
          }
        }
      } else {
        jsonArray.put(jsonObj);
      }
    }
    printJson(jsonArray, "apps");
    if (consolePresent) {
      System.out.println(runningCnt + " active, total " + totalCnt + " applications.");
    }
  } catch (Exception ex) {
    throw new CliException("Failed to retrieve application list", ex);
  }
}
 
Example 12
Source Project: attic-apex-core   File: ApexCli.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void execute(String[] args, ConsoleReader reader) throws Exception
{
  try {
    JSONArray jsonArray = new JSONArray();
    List<ApplicationReport> appList = getApplicationList();
    Collections.sort(appList, new Comparator<ApplicationReport>()
    {
      @Override
      public int compare(ApplicationReport o1, ApplicationReport o2)
      {
        return o1.getApplicationId().getId() - o2.getApplicationId().getId();
      }

    });
    int totalCnt = 0;
    int runningCnt = 0;

    SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z");

    for (ApplicationReport ar : appList) {
      /*
       * This is inefficient, but what the heck, if this can be passed through the command line, can anyone notice slowness.
       */
      JSONObject jsonObj = new JSONObject();
      jsonObj.put("startTime", sdf.format(new java.util.Date(ar.getStartTime())));
      jsonObj.put("id", ar.getApplicationId().getId());
      jsonObj.put("name", ar.getName());
      jsonObj.put("state", ar.getYarnApplicationState().name());
      jsonObj.put("trackingUrl", ar.getTrackingUrl());
      jsonObj.put("finalStatus", ar.getFinalApplicationStatus());
      JSONArray tags = new JSONArray();
      for (String tag : ar.getApplicationTags()) {
        tags.put(tag);
      }
      jsonObj.put("tags", tags);

      totalCnt++;
      if (ar.getYarnApplicationState() == YarnApplicationState.RUNNING) {
        runningCnt++;
      }

      if (args.length > 1) {
        if (StringUtils.isNumeric(args[1])) {
          if (jsonObj.getString("id").equals(args[1])) {
            jsonArray.put(jsonObj);
            break;
          }
        } else {
          @SuppressWarnings("unchecked")
          Iterator<String> keys = jsonObj.keys();
          while (keys.hasNext()) {
            if (jsonObj.get(keys.next()).toString().toLowerCase().contains(args[1].toLowerCase())) {
              jsonArray.put(jsonObj);
              break;
            }
          }
        }
      } else {
        jsonArray.put(jsonObj);
      }
    }
    printJson(jsonArray, "apps");
    if (consolePresent) {
      System.out.println(runningCnt + " active, total " + totalCnt + " applications.");
    }
  } catch (Exception ex) {
    throw new CliException("Failed to retrieve application list", ex);
  }
}
 
Example 13
Source Project: metron   File: Client.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Monitor the submitted application for completion.
 * Kill application if time expires.
 * @param appId Application Id of application to be monitored
 * @return true if application completed successfully
 * @throws YarnException
 * @throws IOException
 */
private boolean monitorApplication(ApplicationId appId)
        throws YarnException, IOException {

  while (true) {

    // Check app status every 1 second.
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      LOG.debug("Thread sleep in monitoring loop interrupted");
    }

    // Get application report for the appId we are interested in
    ApplicationReport report = yarnClient.getApplicationReport(appId);

    LOG.info("Got application report from ASM for"
            + ", appId=" + appId.getId()
            + ", clientToAMToken=" + report.getClientToAMToken()
            + ", appDiagnostics=" + report.getDiagnostics()
            + ", appMasterHost=" + report.getHost()
            + ", appQueue=" + report.getQueue()
            + ", appMasterRpcPort=" + report.getRpcPort()
            + ", appStartTime=" + report.getStartTime()
            + ", yarnAppState=" + report.getYarnApplicationState().toString()
            + ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
            + ", appTrackingUrl=" + report.getTrackingUrl()
            + ", appUser=" + report.getUser());

    YarnApplicationState state = report.getYarnApplicationState();
    FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();
    if(YarnApplicationState.RUNNING == state) {
      LOG.info("Application is running...");
      return true;
    }
    if (YarnApplicationState.FINISHED == state) {
      if (FinalApplicationStatus.SUCCEEDED == dsStatus) {
        LOG.info("Application has completed successfully. Breaking monitoring loop");
        return true;
      }
      else {
        LOG.info("Application did finished unsuccessfully."
                + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
                + ". Breaking monitoring loop");
        return false;
      }
    }
    else if (YarnApplicationState.KILLED == state
            || YarnApplicationState.FAILED == state) {
      LOG.info("Application did not finish."
              + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString()
              + ". Breaking monitoring loop");
      return false;
    }

    if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) {
      LOG.info("Reached client specified timeout for application. Killing application");
      forceKillApplication(appId);
      return false;
    }
  }

}