org.apache.hadoop.yarn.api.records.YarnApplicationState Java Examples

private void killUnFinishedApplication(ApplicationId appId)
    throws IOException {
  ApplicationReport application = null;
  try {
    application = resMgrDelegate.getApplicationReport(appId);
  } catch (YarnException e) {
    throw new IOException(e);
  }
  if (application.getYarnApplicationState() == YarnApplicationState.FINISHED
      || application.getYarnApplicationState() == YarnApplicationState.FAILED
      || application.getYarnApplicationState() == YarnApplicationState.KILLED) {
    return;
  }
  killApplication(appId);
}
 

@Test(timeout=20000)
public void testJobSubmissionFailure() throws Exception {
  when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))).
  thenReturn(appId);
  ApplicationReport report = mock(ApplicationReport.class);
  when(report.getApplicationId()).thenReturn(appId);
  when(report.getDiagnostics()).thenReturn(failString);
  when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
  when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
  Credentials credentials = new Credentials();
  File jobxml = new File(testWorkDir, "job.xml");
  OutputStream out = new FileOutputStream(jobxml);
  conf.writeXml(out);
  out.close();
  try {
    yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
  } catch(IOException io) {
    LOG.info("Logging exception:", io);
    assertTrue(io.getLocalizedMessage().contains(failString));
  }
}
 

public static State fromYarn(YarnApplicationState yarnApplicationState,
    FinalApplicationStatus finalApplicationStatus) {
  switch (yarnApplicationState) {
  case NEW:
  case NEW_SAVING:
  case SUBMITTED:
  case ACCEPTED:
    return State.PREP;
  case RUNNING:
    return State.RUNNING;
  case FINISHED:
    if (finalApplicationStatus == FinalApplicationStatus.SUCCEEDED) {
      return State.SUCCEEDED;
    } else if (finalApplicationStatus == FinalApplicationStatus.KILLED) {
      return State.KILLED;
    }
  case FAILED:
    return State.FAILED;
  case KILLED:
    return State.KILLED;
  }
  throw new YarnRuntimeException("Unrecognized application state: " + yarnApplicationState);
}
 

private ApplicationReport mockRunningApplicationReport(String appId, String appName)
{
  ApplicationReport app = mock(ApplicationReport.class);
  ApplicationId applicationId = mock(ApplicationId.class);

  when(applicationId.toString()).thenReturn(appId);
  when(app.getApplicationId()).thenReturn(applicationId);

  when(app.getName()).thenReturn(appName);

  when(app.getYarnApplicationState()).thenReturn(YarnApplicationState.RUNNING);
  when(app.getFinalApplicationStatus()).thenReturn(FinalApplicationStatus.UNDEFINED);

  when(app.getTrackingUrl()).thenReturn("http://example.com");

  return app;
}
 

private static Map<YarnApplicationState, Map<String, Long>> buildScoreboard(
   Set<String> states, Set<String> types) {
  Map<YarnApplicationState, Map<String, Long>> scoreboard
      = new HashMap<YarnApplicationState, Map<String, Long>>();
  // default states will result in enumerating all YarnApplicationStates
  assert !states.isEmpty();
  for (String state : states) {
    Map<String, Long> partScoreboard = new HashMap<String, Long>();
    scoreboard.put(
        YarnApplicationState.valueOf(StringUtils.toUpperCase(state)),
        partScoreboard);
    // types is verified no to be empty
    for (String type : types) {
      partScoreboard.put(type, 0L);
    }
  }
  return scoreboard;
}
 

@Override
public void render(Block html) {
  html.
      div("#nav").
          h3("Application History").
              ul().
                  li().a(url("apps"), "Applications").
                      ul().
                          li().a(url("apps",
                              YarnApplicationState.FINISHED.toString()),
                              YarnApplicationState.FINISHED.toString()).
                          _().
                          li().a(url("apps",
                              YarnApplicationState.FAILED.toString()),
                              YarnApplicationState.FAILED.toString()).
                          _().
                          li().a(url("apps",
                              YarnApplicationState.KILLED.toString()),
                              YarnApplicationState.KILLED.toString()).
                          _().
                      _().
                  _().
              _().
          _();
}
 

private void waitTillAccepted(YarnClient rmClient, ApplicationId appId)
  throws Exception {
  try {
    long start = System.currentTimeMillis();
    ApplicationReport report = rmClient.getApplicationReport(appId);
    while (YarnApplicationState.ACCEPTED != report.getYarnApplicationState()) {
      if (System.currentTimeMillis() - start > 20 * 1000) {
        throw new Exception("App '" + appId + 
          "' time out, failed to reach ACCEPTED state");
      }
      Thread.sleep(200);
      report = rmClient.getApplicationReport(appId);
    }
  } catch (Exception ex) {
    throw new Exception(ex);
  }
}
 

/**
 * Kills the application with the application id as appId
 * 
 * @param applicationId
 * @throws YarnException
 * @throws IOException
 */
private void killApplication(String applicationId) throws YarnException,
    IOException {
  ApplicationId appId = ConverterUtils.toApplicationId(applicationId);
  ApplicationReport  appReport = null;
  try {
    appReport = client.getApplicationReport(appId);
  } catch (ApplicationNotFoundException e) {
    sysout.println("Application with id '" + applicationId +
        "' doesn't exist in RM.");
    throw e;
  }

  if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
      || appReport.getYarnApplicationState() == YarnApplicationState.KILLED
      || appReport.getYarnApplicationState() == YarnApplicationState.FAILED) {
    sysout.println("Application " + applicationId + " has already finished ");
  } else {
    sysout.println("Killing application " + applicationId);
    client.killApplication(appId);
  }
}
 

private void updateApplicationStatus() {
	if (yarnClient.isInState(Service.STATE.STARTED)) {
		final ApplicationReport applicationReport;

		try {
			applicationReport = yarnClient.getApplicationReport(yarnApplicationId);
		} catch (Exception e) {
			LOG.info("Could not retrieve the Yarn application report for {}.", yarnApplicationId);
			applicationStatus = ApplicationStatus.UNKNOWN;
			return;
		}

		YarnApplicationState yarnApplicationState = applicationReport.getYarnApplicationState();

		if (yarnApplicationState == YarnApplicationState.FAILED || yarnApplicationState == YarnApplicationState.KILLED) {
			applicationStatus = ApplicationStatus.FAILED;
		} else {
			applicationStatus = ApplicationStatus.SUCCEEDED;
		}
	} else {
		LOG.info("Yarn client is no longer in state STARTED. Stopping the Yarn application status monitor.");
		applicationStatusUpdateFuture.cancel(false);
	}
}
 

@Test(timeout=20000)
public void testJobSubmissionFailure() throws Exception {
  when(resourceMgrDelegate.submitApplication(any(ApplicationSubmissionContext.class))).
  thenReturn(appId);
  ApplicationReport report = mock(ApplicationReport.class);
  when(report.getApplicationId()).thenReturn(appId);
  when(report.getDiagnostics()).thenReturn(failString);
  when(report.getYarnApplicationState()).thenReturn(YarnApplicationState.FAILED);
  when(resourceMgrDelegate.getApplicationReport(appId)).thenReturn(report);
  Credentials credentials = new Credentials();
  File jobxml = new File(testWorkDir, "job.xml");
  OutputStream out = new FileOutputStream(jobxml);
  conf.writeXml(out);
  out.close();
  try {
    yarnRunner.submitJob(jobId, testWorkDir.getAbsolutePath().toString(), credentials);
  } catch(IOException io) {
    LOG.info("Logging exception:", io);
    assertTrue(io.getLocalizedMessage().contains(failString));
  }
}
 

public static ApplicationReport getStartedAppInstanceByName(YarnClient clientRMService, String appName, String user, String excludeAppId) throws YarnException, IOException
{
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED), EnumSet.of(YarnApplicationState.RUNNING,
      YarnApplicationState.ACCEPTED,
      YarnApplicationState.NEW,
      YarnApplicationState.NEW_SAVING,
      YarnApplicationState.SUBMITTED));
  // see whether there is an app with the app name and user name running
  for (ApplicationReport app : applications) {
    if (!app.getApplicationId().toString().equals(excludeAppId)
        && app.getName().equals(appName)
        && app.getUser().equals(user)) {
      return app;
    }
  }
  return null;
}
 

public ApplicationFinishedEvent(
    ApplicationId appId,
    String diagnosticsInfo,
    FinalApplicationStatus appStatus,
    YarnApplicationState state,
    ApplicationAttemptId latestAppAttemptId,
    long finishedTime,
    RMAppMetrics appMetrics) {
  super(SystemMetricsEventType.APP_FINISHED, finishedTime);
  this.appId = appId;
  this.diagnosticsInfo = diagnosticsInfo;
  this.appStatus = appStatus;
  this.latestAppAttemptId = latestAppAttemptId;
  this.state = state;
  this.appMetrics=appMetrics;
}
 

private String clarifyAppState(YarnApplicationState state) {
  String ret = state.toString();
  switch (state) {
  case NEW:
    return ret + ": waiting for application to be initialized";
  case NEW_SAVING:
    return ret + ": waiting for application to be persisted in state-store.";
  case SUBMITTED:
    return ret + ": waiting for application to be accepted by scheduler.";
  case ACCEPTED:
    return ret + ": waiting for AM container to be allocated, launched and"
        + " register with RM.";
  case RUNNING:
    return ret + ": AM has registered with RM and started running.";
  default:
    return ret;
  }
}
 

@Test
public void testAppsQueryStatesNone() throws JSONException, Exception {
  rm.start();
  MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
  rm.submitApp(CONTAINER_MB);
  amNodeManager.nodeHeartbeat(true);
  WebResource r = resource();

  ClientResponse response = r.path("ws").path("v1").path("cluster")
      .path("apps")
      .queryParam("states", YarnApplicationState.RUNNING.toString())
      .accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
  assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
  JSONObject json = response.getEntity(JSONObject.class);
  assertEquals("incorrect number of elements", 1, json.length());
  assertEquals("apps is not null", JSONObject.NULL, json.get("apps"));
  rm.stop();
}
 

@Test(timeout = 5000)
public void testStopRetriesUntilTerminalState() throws Exception {
  TezConfiguration conf = new TezConfiguration();
  conf.setBoolean(TezConfiguration.TEZ_CLIENT_ASYNCHRONOUS_STOP, false);
  conf.setLong(TezConfiguration.TEZ_CLIENT_HARD_KILL_TIMEOUT_MS, HARD_KILL_TIMEOUT);
  final TezClientForTest client = configureAndCreateTezClient(conf);
  client.start();
  when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
      .thenReturn(YarnApplicationState.NEW).thenReturn(YarnApplicationState.KILLED);
  try {
    client.stop();
  } catch (Exception e) {
    Assert.fail("Expected ApplicationNotFoundException");
  }
  verify(client.mockYarnClient, atLeast(2)).getApplicationReport(client.mockAppId);
}
 

@Test
public void testLaunchShellInYarn() throws YarnException, InterpreterException, InterruptedException {
  InterpreterSetting shellInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("sh");
  shellInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
  shellInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());

  Interpreter shellInterpreter = interpreterFactory.getInterpreter("sh", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("sh").createExecutionContext());

  InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
  InterpreterResult interpreterResult = shellInterpreter.interpret("pwd", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("/usercache/"));

  Thread.sleep(1000);
  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 

protected ApplicationReport getApplicationByName(String appName)
{
  if (appName == null) {
    throw new CliException("Invalid application name provided by user");
  }
  List<ApplicationReport> appList = getApplicationList();
  for (ApplicationReport ar : appList) {
    if ((ar.getName().equals(appName)) &&
        (ar.getYarnApplicationState() != YarnApplicationState.KILLED) &&
        (ar.getYarnApplicationState() != YarnApplicationState.FINISHED)) {
      LOG.debug("Application Name: {} Application ID: {} Application State: {}",
          ar.getName(), ar.getApplicationId().toString(), YarnApplicationState.FINISHED);
      return ar;
    }
  }
  return null;
}
 

public static List<ApplicationReport> cleanAppDirectories(YarnClient clientRMService, Configuration conf, FileSystem fs, long finishedBefore)
    throws IOException, YarnException
{
  List<ApplicationReport> result = new ArrayList<>();
  List<ApplicationReport> applications = clientRMService.getApplications(Sets.newHashSet(StramClient.YARN_APPLICATION_TYPE, StramClient.YARN_APPLICATION_TYPE_DEPRECATED),
      EnumSet.of(YarnApplicationState.FAILED, YarnApplicationState.FINISHED, YarnApplicationState.KILLED));
  Path appsBasePath = new Path(StramClientUtils.getApexDFSRootDir(fs, conf), StramClientUtils.SUBDIR_APPS);
  for (ApplicationReport ar : applications) {
    long finishTime = ar.getFinishTime();
    if (finishTime < finishedBefore) {
      try {
        Path appPath = new Path(appsBasePath, ar.getApplicationId().toString());
        if (fs.isDirectory(appPath)) {
          LOG.debug("Deleting finished application data for {}", ar.getApplicationId());
          fs.delete(appPath, true);
          result.add(ar);
        }
      } catch (Exception ex) {
        LOG.warn("Cannot delete application data for {}", ar.getApplicationId(), ex);
        continue;
      }
    }
  }
  return result;
}
 

private ApplicationReport getRunningApplicationReport(String host, int port) {
  ApplicationId appId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
      appId, 0);
  return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
    "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
    "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
    YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
}
 

private static void countApp(
    Map<YarnApplicationState, Map<String, Long>> scoreboard,
    YarnApplicationState state, String type) {
  Map<String, Long> partScoreboard = scoreboard.get(state);
  Long count = partScoreboard.get(type);
  partScoreboard.put(type, count + 1L);
}
 

@Test public void testView() {
  Injector injector = WebAppTests.createMockInjector(RMContext.class,
      mockRMContext(15, 1, 2, 8*GiB),
      new Module() {
    @Override
    public void configure(Binder binder) {
      try {
        ResourceManager mockRm = mockRm(3, 1, 2, 8*GiB);
        binder.bind(ResourceManager.class).toInstance(mockRm);
        binder.bind(ApplicationBaseProtocol.class)
            .toInstance(mockRm.getClientRMService());
      } catch (IOException e) {
        throw new IllegalStateException(e);
      }
    }
  });
  RmView rmViewInstance = injector.getInstance(RmView.class);
  rmViewInstance.set(YarnWebParams.APP_STATE,
      YarnApplicationState.RUNNING.toString());
  rmViewInstance.render();
  WebAppTests.flushOutput(injector);

  rmViewInstance.set(YarnWebParams.APP_STATE, StringHelper.cjoin(
      YarnApplicationState.ACCEPTED.toString(),
      YarnApplicationState.RUNNING.toString()));
  rmViewInstance.render();
  WebAppTests.flushOutput(injector);
}
 

private static GetApplicationReportResponse
    createApplicationReportWithFinishedApplication() {
  ApplicationReport report = mock(ApplicationReport.class);
  when(report.getYarnApplicationState()).thenReturn(
    YarnApplicationState.FINISHED);
  GetApplicationReportResponse response =
      mock(GetApplicationReportResponse.class);
  when(response.getApplicationReport()).thenReturn(report);
  return response;
}
 

private static RMContext mockRMContext(List<RMAppState> states) {
  final ConcurrentMap<ApplicationId, RMApp> applicationsMaps = Maps
      .newConcurrentMap();
  int i = 0;
  for (RMAppState state : states) {
    MockRMApp app = new MockRMApp(i, i, state) {
      @Override
      public RMAppMetrics getRMAppMetrics() {
        return new RMAppMetrics(Resource.newInstance(0, 0, 0), 0, 0, 0, 0, 0);
      }
      @Override
      public YarnApplicationState createApplicationState() {
        return YarnApplicationState.ACCEPTED;
      }
    };
    RMAppAttempt attempt = mock(RMAppAttempt.class);
    app.setCurrentAppAttempt(attempt);
    applicationsMaps.put(app.getApplicationId(), app);
    i++;
  }

  RMContextImpl rmContext =  new RMContextImpl(null, null, null, null,
      null, null, null, null, null, null) {
    @Override
    public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
      return applicationsMaps;
    }
    @Override
    public ResourceScheduler getScheduler() {
      return mock(AbstractYarnScheduler.class);
    }
  };
  return rmContext;
}
 

public boolean isStartRunningSparkApplication(String applicationId) throws YarnException, IOException{
	boolean valid = false;
	for(ApplicationReport report : this.yarnClient.getApplications()){
		if(report.getApplicationType().equals("SPARK") && report.getApplicationId().toString().equals(applicationId)){
			if(report.getYarnApplicationState().equals(YarnApplicationState.RUNNING) || 
					report.getYarnApplicationState().equals(YarnApplicationState.FINISHED)){
				valid = true;
			}
			break;
		}
	}
	return valid;
}
 

@Override
public YarnApplicationState createApplicationState() {
  RMAppState rmAppState = getState();
  // If App is in FINAL_SAVING state, return its previous state.
  if (rmAppState.equals(RMAppState.FINAL_SAVING)) {
    rmAppState = stateBeforeFinalSaving;
  }
  if (rmAppState.equals(RMAppState.KILLING)) {
    rmAppState = stateBeforeKilling;
  }
  return RMServerUtils.createApplicationState(rmAppState);
}
 

@Override
public void setApplicationStates(EnumSet<YarnApplicationState> applicationStates) {
  maybeInitBuilder();
  if (applicationStates == null) {
    builder.clearApplicationStates();
  }
  this.applicationStates = applicationStates;
}
 

@Override
public void setApplicationStates(Set<String> applicationStates) {
  EnumSet<YarnApplicationState> appStates = null;
  for (YarnApplicationState state : YarnApplicationState.values()) {
    if (applicationStates.contains(
        StringUtils.toLowerCase(state.name()))) {
      if (appStates == null) {
        appStates = EnumSet.of(state);
      } else {
        appStates.add(state);
      }
    }
  }
  setApplicationStates(appStates);
}
 

@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
    EnumSet<YarnApplicationState> applicationStates) throws YarnException,
    IOException {
  GetApplicationsRequest request =
      GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 

@Override
public List<ApplicationReport> getApplications(
    Set<String> applicationTypes, EnumSet<YarnApplicationState> applicationStates)
    throws YarnException, IOException {
  when(mockAppResponse.getApplicationList()).thenReturn(
      getApplicationReports(reports, applicationTypes, applicationStates));
  return super.getApplications(applicationTypes, applicationStates);
}
 

@Override
public GetApplicationReportResponse getApplicationReport(
    GetApplicationReportRequest request) throws IOException {
  ApplicationId applicationId = request.getApplicationId();
  ApplicationReport application = recordFactory
      .newRecordInstance(ApplicationReport.class);
  application.setApplicationId(applicationId);
  application.setFinalApplicationStatus(FinalApplicationStatus.UNDEFINED);
  if (amRunning) {
    application.setYarnApplicationState(YarnApplicationState.RUNNING);
  } else if (amRestarting) {
    application.setYarnApplicationState(YarnApplicationState.SUBMITTED);
  } else {
    application.setYarnApplicationState(YarnApplicationState.FINISHED);
    application.setFinalApplicationStatus(FinalApplicationStatus.SUCCEEDED);
  }
  String[] split = AMHOSTADDRESS.split(":");
  application.setHost(split[0]);
  application.setRpcPort(Integer.parseInt(split[1]));
  application.setUser("TestClientRedirect-user");
  application.setName("N/A");
  application.setQueue("N/A");
  application.setStartTime(0);
  application.setFinishTime(0);
  application.setTrackingUrl("N/A");
  application.setDiagnostics("N/A");

  GetApplicationReportResponse response = recordFactory
      .newRecordInstance(GetApplicationReportResponse.class);
  response.setApplicationReport(application);
  return response;
}