org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FlinkIntegrationTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testYarnMode() throws IOException, InterpreterException, YarnException {
  InterpreterSetting flinkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("flink");
  flinkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  flinkInterpreterSetting.setProperty("FLINK_HOME", flinkHome);
  flinkInterpreterSetting.setProperty("PATH", hadoopHome + "/bin:" + System.getenv("PATH"));
  flinkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  flinkInterpreterSetting.setProperty("flink.execution.mode", "YARN");
  testInterpreterBasics();

  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example #2
Source File: SparkIntegrationTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
private void waitForYarnAppCompleted(int timeout) throws YarnException {
  long start = System.currentTimeMillis();
  boolean yarnAppCompleted = false;
  while ((System.currentTimeMillis() - start) < timeout ) {
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    if (response.getApplicationList().isEmpty()) {
      yarnAppCompleted = true;
      break;
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  assertTrue("Yarn app is not completed in " + timeout + " milliseconds.", yarnAppCompleted);
}
 
Example #3
Source File: SparkIntegrationTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testLocalMode() throws IOException, YarnException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "local[*]");
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // no yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(0, response.getApplicationList().size());
  } finally {
    interpreterSettingManager.close();
  }
}
 
Example #4
Source File: YarnInterpreterLauncherIntegrationTest.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
@Test
public void testLaunchShellInYarn() throws YarnException, InterpreterException, InterruptedException {
  InterpreterSetting shellInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("sh");
  shellInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
  shellInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());

  Interpreter shellInterpreter = interpreterFactory.getInterpreter("sh", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("sh").createExecutionContext());

  InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
  InterpreterResult interpreterResult = shellInterpreter.interpret("pwd", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("/usercache/"));

  Thread.sleep(1000);
  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example #5
Source File: ProtocolHATestBase.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  // create GetApplicationsResponse with fake applicationList
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(createFakeAppReports());
  return response;
}
 
Example #6
Source File: ApplicationClientProtocolPBClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException,
    IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(
      null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #7
Source File: TestRMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example #8
Source File: ApplicationHistoryProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #9
Source File: TestApplicationACLs.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyOwnerAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    // View as owner
    rmClient.getApplicationReport(appReportRequest);

    // List apps as owner
    Assert.assertEquals("App view by owner should list the apps!!", 1,
        rmClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as owner
    rmClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #10
Source File: TestApplicationACLs.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifySuperUserAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol superUserClient = getRMClientForUser(SUPER_USER);

    // View as the superUser
    superUserClient.getApplicationReport(appReportRequest);

    // List apps as superUser
    Assert.assertEquals("App view by super-user should list the apps!!", 2,
        superUserClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the superUser
    superUserClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #11
Source File: TestApplicationACLs.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyFriendAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol friendClient = getRMClientForUser(FRIEND);

    // View as the friend
    friendClient.getApplicationReport(appReportRequest);

    // List apps as friend
    Assert.assertEquals("App view by a friend should list the apps!!", 3,
        friendClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the friend
    friendClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #12
Source File: TestApplicationACLs.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyAdministerQueueUserAccess() throws Exception {
  isQueueUser = true;
  AccessControlList viewACL = new AccessControlList("");
  viewACL.addGroup(FRIENDLY_GROUP);
  AccessControlList modifyACL = new AccessControlList("");
  modifyACL.addUser(FRIEND);
  ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

  final GetApplicationReportRequest appReportRequest = recordFactory
      .newRecordInstance(GetApplicationReportRequest.class);
  appReportRequest.setApplicationId(applicationId);
  final KillApplicationRequest finishAppRequest = recordFactory
      .newRecordInstance(KillApplicationRequest.class);
  finishAppRequest.setApplicationId(applicationId);

  ApplicationClientProtocol administerQueueUserRmClient =
      getRMClientForUser(QUEUE_ADMIN_USER);

  // View as the administerQueueUserRmClient
  administerQueueUserRmClient.getApplicationReport(appReportRequest);

  // List apps as administerQueueUserRmClient
  Assert.assertEquals("App view by queue-admin-user should list the apps!!",
      5, administerQueueUserRmClient.getApplications(
             recordFactory.newRecordInstance(GetApplicationsRequest.class))
             .getApplicationList().size());

  // Kill app as the administerQueueUserRmClient
  administerQueueUserRmClient.forceKillApplication(finishAppRequest);
  resourceManager.waitForState(applicationId, RMAppState.KILLED);
}
 
Example #13
Source File: ApplicationHistoryClientService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
        history.getAllApplications().values()));
  return response;
}
 
Example #14
Source File: TestApplicationHistoryClientService.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 
Example #15
Source File: AHSClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications() throws YarnException,
    IOException {
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(null,
      null);
  GetApplicationsResponse response = ahsClient.getApplications(request);
  return response.getApplicationList();
}
 
Example #16
Source File: YarnClientImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
    EnumSet<YarnApplicationState> applicationStates) throws YarnException,
    IOException {
  GetApplicationsRequest request =
      GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 
Example #17
Source File: TestRMWebApp.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #18
Source File: TestResourceMgrDelegate.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void tesAllJobs() throws Exception {
  final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
  GetApplicationsResponse allApplicationsResponse = Records
      .newRecord(GetApplicationsResponse.class);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.FAILED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.SUCCEEDED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.KILLED));
  applications.add(getApplicationReport(YarnApplicationState.FAILED,
      FinalApplicationStatus.FAILED));
  allApplicationsResponse.setApplicationList(applications);
  Mockito.when(
      applicationsManager.getApplications(Mockito
          .any(GetApplicationsRequest.class))).thenReturn(
      allApplicationsResponse);
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
    new YarnConfiguration()) {
    @Override
    protected void serviceStart() throws Exception {
      Assert.assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(applicationsManager);
    }
  };
  JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();

  Assert.assertEquals(State.FAILED, allJobs[0].getState());
  Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState());
  Assert.assertEquals(State.KILLED, allJobs[2].getState());
  Assert.assertEquals(State.FAILED, allJobs[3].getState());
}
 
Example #19
Source File: SparkIntegrationTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Test
public void testYarnClientMode() throws IOException, YarnException, InterruptedException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "yarn-client");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // 1 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(1, response.getApplicationList().size());

  } finally {
    interpreterSettingManager.close();
    waitForYarnAppCompleted(30 * 1000);
  }
}
 
Example #20
Source File: SparkIntegrationTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Test
public void testYarnClusterMode() throws IOException, YarnException, InterruptedException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "yarn-cluster");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // 1 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(1, response.getApplicationList().size());

  } finally {
    interpreterSettingManager.close();
    waitForYarnAppCompleted(30 * 1000);
  }
}
 
Example #21
Source File: FlinkIntegrationTest.java    From zeppelin with Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalMode() throws IOException, YarnException, InterpreterException {
  InterpreterSetting flinkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("flink");
  flinkInterpreterSetting.setProperty("FLINK_HOME", flinkHome);
  flinkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());

  testInterpreterBasics();

  // no yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(0, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example #22
Source File: RMCommunicator.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
 * Get list of all applications known to RM.
 *
 * @return list of application report
 * @throws YarnException the yarn exception
 * @throws IOException Signals that an I/O exception has occurred.
 */
public List<ApplicationReport> getApplications() throws YarnException, IOException{
	if ((System.currentTimeMillis() - lastCacheUpdateTime) > 60000) {
		synchronized (this) {
			if ( (System.currentTimeMillis() - lastCacheUpdateTime) > 60000) {
				GetApplicationsResponse response = proxy.getApplications(GetApplicationsRequest.newInstance());
				appListCache = response.getApplicationList();
				lastCacheUpdateTime = System.currentTimeMillis();
			}
		}
	}
	return appListCache;
}
 
Example #23
Source File: RMCommunicator.java    From jumbune with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
 * Get list of all applications known to RM according to time range provided.
 *@param set of application types
 * @return list of application report
 * @throws YarnException the yarn exception
 * @throws IOException Signals that an I/O exception has occurred.
 */

public List<ApplicationReport> getApplications(ApplicationsRequestScope scope, Set<String> users,
		Set<String> queues, Set<String> applicationTypes, Set<String> applicationTags,
		EnumSet<YarnApplicationState> applicationStates, Long finishFrom, Long finishTo, Long limit)
				throws YarnException, IOException {
	return proxy
			.getApplications(GetApplicationsRequest.newInstance(scope, users, queues, applicationTypes,
					applicationTags, applicationStates, null, new LongRange(finishFrom, finishTo), limit))
			.getApplicationList();
}
 
Example #24
Source File: TestResourceMgrDelegate.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void tesAllJobs() throws Exception {
  final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
  GetApplicationsResponse allApplicationsResponse = Records
      .newRecord(GetApplicationsResponse.class);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.FAILED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.SUCCEEDED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.KILLED));
  applications.add(getApplicationReport(YarnApplicationState.FAILED,
      FinalApplicationStatus.FAILED));
  allApplicationsResponse.setApplicationList(applications);
  Mockito.when(
      applicationsManager.getApplications(Mockito
          .any(GetApplicationsRequest.class))).thenReturn(
      allApplicationsResponse);
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
    new YarnConfiguration()) {
    @Override
    protected void serviceStart() throws Exception {
      Assert.assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(applicationsManager);
    }
  };
  JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();

  Assert.assertEquals(State.FAILED, allJobs[0].getState());
  Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState());
  Assert.assertEquals(State.KILLED, allJobs[2].getState());
  Assert.assertEquals(State.FAILED, allJobs[3].getState());
}
 
Example #25
Source File: ApplicationClientProtocolPBClientImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException,
    IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(
      null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example #26
Source File: TestRMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example #27
Source File: TestRMWebApp.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example #28
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyOwnerAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    // View as owner
    rmClient.getApplicationReport(appReportRequest);

    // List apps as owner
    Assert.assertEquals("App view by owner should list the apps!!", 1,
        rmClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as owner
    rmClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #29
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifySuperUserAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol superUserClient = getRMClientForUser(SUPER_USER);

    // View as the superUser
    superUserClient.getApplicationReport(appReportRequest);

    // List apps as superUser
    Assert.assertEquals("App view by super-user should list the apps!!", 2,
        superUserClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the superUser
    superUserClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }
 
Example #30
Source File: TestApplicationACLs.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyFriendAccess() throws Exception {

    AccessControlList viewACL = new AccessControlList("");
    viewACL.addGroup(FRIENDLY_GROUP);
    AccessControlList modifyACL = new AccessControlList("");
    modifyACL.addUser(FRIEND);
    ApplicationId applicationId = submitAppAndGetAppId(viewACL, modifyACL);

    final GetApplicationReportRequest appReportRequest = recordFactory
        .newRecordInstance(GetApplicationReportRequest.class);
    appReportRequest.setApplicationId(applicationId);
    final KillApplicationRequest finishAppRequest = recordFactory
        .newRecordInstance(KillApplicationRequest.class);
    finishAppRequest.setApplicationId(applicationId);

    ApplicationClientProtocol friendClient = getRMClientForUser(FRIEND);

    // View as the friend
    friendClient.getApplicationReport(appReportRequest);

    // List apps as friend
    Assert.assertEquals("App view by a friend should list the apps!!", 3,
        friendClient.getApplications(
            recordFactory.newRecordInstance(GetApplicationsRequest.class))
            .getApplicationList().size());

    // Kill app as the friend
    friendClient.forceKillApplication(finishAppRequest);
    resourceManager.waitForState(applicationId, RMAppState.KILLED);
  }