Java Code Examples for org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse

The following examples show how to use org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
@Test
public void testLaunchShellInYarn() throws YarnException, InterpreterException, InterruptedException {
  InterpreterSetting shellInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("sh");
  shellInterpreterSetting.setProperty("zeppelin.interpreter.launcher", "yarn");
  shellInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());

  Interpreter shellInterpreter = interpreterFactory.getInterpreter("sh", new ExecutionContextBuilder().setUser("user1").setNoteId("note1").setDefaultInterpreterGroup("sh").createExecutionContext());

  InterpreterContext context = new InterpreterContext.Builder().setNoteId("note1").setParagraphId("paragraph_1").build();
  InterpreterResult interpreterResult = shellInterpreter.interpret("pwd", context);
  assertEquals(InterpreterResult.Code.SUCCESS, interpreterResult.code());
  assertTrue(interpreterResult.toString(), interpreterResult.message().get(0).getData().contains("/usercache/"));

  Thread.sleep(1000);
  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example 2
Source Project: zeppelin   Source File: SparkIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testLocalMode() throws IOException, YarnException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "local[*]");
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // no yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(0, response.getApplicationList().size());
  } finally {
    interpreterSettingManager.close();
  }
}
 
Example 3
Source Project: zeppelin   Source File: SparkIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
private void waitForYarnAppCompleted(int timeout) throws YarnException {
  long start = System.currentTimeMillis();
  boolean yarnAppCompleted = false;
  while ((System.currentTimeMillis() - start) < timeout ) {
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    if (response.getApplicationList().isEmpty()) {
      yarnAppCompleted = true;
      break;
    }
    try {
      Thread.sleep(1000);
    } catch (InterruptedException e) {
      e.printStackTrace();
    }
  }
  assertTrue("Yarn app is not completed in " + timeout + " milliseconds.", yarnAppCompleted);
}
 
Example 4
Source Project: zeppelin   Source File: FlinkIntegrationTest.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testYarnMode() throws IOException, InterpreterException, YarnException {
  InterpreterSetting flinkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("flink");
  flinkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  flinkInterpreterSetting.setProperty("FLINK_HOME", flinkHome);
  flinkInterpreterSetting.setProperty("PATH", hadoopHome + "/bin:" + System.getenv("PATH"));
  flinkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  flinkInterpreterSetting.setProperty("flink.execution.mode", "YARN");
  testInterpreterBasics();

  // 1 yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(1, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example 5
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example 6
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException,
    IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(
      null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example 7
Source Project: hadoop   Source File: TestRMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example 8
Source Project: hadoop   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example 9
Source Project: hadoop   Source File: ApplicationHistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
        history.getAllApplications().values()));
  return response;
}
 
Example 10
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 
Example 11
Source Project: hadoop   Source File: AHSClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications() throws YarnException,
    IOException {
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(null,
      null);
  GetApplicationsResponse response = ahsClient.getApplications(request);
  return response.getApplicationList();
}
 
Example 12
Source Project: hadoop   Source File: YarnClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
    EnumSet<YarnApplicationState> applicationStates) throws YarnException,
    IOException {
  GetApplicationsRequest request =
      GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 
Example 13
Source Project: hadoop   Source File: ProtocolHATestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  // create GetApplicationsResponse with fake applicationList
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(createFakeAppReports());
  return response;
}
 
Example 14
Source Project: hadoop   Source File: TestResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void tesAllJobs() throws Exception {
  final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
  GetApplicationsResponse allApplicationsResponse = Records
      .newRecord(GetApplicationsResponse.class);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.FAILED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.SUCCEEDED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.KILLED));
  applications.add(getApplicationReport(YarnApplicationState.FAILED,
      FinalApplicationStatus.FAILED));
  allApplicationsResponse.setApplicationList(applications);
  Mockito.when(
      applicationsManager.getApplications(Mockito
          .any(GetApplicationsRequest.class))).thenReturn(
      allApplicationsResponse);
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
    new YarnConfiguration()) {
    @Override
    protected void serviceStart() throws Exception {
      Assert.assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(applicationsManager);
    }
  };
  JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();

  Assert.assertEquals(State.FAILED, allJobs[0].getState());
  Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState());
  Assert.assertEquals(State.KILLED, allJobs[2].getState());
  Assert.assertEquals(State.FAILED, allJobs[3].getState());
}
 
Example 15
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(null,
      requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example 16
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException,
    IOException {
  GetApplicationsRequestProto requestProto =
      ((GetApplicationsRequestPBImpl) request).getProto();
  try {
    return new GetApplicationsResponsePBImpl(proxy.getApplications(
      null, requestProto));
  } catch (ServiceException e) {
    RPCUtil.unwrapAndThrowException(e);
    return null;
  }
}
 
Example 17
Source Project: big-c   Source File: TestRMWebServices.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testAppsRace() throws Exception {
  // mock up an RM that returns app reports for apps that don't exist
  // in the RMApps list
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationReport mockReport = mock(ApplicationReport.class);
  when(mockReport.getApplicationId()).thenReturn(appId);
  GetApplicationsResponse mockAppsResponse =
      mock(GetApplicationsResponse.class);
  when(mockAppsResponse.getApplicationList())
    .thenReturn(Arrays.asList(new ApplicationReport[] { mockReport }));
  ClientRMService mockClientSvc = mock(ClientRMService.class);
  when(mockClientSvc.getApplications(isA(GetApplicationsRequest.class),
      anyBoolean())).thenReturn(mockAppsResponse);
  ResourceManager mockRM = mock(ResourceManager.class);
  RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null,
      null, null, null, null, null);
  when(mockRM.getRMContext()).thenReturn(rmContext);
  when(mockRM.getClientRMService()).thenReturn(mockClientSvc);

  RMWebServices webSvc = new RMWebServices(mockRM, new Configuration(),
      mock(HttpServletResponse.class));

  final Set<String> emptySet =
      Collections.unmodifiableSet(Collections.<String>emptySet());

  // verify we don't get any apps when querying
  HttpServletRequest mockHsr = mock(HttpServletRequest.class);
  AppsInfo appsInfo = webSvc.getApps(mockHsr, null, emptySet, null,
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());

  // verify we don't get an NPE when specifying a final status query
  appsInfo = webSvc.getApps(mockHsr, null, emptySet, "FAILED",
      null, null, null, null, null, null, null, emptySet, emptySet);
  assertTrue(appsInfo.getApps().isEmpty());
}
 
Example 18
Source Project: big-c   Source File: TestRMWebApp.java    License: Apache License 2.0 5 votes vote down vote up
public static ClientRMService mockClientRMService(RMContext rmContext) {
  ClientRMService clientRMService = mock(ClientRMService.class);
  List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
  for (RMApp app : rmContext.getRMApps().values()) {
    ApplicationReport appReport =
        ApplicationReport.newInstance(
            app.getApplicationId(), (ApplicationAttemptId) null,
            app.getUser(), app.getQueue(),
            app.getName(), (String) null, 0, (Token) null,
            app.createApplicationState(),
            app.getDiagnostics().toString(), (String) null,
            app.getStartTime(), app.getFinishTime(),
            app.getFinalApplicationStatus(),
            (ApplicationResourceUsageReport) null, app.getTrackingUrl(),
            app.getProgress(), app.getApplicationType(), (Token) null);
    appReports.add(appReport);
  }
  GetApplicationsResponse response = mock(GetApplicationsResponse.class);
  when(response.getApplicationList()).thenReturn(appReports);
  try {
    when(clientRMService.getApplications(any(GetApplicationsRequest.class)))
        .thenReturn(response);
  } catch (YarnException e) {
    Assert.fail("Exception is not expteced.");
  }
  return clientRMService;
}
 
Example 19
Source Project: big-c   Source File: ApplicationHistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse
    getApplications(GetApplicationsRequest request) throws YarnException,
        IOException {
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(new ArrayList<ApplicationReport>(
        history.getAllApplications().values()));
  return response;
}
 
Example 20
Source Project: big-c   Source File: TestApplicationHistoryClientService.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testApplications() throws IOException, YarnException {
  ApplicationId appId = null;
  appId = ApplicationId.newInstance(0, 1);
  ApplicationId appId1 = ApplicationId.newInstance(0, 2);
  GetApplicationsRequest request = GetApplicationsRequest.newInstance();
  GetApplicationsResponse response =
      clientService.getApplications(request);
  List<ApplicationReport> appReport = response.getApplicationList();
  Assert.assertNotNull(appReport);
  Assert.assertEquals(appId, appReport.get(0).getApplicationId());
  Assert.assertEquals(appId1, appReport.get(1).getApplicationId());
}
 
Example 21
Source Project: big-c   Source File: AHSClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications() throws YarnException,
    IOException {
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(null,
      null);
  GetApplicationsResponse response = ahsClient.getApplications(request);
  return response.getApplicationList();
}
 
Example 22
Source Project: big-c   Source File: YarnClientImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public List<ApplicationReport> getApplications(Set<String> applicationTypes,
    EnumSet<YarnApplicationState> applicationStates) throws YarnException,
    IOException {
  GetApplicationsRequest request =
      GetApplicationsRequest.newInstance(applicationTypes, applicationStates);
  GetApplicationsResponse response = rmClient.getApplications(request);
  return response.getApplicationList();
}
 
Example 23
Source Project: big-c   Source File: ProtocolHATestBase.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException {
  resetStartFailoverFlag(true);

  // make sure failover has been triggered
  Assert.assertTrue(waittingForFailOver());

  // create GetApplicationsResponse with fake applicationList
  GetApplicationsResponse response =
      GetApplicationsResponse.newInstance(createFakeAppReports());
  return response;
}
 
Example 24
Source Project: big-c   Source File: TestResourceMgrDelegate.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void tesAllJobs() throws Exception {
  final ApplicationClientProtocol applicationsManager = Mockito.mock(ApplicationClientProtocol.class);
  GetApplicationsResponse allApplicationsResponse = Records
      .newRecord(GetApplicationsResponse.class);
  List<ApplicationReport> applications = new ArrayList<ApplicationReport>();
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.FAILED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.SUCCEEDED));
  applications.add(getApplicationReport(YarnApplicationState.FINISHED,
      FinalApplicationStatus.KILLED));
  applications.add(getApplicationReport(YarnApplicationState.FAILED,
      FinalApplicationStatus.FAILED));
  allApplicationsResponse.setApplicationList(applications);
  Mockito.when(
      applicationsManager.getApplications(Mockito
          .any(GetApplicationsRequest.class))).thenReturn(
      allApplicationsResponse);
  ResourceMgrDelegate resourceMgrDelegate = new ResourceMgrDelegate(
    new YarnConfiguration()) {
    @Override
    protected void serviceStart() throws Exception {
      Assert.assertTrue(this.client instanceof YarnClientImpl);
      ((YarnClientImpl) this.client).setRMClient(applicationsManager);
    }
  };
  JobStatus[] allJobs = resourceMgrDelegate.getAllJobs();

  Assert.assertEquals(State.FAILED, allJobs[0].getState());
  Assert.assertEquals(State.SUCCEEDED, allJobs[1].getState());
  Assert.assertEquals(State.KILLED, allJobs[2].getState());
  Assert.assertEquals(State.FAILED, allJobs[3].getState());
}
 
Example 25
Source Project: zeppelin   Source File: SparkIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testYarnClientMode() throws IOException, YarnException, InterruptedException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "yarn-client");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // 1 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(1, response.getApplicationList().size());

  } finally {
    interpreterSettingManager.close();
    waitForYarnAppCompleted(30 * 1000);
  }
}
 
Example 26
Source Project: zeppelin   Source File: SparkIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testYarnClusterMode() throws IOException, YarnException, InterruptedException, InterpreterException, XmlPullParserException {
  InterpreterSetting sparkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("spark");
  sparkInterpreterSetting.setProperty("spark.master", "yarn-cluster");
  sparkInterpreterSetting.setProperty("HADOOP_CONF_DIR", hadoopCluster.getConfigPath());
  sparkInterpreterSetting.setProperty("SPARK_HOME", sparkHome);
  sparkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());
  sparkInterpreterSetting.setProperty("zeppelin.spark.useHiveContext", "false");
  sparkInterpreterSetting.setProperty("zeppelin.pyspark.useIPython", "false");
  sparkInterpreterSetting.setProperty("PYSPARK_PYTHON", getPythonExec());
  sparkInterpreterSetting.setProperty("spark.driver.memory", "512m");
  sparkInterpreterSetting.setProperty("zeppelin.spark.scala.color", "false");
  sparkInterpreterSetting.setProperty("zeppelin.spark.deprecatedMsg.show", "false");

  try {
    setUpSparkInterpreterSetting(sparkInterpreterSetting);
    testInterpreterBasics();

    // 1 yarn application launched
    GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
    GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
    assertEquals(1, response.getApplicationList().size());

  } finally {
    interpreterSettingManager.close();
    waitForYarnAppCompleted(30 * 1000);
  }
}
 
Example 27
Source Project: zeppelin   Source File: FlinkIntegrationTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testLocalMode() throws IOException, YarnException, InterpreterException {
  InterpreterSetting flinkInterpreterSetting = interpreterSettingManager.getInterpreterSettingByName("flink");
  flinkInterpreterSetting.setProperty("FLINK_HOME", flinkHome);
  flinkInterpreterSetting.setProperty("ZEPPELIN_CONF_DIR", zeppelin.getZeppelinConfDir().getAbsolutePath());

  testInterpreterBasics();

  // no yarn application launched
  GetApplicationsRequest request = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.RUNNING));
  GetApplicationsResponse response = hadoopCluster.getYarnCluster().getResourceManager().getClientRMService().getApplications(request);
  assertEquals(0, response.getApplicationList().size());

  interpreterSettingManager.close();
}
 
Example 28
/**
 * Get list of all applications known to RM.
 *
 * @return list of application report
 * @throws YarnException the yarn exception
 * @throws IOException Signals that an I/O exception has occurred.
 */
public List<ApplicationReport> getApplications() throws YarnException, IOException{
	if ((System.currentTimeMillis() - lastCacheUpdateTime) > 60000) {
		synchronized (this) {
			if ( (System.currentTimeMillis() - lastCacheUpdateTime) > 60000) {
				GetApplicationsResponse response = proxy.getApplications(GetApplicationsRequest.newInstance());
				appListCache = response.getApplicationList();
				lastCacheUpdateTime = System.currentTimeMillis();
			}
		}
	}
	return appListCache;
}
 
Example 29
Source Project: hadoop   Source File: ClientRMService.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public GetApplicationsResponse getApplications(
    GetApplicationsRequest request) throws YarnException {
  return getApplications(request, true);
}
 
Example 30
Source Project: hadoop   Source File: TestRM.java    License: Apache License 2.0 4 votes vote down vote up
@Test (timeout = 80000)
public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
  conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
  MockRM rm1 = new MockRM(conf);
  rm1.start();

  // a succeeded app
  RMApp app1 = rm1.submitApp(200);
  MockNM nm1 =
      new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
  nm1.registerNode();
  MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
  MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);

  // a failed app
  RMApp app2 = rm1.submitApp(200);
  MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
  nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
  am2.waitForState(RMAppAttemptState.FAILED);
  rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED);

  // a killed app
  RMApp app3 = rm1.submitApp(200);
  MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
  rm1.killApp(app3.getApplicationId());
  rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED);
  rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED);

  GetApplicationsRequest request1 =
      GetApplicationsRequest.newInstance(EnumSet.of(
        YarnApplicationState.FINISHED, YarnApplicationState.KILLED,
        YarnApplicationState.FAILED));
  GetApplicationsResponse response1 =
      rm1.getClientRMService().getApplications(request1);
  List<ApplicationReport> appList1 = response1.getApplicationList();

  Assert.assertEquals(3, appList1.size());
  for (ApplicationReport report : appList1) {
    // killed/failed apps host and rpc port are invalidated.
    if (report.getApplicationId().equals(app2.getApplicationId())
        || report.getApplicationId().equals(app3.getApplicationId())) {
      Assert.assertEquals("N/A", report.getHost());
      Assert.assertEquals(-1, report.getRpcPort());
    }
    // succeeded app's host and rpc port is not invalidated
    if (report.getApplicationId().equals(app1.getApplicationId())) {
      Assert.assertFalse(report.getHost().equals("N/A"));
      Assert.assertTrue(report.getRpcPort() != -1);
    }
  }
}