Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#init()

@Test(timeout = 10000)
public void testGetApplicationAttempt() throws YarnException, IOException {
  Configuration conf = new Configuration();
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ApplicationAttemptReport report = client
      .getApplicationAttemptReport(appAttemptId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getApplicationAttemptId().toString(),
      expectedReports.get(0).getCurrentApplicationAttemptId().toString());
  client.stop();
}
 

@Test
public void testKillApplication() throws Exception {
  MockRM rm = new MockRM();
  rm.start();
  RMApp app = rm.submitApp(2000);

  Configuration conf = new Configuration();
  @SuppressWarnings("resource")
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  client.killApplication(app.getApplicationId());
  verify(((MockYarnClient) client).getRMClient(), times(2))
    .forceKillApplication(any(KillApplicationRequest.class));
}
 

@Test
public void testKillApplication() throws Exception {
  MockRM rm = new MockRM();
  rm.start();
  RMApp app = rm.submitApp(2000);

  Configuration conf = new Configuration();
  @SuppressWarnings("resource")
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  client.killApplication(app.getApplicationId());
  verify(((MockYarnClient) client).getRMClient(), times(2))
    .forceKillApplication(any(KillApplicationRequest.class));
}
 

private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
 

/**
 * Tests that the YarnClient is only shut down if it is not shared.
 */
@Test
public void testYarnClientShutDown() {
	YarnClusterDescriptor yarnClusterDescriptor = createYarnClusterDescriptor();

	yarnClusterDescriptor.close();

	assertTrue(yarnClient.isInState(Service.STATE.STARTED));

	final YarnClient closableYarnClient = YarnClient.createYarnClient();
	closableYarnClient.init(yarnConfiguration);
	closableYarnClient.start();

	yarnClusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(
			temporaryFolder.getRoot().getAbsolutePath(),
			new Configuration(),
			yarnConfiguration,
			closableYarnClient,
			false);

	yarnClusterDescriptor.close();

	assertTrue(closableYarnClient.isInState(Service.STATE.STOPPED));
}
 

/**
 * Tests that the cluster retrieval of a finished YARN application fails.
 */
@Test(expected = ClusterRetrieveException.class)
public void testClusterClientRetrievalOfFinishedYarnApplication() throws Exception {
	final ApplicationId applicationId = ApplicationId.newInstance(System.currentTimeMillis(), 42);
	final ApplicationReport applicationReport = createApplicationReport(
		applicationId,
		YarnApplicationState.FINISHED,
		FinalApplicationStatus.SUCCEEDED);

	final YarnClient yarnClient = new TestingYarnClient(Collections.singletonMap(applicationId, applicationReport));
	final YarnConfiguration yarnConfiguration = new YarnConfiguration();
	yarnClient.init(yarnConfiguration);
	yarnClient.start();

	final YarnClusterDescriptor clusterDescriptor = YarnTestUtils.createClusterDescriptorWithLogging(
			temporaryFolder.newFolder().getAbsolutePath(),
			new Configuration(),
			yarnConfiguration,
			yarnClient,
			false);

	try {
		clusterDescriptor.retrieve(applicationId);
	} finally {
		clusterDescriptor.close();
	}
}
 

@Test(timeout = 60000)
public void testInvalidQueueSubmission() throws Exception {

  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  YarnClient yarnClient = YarnClient.createYarnClient();
  try {

    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
    tezConf.set(TezConfiguration.TEZ_QUEUE_NAME, "nonexistent");

    String[] inputPaths = new String[1];
    String[] outputPaths = new String[1];
    String inputDirStr = "/tmp/owc-input";
    inputPaths[0] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    String outputDirStr = "/tmp/owc-output";
    outputPaths[0] = outputDirStr;
    int result = job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
        StringUtils.join(",", outputPaths), "2" }, null);
    Assert.assertTrue("Job should have failed", result != 0);
  } catch (TezException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit application"));
  } finally {
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }
}
 

public static YarnClient createYarnClient(Configuration conf)
{
  YarnClient client = YarnClient.createYarnClient();
  client.init(conf);
  client.start();
  return client;
}
 

@Test
public void testClientStop() {
  Configuration conf = new Configuration();
  ResourceManager rm = new ResourceManager();
  rm.init(conf);
  rm.start();

  YarnClient client = YarnClient.createYarnClient();
  client.init(conf);
  client.start();
  client.stop();
  rm.stop();
}
 

public static String getYarnAppTrackingUrl(ClusterClient clusterClient) throws IOException, YarnException {
  ApplicationId yarnAppId = (ApplicationId) clusterClient.getClusterId();
  YarnClient yarnClient = YarnClient.createYarnClient();
  YarnConfiguration yarnConf = new YarnConfiguration();
  // disable timeline service as we only query yarn app here.
  // Otherwise we may hit this kind of ERROR:
  // java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
  yarnConf.set("yarn.timeline-service.enabled", "false");
  yarnClient.init(yarnConf);
  yarnClient.start();
  return yarnClient.getApplicationReport(yarnAppId).getTrackingUrl();
}
 

@VisibleForTesting
protected YarnClient createYarnClient() {
  YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(getConf());
  yarnClient.start();
  return yarnClient;
}
 

private YarnClusterDescriptor getClusterDescriptor(Configuration configuration) {
	final YarnClient yarnClient = YarnClient.createYarnClient();
	final YarnConfiguration yarnConfiguration = new YarnConfiguration();

	yarnClient.init(yarnConfiguration);
	yarnClient.start();

	return new YarnClusterDescriptor(
			configuration,
			yarnConfiguration,
			yarnClient,
			YarnClientYarnClusterInformationRetriever.create(yarnClient),
			false);
}
 

private AbstractYarnClusterDescriptor getClusterDescriptor(
		Configuration configuration,
		YarnConfiguration yarnConfiguration,
		String configurationDirectory) {
	final YarnClient yarnClient = YarnClient.createYarnClient();
	yarnClient.init(yarnConfiguration);
	yarnClient.start();

	return new YarnClusterDescriptor(
		configuration,
		yarnConfiguration,
		configurationDirectory,
		yarnClient,
		false);
}
 

@Test(timeout = 10000)
public void testGetContainerReport() throws YarnException, IOException {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
      true);
  final YarnClient client = new MockYarnClient();
  client.init(conf);
  client.start();

  List<ApplicationReport> expectedReports = ((MockYarnClient) client)
      .getReports();

  ApplicationId applicationId = ApplicationId.newInstance(1234, 5);
  ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(
      applicationId, 1);
  ContainerId containerId = ContainerId.newContainerId(appAttemptId, 1);
  ContainerReport report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 1)).toString());
  containerId = ContainerId.newContainerId(appAttemptId, 3);
  report = client.getContainerReport(containerId);
  Assert.assertNotNull(report);
  Assert.assertEquals(report.getContainerId().toString(),
      (ContainerId.newContainerId(expectedReports.get(0)
          .getCurrentApplicationAttemptId(), 3)).toString());
  client.stop();
}
 

private AbstractYarnClusterDescriptor getClusterDescriptor(
		Configuration configuration,
		YarnConfiguration yarnConfiguration,
		String configurationDirectory) {
	final YarnClient yarnClient = YarnClient.createYarnClient();
	yarnClient.init(yarnConfiguration);
	yarnClient.start();

	return new YarnClusterDescriptor(
		configuration,
		yarnConfiguration,
		configurationDirectory,
		yarnClient,
		false);
}
 

private DAG createDAG(FileSystem fs, TezConfiguration tezConf,
    Path stagingDir, boolean doLocalityCheck) throws IOException, YarnException {

  int numBroadcastTasks = 2;
  int numOneToOneTasks = 3;
  if (doLocalityCheck) {
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(tezConf);
    yarnClient.start();
    int numNMs = yarnClient.getNodeReports(NodeState.RUNNING).size();
    yarnClient.stop();
    // create enough 1-1 tasks to run in parallel
    numOneToOneTasks = numNMs - numBroadcastTasks - 1;// 1 AM
    if (numOneToOneTasks < 1) {
      numOneToOneTasks = 1;
    }
  }
  byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
  UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte));

  System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");

  Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create(
      InputProcessor.class.getName()), numBroadcastTasks);
  
  Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create(
      InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks);

  Vertex oneToOneVertex = Vertex.create("OneToOne",
      ProcessorDescriptor.create(
          OneToOneProcessor.class.getName()).setUserPayload(procPayload));
  oneToOneVertex.setVertexManagerPlugin(
      VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));

  UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
      .newBuilder(Text.class.getName(), IntWritable.class.getName())
      .setFromConfiguration(tezConf).build();

  DAG dag = DAG.create("BroadcastAndOneToOneExample");
  dag.addVertex(inputVertex)
      .addVertex(broadcastVertex)
      .addVertex(oneToOneVertex)
      .addEdge(
          Edge.create(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
      .addEdge(
          Edge.create(broadcastVertex, oneToOneVertex,
              edgeConf.createDefaultBroadcastEdgeProperty()));
  return dag;
}
 

private YarnClient createYarnClient() {
  YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(configuration);
  yarnClient.start();
  return yarnClient;
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}