Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClient#createYarnClient()

public Client() {
	conf = new HiWayConfiguration();
	yarnClient = YarnClient.createYarnClient();
	yarnClient.init(conf);
	opts = new Options();
	opts.addOption("w", "workflow", false, "(Deprecated) The workflow file to be executed by the Application Master.");
	opts.addOption("u", "summary", true, "The name of the json summary file. No file is created if this parameter is not specified.");
	opts.addOption("m", "memory", true, "The amount of memory (in MB) to be allocated per worker container. Overrides settings in hiway-site.xml.");
	opts.addOption("c", "custom", true, "The name of an (optional) JSON file, in which custom amounts of memory can be specified per task.");
	String schedulers = "";
	for (HiWayConfiguration.HIWAY_SCHEDULER_OPTS policy : HiWayConfiguration.HIWAY_SCHEDULER_OPTS.values()) {
		schedulers += ", " + policy.toString();
	}
	opts.addOption("s", "scheduler", true, "The scheduling policy that is to be employed. Valid arguments: " + schedulers.substring(2) + "."
			+ " Overrides settings in hiway-site.xml.");
	String workflowFormats = "";
	for (HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_OPTS language : HiWayConfiguration.HIWAY_WORKFLOW_LANGUAGE_OPTS.values()) {
		workflowFormats += ", " + language.toString();
	}
	opts.addOption("l", "language", true, "The input file format. Will be automatically detected if not specified explicitly. Valid arguments: "
			+ workflowFormats.substring(2) + ".");
	opts.addOption("v", "verbose", false, "Increase verbosity of output / reporting.");
	opts.addOption("d", "debug", false, "Provide additional logs and information for debugging");
	opts.addOption("h", "help", false, "Print usage");
}
 

public void addClusterLimits(JobConf jobConf, ResourceConfig resConf) throws IOException {
	if ( resConf.isEmpty() ) {
		return;
	}
	Configuration conf = new Configuration(jobConf);
	YarnClient client = YarnClient.createYarnClient();
	client.init(conf);
	client.start();
	try {
		for (ResourceLimit limit : computeResourceLimits(client.getNodeReports()) ) {
			resConf.addLimit(limit);
		}
	} catch ( YarnException ye ) {
		throw new IOException(ye);
	} finally {
		client.stop();
	}
}
 

private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
 

@Override
public YarnClient get()
{
    YarnClient client = YarnClient.createYarnClient();
    if (yarnConfiguration.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
        try {
            TimelineClient.createTimelineClient();
        }
        catch (NoClassDefFoundError e) {
            logger.warn("createTimelineClient() error with {}", TimelineClient.class.getResource(TimelineClient.class.getSimpleName() + ".class"), e);
            yarnConfiguration.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
        }
    }
    client.init(yarnConfiguration);
    client.start();
    return client;
}
 

private void verifyClientConnection() {
  int numRetries = 3;
  while(numRetries-- > 0) {
    Configuration conf = new YarnConfiguration(this.conf);
    YarnClient client = YarnClient.createYarnClient();
    client.init(conf);
    client.start();
    try {
      client.getApplications();
      return;
    } catch (Exception e) {
      LOG.error(e);
    } finally {
      client.stop();
    }
  }
  fail("Client couldn't connect to the Active RM");
}
 

public static String getYarnAppTrackingUrl(ClusterClient clusterClient) throws IOException, YarnException {
  ApplicationId yarnAppId = (ApplicationId) clusterClient.getClusterId();
  YarnClient yarnClient = YarnClient.createYarnClient();
  YarnConfiguration yarnConf = new YarnConfiguration();
  // disable timeline service as we only query yarn app here.
  // Otherwise we may hit this kind of ERROR:
  // java.lang.ClassNotFoundException: com.sun.jersey.api.client.config.ClientConfig
  yarnConf.set("yarn.timeline-service.enabled", "false");
  yarnClient.init(yarnConf);
  yarnClient.start();
  return yarnClient.getApplicationReport(yarnAppId).getTrackingUrl();
}
 

@BeforeClass
public static void setup() throws Exception {
  // start minicluster
  conf = new YarnConfiguration();
  conf.setLong(
    YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
    rolling_interval_sec);
  conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
  conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();

  // start rm client
  yarnClient = YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
  
  priority = Priority.newInstance(1);
  priority2 = Priority.newInstance(2);
  capability = Resource.newInstance(1024, 1);

  node = nodeReports.get(0).getNodeId().getHost();
  rack = nodeReports.get(0).getRackName();
  nodes = new String[]{ node };
  racks = new String[]{ rack };
}
 

/**
 * Tests that the YarnClient is only shut down if it is not shared.
 */
@Test
public void testYarnClientShutDown() {
	YarnClusterDescriptor yarnClusterDescriptor = new YarnClusterDescriptor(
		new Configuration(),
		yarnConfiguration,
		temporaryFolder.getRoot().getAbsolutePath(),
		yarnClient,
		true);

	yarnClusterDescriptor.close();

	assertTrue(yarnClient.isInState(Service.STATE.STARTED));

	final YarnClient closableYarnClient = YarnClient.createYarnClient();
	closableYarnClient.init(yarnConfiguration);
	closableYarnClient.start();

	yarnClusterDescriptor = new YarnClusterDescriptor(
		new Configuration(),
		yarnConfiguration,
		temporaryFolder.getRoot().getAbsolutePath(),
		closableYarnClient,
		false);

	yarnClusterDescriptor.close();

	assertTrue(closableYarnClient.isInState(Service.STATE.STOPPED));
}
 

@Before
public void checkClusterEmpty() {
	if (yarnClient == null) {
		yarnClient = YarnClient.createYarnClient();
		yarnClient.init(getYarnConfiguration());
		yarnClient.start();
	}

	flinkConfiguration = new org.apache.flink.configuration.Configuration(globalConfiguration);
}
 

@BeforeClass
public static void setupClass() {
	yarnConfiguration = new YarnConfiguration();
	yarnClient = YarnClient.createYarnClient();
	yarnClient.init(yarnConfiguration);
	yarnClient.start();
}
 

protected YarnClient createAndStartYarnClient(Configuration conf) {
  Configuration configuration = new YarnConfiguration(conf);
  YarnClient client = YarnClient.createYarnClient();
  client.init(configuration);
  client.start();
  return client;
}
 

@BeforeClass
public static void setupClass() {
	yarnConfiguration = new YarnConfiguration();
	yarnClient = YarnClient.createYarnClient();
	yarnClient.init(yarnConfiguration);
	yarnClient.start();
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

public YarnCLI() {
  super(new YarnConfiguration());
  client = YarnClient.createYarnClient();
  client.init(getConf());
  client.start();
}
 

/**
 * verify the diagnostics in {@link DAGStatus} is correct in non-session mode,
 * and also verify that diagnostics from {@link DAGStatus} should match that
 * from {@link ApplicationReport}
 * 
 * @throws Exception
 */
@Test(timeout = 120000)
public void testExceptionPropagationNonSession() throws Exception {
  try {
    startMiniTezCluster();
    startNonSessionClient();

    ExceptionLocation exLocation = ExceptionLocation.EM_GetNumSourceTaskPhysicalOutputs;
    LOG.info("NonSession mode, Test for Exception from:" + exLocation.name());
    DAG dag = createDAG(exLocation);
    DAGClient dagClient = tezClient.submitDAG(dag);
    DAGStatus dagStatus = dagClient.waitForCompletion();
    String diagnostics = StringUtils.join(dagStatus.getDiagnostics(), ",");
    LOG.info("Diagnostics:" + diagnostics);
    assertTrue(diagnostics.contains(exLocation.name()));

    // wait for app complete (unregisterApplicationMaster is done)
    ApplicationId appId = tezClient.getAppMasterApplicationId();
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(tezConf);
    yarnClient.start();
    Set<YarnApplicationState> FINAL_APPLICATION_STATES =
        EnumSet.of(YarnApplicationState.KILLED, YarnApplicationState.FAILED,
            YarnApplicationState.FINISHED);
    ApplicationReport appReport = null;
    while (true) {
      appReport = yarnClient.getApplicationReport(appId);
      Thread.sleep(1000);
      LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
      LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
      if (FINAL_APPLICATION_STATES.contains(appReport
          .getYarnApplicationState())) {
        break;
      }
    }
    // wait for 1 second and call getApplicationReport again to ensure get the
    // diagnostics
    // TODO remove it after YARN-2560
    Thread.sleep(1000);
    appReport = yarnClient.getApplicationReport(appId);

    LOG.info("FinalAppStatus:" + appReport.getFinalApplicationStatus());
    LOG.info("Diagnostics from appReport:" + appReport.getDiagnostics());
    assertTrue(appReport.getDiagnostics().contains(exLocation.name()));
    // use "\n" as separator, because we also use it in Tez internally when
    // assembling the application diagnostics.
    assertEquals(StringUtils.join(dagStatus.getDiagnostics(), "\n").trim(),
        appReport.getDiagnostics().trim());
  } finally {
    stopNonSessionClient();
    stopTezMiniCluster();
  }
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

@Test(timeout = 60000)
public void testSimpleSessionExample() throws Exception {
  Path stagingDirPath = new Path("/tmp/owc-staging-dir");
  remoteFs.mkdirs(stagingDirPath);

  int numIterations = 2;
  String[] inputPaths = new String[numIterations];
  String[] outputPaths = new String[numIterations];
  Path[] outputDirs = new Path[numIterations];
  for (int i=0; i<numIterations; ++i) {
    String inputDirStr = "/tmp/owc-input-" + i + "/";
    inputPaths[i] = inputDirStr;
    Path inputDir = new Path(inputDirStr);
    remoteFs.mkdirs(inputDir);
    generateOrderedWordCountInput(inputDir, remoteFs);
    String outputDirStr = "/tmp/owc-output-" + i + "/"; 
    outputPaths[i] = outputDirStr;
    Path outputDir = new Path(outputDirStr);
    outputDirs[i] = outputDir;
  }


  TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
  tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
  YarnClient yarnClient = YarnClient.createYarnClient();

  try {
    
    yarnClient.init(mrrTezCluster.getConfig());
    yarnClient.start();
    
    List<ApplicationReport> apps = yarnClient.getApplications();
    int appsBeforeCount = apps != null ? apps.size() : 0;

    SimpleSessionExample job = new SimpleSessionExample();
    tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
    Assert.assertTrue(
        "SimpleSessionExample failed",
        job.run(tezConf, new String[] { StringUtils.join(",", inputPaths),
            StringUtils.join(",", outputPaths), "2" }, null) == 0);

    for (int i=0; i<numIterations; ++i) {
      verifyOutput(outputDirs[i], remoteFs);
    }
    
    apps = yarnClient.getApplications();
    int appsAfterCount = apps != null ? apps.size() : 0;
    
    // Running in session mode. So should only create 1 more app.
    Assert.assertEquals(appsBeforeCount + 1, appsAfterCount);
  } finally {
    remoteFs.delete(stagingDirPath, true);
    if (yarnClient != null) {
      yarnClient.stop();
    }
  }

}
 

public YarnService(Configuration config){
	YarnConfiguration conf = new YarnConfiguration(config);
	this.yarnClient = YarnClient.createYarnClient();
	this.yarnClient.init(conf);
	this.yarnClient.start();
}
 

protected YarnClient createYarnClient() {
  return YarnClient.createYarnClient();
}