Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClientApplication#getApplicationSubmissionContext()

ApplicationId submitApplication(
    YarnClientApplication app,
    String appName,
    ContainerLaunchContext launchContext,
    Resource resource,
    String queue) throws Exception {
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  appContext.setApplicationName(appName);
  appContext.setApplicationTags(new HashSet<>());
  appContext.setAMContainerSpec(launchContext);
  appContext.setResource(resource);
  appContext.setQueue(queue);

  return yarnClient.submitApplication(appContext);
}
 

public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration,
                            final REEFFileNames fileNames,
                            final ClasspathProvider classpath,
                            final YarnProxyUser yarnProxyUser,
                            final SecurityTokenProvider tokenProvider,
                            final boolean isUnmanaged,
                            final List<String> commandPrefixList) throws IOException, YarnException {

  this.classpath = classpath;
  this.yarnProxyUser = yarnProxyUser;
  this.isUnmanaged = isUnmanaged;

  this.driverStdoutFilePath =
      ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStdoutFileName();

  this.driverStderrFilePath =
      ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStderrFileName();

  LOG.log(Level.FINE, "Initializing YARN Client");
  this.yarnClient = YarnClient.createYarnClient();
  this.yarnClient.init(yarnConfiguration);
  this.yarnClient.start();
  LOG.log(Level.FINE, "Initialized YARN Client");

  LOG.log(Level.FINE, "Requesting Application ID from YARN.");
  final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication();
  this.applicationResponse = yarnClientApplication.getNewApplicationResponse();
  this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
  this.applicationSubmissionContext.setUnmanagedAM(isUnmanaged);
  this.applicationId = this.applicationSubmissionContext.getApplicationId();
  this.tokenProvider = tokenProvider;
  this.commandPrefixList = commandPrefixList;
  this.configurationFilePaths = Collections.singletonList(fileNames.getDriverConfigurationPath());
  LOG.log(Level.INFO, "YARN Application ID: {0}", this.applicationId);
}
 

UnmanagedAmYarnSubmissionHelper(
    final YarnConfiguration yarnConfiguration,
    final YarnProxyUser yarnProxyUser,
    final SecurityTokenProvider tokenProvider) throws IOException, YarnException {

  this.tokenProvider = tokenProvider;
  this.yarnProxyUser = yarnProxyUser;

  LOG.log(Level.FINE, "Initializing YARN Client");
  this.yarnClient = YarnClient.createYarnClient();
  this.yarnClient.init(yarnConfiguration);
  this.yarnClient.start();
  LOG.log(Level.FINE, "Initialized YARN Client");

  LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN.");

  final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext(
      Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens());

  final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication();

  this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext();
  this.applicationSubmissionContext.setAMContainerSpec(launchContext);
  this.applicationSubmissionContext.setUnmanagedAM(true);

  this.applicationId = this.applicationSubmissionContext.getApplicationId();

  LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId);
}
 

public void run(VMConfig vmConfig, Configuration conf) throws Exception {
  try {
    vmConfig.overrideHadoopConfiguration(conf);
    System.out.println("Create YarnClient") ;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    System.out.println("Create YarnClientApplication via YarnClient") ;
    YarnClientApplication app = yarnClient.createApplication();
    String appId =  app.getApplicationSubmissionContext().getApplicationId().toString() ;
    System.out.println("Application Id = " + appId) ;
    System.out.println("Set up the container launch context for the application master") ;
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    StringBuilder sb = new StringBuilder();
    List<String> commands = Collections.singletonList(
        sb.append(vmConfig.buildCommand()).
        append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
        append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr")
        .toString()
    );
    amContainer.setCommands(commands) ;

    System.out.println("Setup the app classpath and resources") ;
    if(vmConfig.getVmResources().size() > 0) {
      amContainer.setLocalResources(new VMResources(conf, vmConfig));
    }
    
    System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ;
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN;
    Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    System.out.println("Set up resource type requirements for ApplicationMaster") ;
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemory(256);
    resource.setVirtualCores(1);

    System.out.println("Finally, set-up ApplicationSubmissionContext for the application");
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(vmConfig.getName()); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(resource);
    appContext.setQueue("default"); // queue 

    // Submit application
    ApplicationId applicationId = appContext.getApplicationId();
    System.out.println("Submitting application " + applicationId);
    yarnClient.submitApplication(appContext);
  } catch(Exception ex) {
    ex.printStackTrace(); 
    throw ex ;
  }
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
  appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory())));

  Map<ApplicationAccessType, String> acls = new HashMap<>(1);
  acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
  amContainerLaunchContext.setApplicationACLs(acls);

  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString()));
  this.yarnClient.submitApplication(appSubmissionContext);

  LOGGER.info("Application successfully submitted and accepted");
  ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
  LOGGER.info("Application Name: " + applicationReport.getName());
  LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
  LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

  return applicationId;
}
 

/**
 * Main run function for the client
 * @return true if application completed successfully
 * @throws java.io.IOException
 * @throws org.apache.hadoop.yarn.exceptions.YarnException
 */
public boolean run() throws IOException, YarnException {

  LOG.info("Running Client");
  yarnClient.start();

  // Get a new application id
  YarnClientApplication app = yarnClient.createApplication();
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();

  int maxMem = appResponse.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);

  // A resource ask cannot exceed the max.
  if (amMemory > maxMem) {
    LOG.info("AM memory specified above max threshold of cluster. Using max value."
        + ", specified=" + amMemory
        + ", max=" + maxMem);
    amMemory = maxMem;
  }

  int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores);

  if (amVCores > maxVCores) {
    LOG.info("AM virtual cores specified above max threshold of cluster. "
        + "Using max value." + ", specified=" + amVCores
        + ", max=" + maxVCores);
    amVCores = maxVCores;
  }

  // set the application name
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();

  appContext.setApplicationName(appName);

  // Set up resource type requirements
  // For now, both memory and vcores are supported, so we set memory and
  // vcores requirements
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(amMemory);
  capability.setVirtualCores(amVCores);
  appContext.setResource(capability);

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(amPriority);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue(amQueue);

  // Set the ContainerLaunchContext to describe the Container ith which the ApplicationMaster is launched.
  appContext.setAMContainerSpec(getAMContainerSpec(appId.getId()));

  // Submit the application to the applications manager
  // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest);
  // Ignore the response as either a valid response object is returned on success
  // or an exception thrown to denote some form of a failure
  LOG.info("Submitting application to ASM");

  yarnClient.submitApplication(appContext);

  // Monitor the application
  return monitorApplication(appId);
}
 

public ApplicationId run() throws IOException, YarnException {
  LOG.info("calling run.");
  yarnClient.start();

  // YarnClientApplication is used to populate:
  //   1. GetNewApplication Response
  //   2. ApplicationSubmissionContext
  YarnClientApplication app = yarnClient.createApplication();
  
  // GetNewApplicationResponse can be used to determined resources available.
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
  
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  this.appId = appContext.getApplicationId();
  appContext.setApplicationName(this.appname);

  // Set up the container launch context for AM.
  ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

  LocalResource appMasterJar;
  FileSystem fs = FileSystem.get(this.conf);
  
  amContainer.setLocalResources(
      Collections.singletonMap("master.jar",
          Util.newYarnAppResource(fs, new Path(this.hdfsJar), LocalResourceType.FILE,
              LocalResourceVisibility.APPLICATION)));
  // Set up CLASSPATH for ApplicationMaster
  Map<String, String> appMasterEnv = new HashMap<String, String>();
  setupAppMasterEnv(appMasterEnv);
  amContainer.setEnvironment(appMasterEnv);

  // Set up resource requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(this.applicationMasterMem);
  capability.setVirtualCores(1); //TODO: Can we really setVirtualCores ?
  amContainer.setCommands(Collections.singletonList(this.getAppMasterCommand()));

  // put everything together.
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setQueue("default"); // TODO: Need to investigate more on queuing an scheduling.

  // Submit application
  yarnClient.submitApplication(appContext);
  LOG.info("APPID: "+this.appId.toString());
  return this.appId;
  //return this.monitorApplication(appId);
}