Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setResource()

@SuppressWarnings("deprecation")
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
      String name, String queue, Set<String> tags, boolean unmanaged) {

  ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);

  Resource resource = Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  ApplicationSubmissionContext submissionContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  submissionContext.setAMContainerSpec(amContainerSpec);
  submissionContext.setApplicationName(name);
  submissionContext.setQueue(queue);
  submissionContext.setApplicationId(appId);
  submissionContext.setResource(resource);
  submissionContext.setApplicationType(appType);
  submissionContext.setApplicationTags(tags);
  submissionContext.setUnmanagedAM(unmanaged);

  SubmitApplicationRequest submitRequest =
      recordFactory.newRecordInstance(SubmitApplicationRequest.class);
  submitRequest.setApplicationSubmissionContext(submissionContext);
  return submitRequest;
}
 

@Test(timeout = 15000)
public void testSubmitApplicationOnHA() throws Exception {
  ApplicationSubmissionContext appContext =
      Records.newRecord(ApplicationSubmissionContext.class);
  appContext.setApplicationId(cluster.createFakeAppId());
  ContainerLaunchContext amContainer =
      Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(10);
  capability.setVirtualCores(1);
  capability.setGpuCores(1);
  appContext.setResource(capability);
  ApplicationId appId = client.submitApplication(appContext);
  Assert.assertTrue(getActiveRM().getRMContext().getRMApps()
      .containsKey(appId));
}
 

public static ApplicationSubmissionContext newApplicationSubmissionContext(
    ApplicationId applicationId, String applicationName, String queue,
    Priority priority, ContainerLaunchContext amContainer,
    boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
    int maxAppAttempts, Resource resource, String applicationType) {
  ApplicationSubmissionContext context =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(applicationId);
  context.setApplicationName(applicationName);
  context.setQueue(queue);
  context.setPriority(priority);
  context.setAMContainerSpec(amContainer);
  context.setUnmanagedAM(isUnmanagedAM);
  context.setCancelTokensWhenComplete(cancelTokensWhenComplete);
  context.setMaxAppAttempts(maxAppAttempts);
  context.setResource(resource);
  context.setApplicationType(applicationType);
  return context;
}
 

@Test(timeout = 15000)
public void testSubmitApplicationOnHA() throws Exception {
  ApplicationSubmissionContext appContext =
      Records.newRecord(ApplicationSubmissionContext.class);
  appContext.setApplicationId(cluster.createFakeAppId());
  ContainerLaunchContext amContainer =
      Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(10);
  capability.setVirtualCores(1);
  appContext.setResource(capability);
  ApplicationId appId = client.submitApplication(appContext);
  Assert.assertTrue(getActiveRM().getRMContext().getRMApps()
      .containsKey(appId));
}
 

private void setResources(ApplicationSubmissionContext appContext) {
  int memory = Integer.parseInt(
          properties.getProperty("zeppelin.interpreter.yarn.resource.memory", "1024"));
  int memoryOverHead = Integer.parseInt(
          properties.getProperty("zeppelin.interpreter.yarn.resource.memoryOverhead", "384"));
  if (memoryOverHead < memory * 0.1) {
    memoryOverHead = 384;
  }
  int cores = Integer.parseInt(
          properties.getProperty("zeppelin.interpreter.yarn.resource.cores", "1"));
  final Resource resource = Resource.newInstance(memory + memoryOverHead, cores);
  appContext.setResource(resource);
}
 

@SuppressWarnings("deprecation")
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
    AccessControlList modifyACL) throws Exception {
  SubmitApplicationRequest submitRequest = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  ApplicationSubmissionContext context = recordFactory
      .newRecordInstance(ApplicationSubmissionContext.class);

  ApplicationId applicationId = rmClient.getNewApplication(
      recordFactory.newRecordInstance(GetNewApplicationRequest.class))
      .getApplicationId();
  context.setApplicationId(applicationId);

  Map<ApplicationAccessType, String> acls
      = new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
  acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());

  ContainerLaunchContext amContainer = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  Resource resource = BuilderUtils.newResource(1024, 1);
  context.setResource(resource);
  amContainer.setApplicationACLs(acls);
  context.setAMContainerSpec(amContainer);
  submitRequest.setApplicationSubmissionContext(context);
  rmClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 

@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
  ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(this.applicationId);
  context.setQueue(this.queue);
  
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
  context.setAMContainerSpec(amContainer);
  context.setResource(Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  
  SubmitApplicationRequest request = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(context);
  final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  resourceManager.getClientRMService().submitApplication(request);

  // Notify scheduler
  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
  scheduler.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
  scheduler.handle(addAttemptEvent);
}
 

private ApplicationId createApp(YarnClient rmClient, boolean unmanaged) 
  throws Exception {
  YarnClientApplication newApp = rmClient.createApplication();

  ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

  // Create launch context for app master
  ApplicationSubmissionContext appContext
    = Records.newRecord(ApplicationSubmissionContext.class);

  // set the application id
  appContext.setApplicationId(appId);

  // set the application name
  appContext.setApplicationName("test");

  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(1);
  appContext.setPriority(pri);

  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");

  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
    = Records.newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  appContext.setUnmanagedAM(unmanaged);

  // Submit the application to the applications manager
  rmClient.submitApplication(appContext);

  return appId;
}
 

private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
 

@SuppressWarnings("deprecation")
private ApplicationId submitAppAndGetAppId(AccessControlList viewACL,
    AccessControlList modifyACL) throws Exception {
  SubmitApplicationRequest submitRequest = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  ApplicationSubmissionContext context = recordFactory
      .newRecordInstance(ApplicationSubmissionContext.class);

  ApplicationId applicationId = rmClient.getNewApplication(
      recordFactory.newRecordInstance(GetNewApplicationRequest.class))
      .getApplicationId();
  context.setApplicationId(applicationId);

  Map<ApplicationAccessType, String> acls
      = new HashMap<ApplicationAccessType, String>();
  acls.put(ApplicationAccessType.VIEW_APP, viewACL.getAclString());
  acls.put(ApplicationAccessType.MODIFY_APP, modifyACL.getAclString());

  ContainerLaunchContext amContainer = recordFactory
      .newRecordInstance(ContainerLaunchContext.class);
  Resource resource = BuilderUtils.newResource(1024, 1);
  context.setResource(resource);
  amContainer.setApplicationACLs(acls);
  context.setAMContainerSpec(amContainer);
  submitRequest.setApplicationSubmissionContext(context);
  rmClient.submitApplication(submitRequest);
  resourceManager.waitForState(applicationId, RMAppState.ACCEPTED);
  return applicationId;
}
 

@SuppressWarnings("deprecation")
public synchronized void submit() throws IOException, YarnException {
  ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  context.setApplicationId(this.applicationId);
  context.setQueue(this.queue);
  
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
  context.setAMContainerSpec(amContainer);
  context.setResource(Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB));
  
  SubmitApplicationRequest request = recordFactory
      .newRecordInstance(SubmitApplicationRequest.class);
  request.setApplicationSubmissionContext(context);
  final ResourceScheduler scheduler = resourceManager.getResourceScheduler();
  
  resourceManager.getClientRMService().submitApplication(request);

  // Notify scheduler，同时提交app和appAttempt
  AppAddedSchedulerEvent addAppEvent =
      new AppAddedSchedulerEvent(this.applicationId, this.queue, "user");
  scheduler.handle(addAppEvent);
  AppAttemptAddedSchedulerEvent addAttemptEvent =
      new AppAttemptAddedSchedulerEvent(this.applicationAttemptId, false);
  scheduler.handle(addAttemptEvent);
}
 

public void run(VMConfig vmConfig, Configuration conf) throws Exception {
  try {
    vmConfig.overrideHadoopConfiguration(conf);
    System.out.println("Create YarnClient") ;
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    System.out.println("Create YarnClientApplication via YarnClient") ;
    YarnClientApplication app = yarnClient.createApplication();
    String appId =  app.getApplicationSubmissionContext().getApplicationId().toString() ;
    System.out.println("Application Id = " + appId) ;
    System.out.println("Set up the container launch context for the application master") ;
    ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
    StringBuilder sb = new StringBuilder();
    List<String> commands = Collections.singletonList(
        sb.append(vmConfig.buildCommand()).
        append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout").
        append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr")
        .toString()
    );
    amContainer.setCommands(commands) ;

    System.out.println("Setup the app classpath and resources") ;
    if(vmConfig.getVmResources().size() > 0) {
      amContainer.setLocalResources(new VMResources(conf, vmConfig));
    }
    
    System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ;
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN;
    Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    System.out.println("Set up resource type requirements for ApplicationMaster") ;
    Resource resource = Records.newRecord(Resource.class);
    resource.setMemory(256);
    resource.setVirtualCores(1);

    System.out.println("Finally, set-up ApplicationSubmissionContext for the application");
    ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
    appContext.setApplicationName(vmConfig.getName()); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(resource);
    appContext.setQueue("default"); // queue 

    // Submit application
    ApplicationId applicationId = appContext.getApplicationId();
    System.out.println("Submitting application " + applicationId);
    yarnClient.submitApplication(appContext);
  } catch(Exception ex) {
    ex.printStackTrace(); 
    throw ex ;
  }
}
 

private void startApp() throws Exception {
  // submit a dummy app
  ApplicationSubmissionContext appSubmissionContext =
      yarnClient.createApplication().getApplicationSubmissionContext();
  this.applicationId = appSubmissionContext.getApplicationId();

  ContainerLaunchContext containerLaunchContext =
      BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(),
          Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap());

  // Setup the application submission context
  appSubmissionContext.setApplicationName("TestApp");
  appSubmissionContext.setResource(Resource.newInstance(128, 1));
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(containerLaunchContext);

  this.yarnClient.submitApplication(appSubmissionContext);

  // wait for application to be accepted
  int i;
  RMAppAttempt attempt = null;
  for (i = 0; i < 120; i++) {
    ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);

    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      this.applicationAttemptId = appReport.getCurrentApplicationAttemptId();
      attempt = yarnCluster.getResourceManager().getRMContext().getRMApps()
          .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt();
      break;
    }
    Thread.sleep(1000);
  }

  Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state");

  // Set the AM-RM token in the UGI for access during testing
  UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser()
      .getUserName()));
  UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken());
}
 

public void run(String[] args) throws Exception {
    final String command = args[0];
    final int n = Integer.valueOf(args[1]);
    final Path jarPath = new Path(args[2]);
    final String resourceManagerAddress = args[3];
    final String resourceManagerHostname = args[4];
    final String resourceManagerSchedulerAddress = args[5];
    final String resourceManagerResourceTrackerAddress = args[6];

    // Create yarnClient
    YarnConfiguration conf = new YarnConfiguration();
    conf.set("yarn.resourcemanager.address", resourceManagerAddress);
    conf.set("yarn.resourcemanager.hostname", resourceManagerHostname);
    conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress);
    conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress);
    YarnClient yarnClient = YarnClient.createYarnClient();
    yarnClient.init(conf);
    yarnClient.start();

    // Create application via yarnClient
    YarnClientApplication app = yarnClient.createApplication();

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer =
            Records.newRecord(ContainerLaunchContext.class);
    amContainer.setCommands(
            Collections.singletonList(
                    "$JAVA_HOME/bin/java" +
                            " -Xmx256M" +
                            " com.hortonworks.simpleyarnapp.ApplicationMaster" +
                            " " + command +
                            " " + String.valueOf(n) +
                            " " + resourceManagerAddress +
                            " " + resourceManagerHostname +
                            " " + resourceManagerSchedulerAddress +
                            " " + resourceManagerResourceTrackerAddress +
                            " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
                            " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
            )
    );

    // Setup jar for ApplicationMaster
    LocalResource appMasterJar = Records.newRecord(LocalResource.class);
    setupAppMasterJar(jarPath, appMasterJar);
    amContainer.setLocalResources(
            Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar));

    // Setup CLASSPATH for ApplicationMaster
    Map<String, String> appMasterEnv = new HashMap<String, String>();
    setupAppMasterEnv(appMasterEnv);
    amContainer.setEnvironment(appMasterEnv);

    // Set up resource type requirements for ApplicationMaster
    Resource capability = Records.newRecord(Resource.class);
    capability.setMemory(256);
    capability.setVirtualCores(1);

    // Finally, set-up ApplicationSubmissionContext for the application
    ApplicationSubmissionContext appContext =
            app.getApplicationSubmissionContext();
    appContext.setApplicationName("simple-yarn-app"); // application name
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(capability);
    appContext.setQueue("default"); // queue

    // Submit application
    ApplicationId appId = appContext.getApplicationId();
    System.out.println("Submitting application " + appId);
    yarnClient.submitApplication(appContext);

    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    YarnApplicationState appState = appReport.getYarnApplicationState();
    while (appState != YarnApplicationState.FINISHED &&
            appState != YarnApplicationState.KILLED &&
            appState != YarnApplicationState.FAILED) {
        Thread.sleep(100);
        appReport = yarnClient.getApplicationReport(appId);
        appState = appReport.getYarnApplicationState();
    }

    System.out.println(
            "Application " + appId + " finished with" +
                    " state " + appState +
                    " at " + appReport.getFinishTime());

}
 

private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception {
  appContext.setMaxAppAttempts(MAX_ATTEMPT);

  Map<String, LocalResource> localResources = new HashMap<>();
  Set<Path> shippedPaths = new HashSet<>();
  collectLocalResources(localResources, shippedPaths);
  final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
      this.getYarnSessionClusterEntrypoint(),
      false,
      true,
      false, (int) job.taskManagerMemoryMb());

  amContainer.setLocalResources(localResources);

  final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator));
  final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString())
      .collect(Collectors.joining(","));

  // Setup CLASSPATH and environment variables for ApplicationMaster
  ApplicationId appId = appContext.getApplicationId();
  final Map<String, String> appMasterEnv = setUpAmEnvironment(
      appId,
      classPath,
      shippedFiles,
      getDynamicPropertiesEncoded()
  );

  amContainer.setEnvironment(appMasterEnv);

  // Set up resource type requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(getFlinkConfiguration()
      .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
  capability.setVirtualCores(1);

  appContext.setApplicationName(job.name());
  appContext.setApplicationType(ATHENAX_APPLICATION_TYPE);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
  if (job.queue() != null) {
    appContext.setQueue(job.queue());
  }

  LOG.info("Submitting application master {}", appId);
  yarnClient.submitApplication(appContext);

  PollDeploymentStatus poll = new PollDeploymentStatus(appId);
  YARN_POLL_EXECUTOR.submit(poll);
  try {
    return poll.result.get();
  } catch (ExecutionException e) {
    LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause());
    yarnClient.killApplication(appId);
    throw (Exception) e.getCause();
  }
}
 

@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

@Before
public void startApp() throws Exception {
  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Records.newRecord(Priority.class);
  pri.setPriority(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer =
      BuilderUtils.newContainerLaunchContext(
        Collections.<String, LocalResource> emptyMap(),
        new HashMap<String, String>(), Arrays.asList("sleep", "100"),
        new HashMap<String, ByteBuffer>(), null,
        new HashMap<ApplicationAccessType, String>());
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(Resource.newInstance(1024, 1, 1));
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  RMAppAttempt appAttempt = null;
  while (true) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
  }
  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));

  // emulate RM setup of AMRM token in credentials by adding the token
  // *before* setting the token service
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());
  appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf));
}
 

@Test (timeout = 30000)
public void testSubmitIncorrectQueue() throws IOException {
  MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
  YarnClient rmClient = null;
  try {
    cluster.init(new YarnConfiguration());
    cluster.start();
    final Configuration yarnConf = cluster.getConfig();
    rmClient = YarnClient.createYarnClient();
    rmClient.init(yarnConf);
    rmClient.start();
    YarnClientApplication newApp = rmClient.createApplication();

    ApplicationId appId = newApp.getNewApplicationResponse().getApplicationId();

    // Create launch context for app master
    ApplicationSubmissionContext appContext
      = Records.newRecord(ApplicationSubmissionContext.class);

    // set the application id
    appContext.setApplicationId(appId);

    // set the application name
    appContext.setApplicationName("test");

    // Set the queue to which this application is to be submitted in the RM
    appContext.setQueue("nonexist");

    // Set up the container launch context for the application master
    ContainerLaunchContext amContainer
      = Records.newRecord(ContainerLaunchContext.class);
    appContext.setAMContainerSpec(amContainer);
    appContext.setResource(Resource.newInstance(1024, 1));
    // appContext.setUnmanagedAM(unmanaged);

    // Submit the application to the applications manager
    rmClient.submitApplication(appContext);
    Assert.fail("Job submission should have thrown an exception");
  } catch (YarnException e) {
    Assert.assertTrue(e.getMessage().contains("Failed to submit"));
  } finally {
    if (rmClient != null) {
      rmClient.stop();
    }
    cluster.stop();
  }
}
 

/**
 * Setup and submit the Gobblin Yarn application.
 *
 * @throws IOException if there's anything wrong setting up and submitting the Yarn application
 * @throws YarnException if there's anything wrong setting up and submitting the Yarn application
 */
@VisibleForTesting
ApplicationId setupAndSubmitApplication() throws IOException, YarnException {
  YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication();
  ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext();
  appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE);
  appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY));
  ApplicationId applicationId = appSubmissionContext.getApplicationId();

  GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse();
  // Set up resource type requirements for ApplicationMaster
  Resource resource = prepareContainerResource(newApplicationResponse);

  // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources
  Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId);

  ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class);
  amContainerLaunchContext.setLocalResources(appMasterLocalResources);
  amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration));
  amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory())));

  Map<ApplicationAccessType, String> acls = new HashMap<>(1);
  acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl);
  amContainerLaunchContext.setApplicationACLs(acls);

  if (UserGroupInformation.isSecurityEnabled()) {
    setupSecurityTokens(amContainerLaunchContext);
  }

  // Setup the application submission context
  appSubmissionContext.setApplicationName(this.applicationName);
  appSubmissionContext.setResource(resource);
  appSubmissionContext.setQueue(this.appQueueName);
  appSubmissionContext.setPriority(Priority.newInstance(0));
  appSubmissionContext.setAMContainerSpec(amContainerLaunchContext);
  // Also setup container local resources by copying local jars and files the container need to HDFS
  addContainerLocalResources(applicationId);

  // Submit the application
  LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString()));
  this.yarnClient.submitApplication(appSubmissionContext);

  LOGGER.info("Application successfully submitted and accepted");
  ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId);
  LOGGER.info("Application Name: " + applicationReport.getName());
  LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl());
  LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue());

  return applicationId;
}
 

public ApplicationId run() throws IOException, YarnException {
  LOG.info("calling run.");
  yarnClient.start();

  // YarnClientApplication is used to populate:
  //   1. GetNewApplication Response
  //   2. ApplicationSubmissionContext
  YarnClientApplication app = yarnClient.createApplication();
  
  // GetNewApplicationResponse can be used to determined resources available.
  GetNewApplicationResponse appResponse = app.getNewApplicationResponse();
  
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  this.appId = appContext.getApplicationId();
  appContext.setApplicationName(this.appname);

  // Set up the container launch context for AM.
  ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);

  LocalResource appMasterJar;
  FileSystem fs = FileSystem.get(this.conf);
  
  amContainer.setLocalResources(
      Collections.singletonMap("master.jar",
          Util.newYarnAppResource(fs, new Path(this.hdfsJar), LocalResourceType.FILE,
              LocalResourceVisibility.APPLICATION)));
  // Set up CLASSPATH for ApplicationMaster
  Map<String, String> appMasterEnv = new HashMap<String, String>();
  setupAppMasterEnv(appMasterEnv);
  amContainer.setEnvironment(appMasterEnv);

  // Set up resource requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(this.applicationMasterMem);
  capability.setVirtualCores(1); //TODO: Can we really setVirtualCores ?
  amContainer.setCommands(Collections.singletonList(this.getAppMasterCommand()));

  // put everything together.
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setQueue("default"); // TODO: Need to investigate more on queuing an scheduling.

  // Submit application
  yarnClient.submitApplication(appContext);
  LOG.info("APPID: "+this.appId.toString());
  return this.appId;
  //return this.monitorApplication(appId);
}