Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationTags()

The following examples show how to use org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationTags() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestClientRMService.java    From hadoop with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
      String name, String queue, Set<String> tags, boolean unmanaged) {

  ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);

  Resource resource = Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  ApplicationSubmissionContext submissionContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  submissionContext.setAMContainerSpec(amContainerSpec);
  submissionContext.setApplicationName(name);
  submissionContext.setQueue(queue);
  submissionContext.setApplicationId(appId);
  submissionContext.setResource(resource);
  submissionContext.setApplicationType(appType);
  submissionContext.setApplicationTags(tags);
  submissionContext.setUnmanagedAM(unmanaged);

  SubmitApplicationRequest submitRequest =
      recordFactory.newRecordInstance(SubmitApplicationRequest.class);
  submitRequest.setApplicationSubmissionContext(submissionContext);
  return submitRequest;
}
Example 2
Source File: TestClientRMService.java    From big-c with Apache License 2.0 6 votes vote down vote up 
@SuppressWarnings("deprecation")
private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId,
      String name, String queue, Set<String> tags, boolean unmanaged) {

  ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class);

  Resource resource = Resources.createResource(
      YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);

  ApplicationSubmissionContext submissionContext =
      recordFactory.newRecordInstance(ApplicationSubmissionContext.class);
  submissionContext.setAMContainerSpec(amContainerSpec);
  submissionContext.setApplicationName(name);
  submissionContext.setQueue(queue);
  submissionContext.setApplicationId(appId);
  submissionContext.setResource(resource);
  submissionContext.setApplicationType(appType);
  submissionContext.setApplicationTags(tags);
  submissionContext.setUnmanagedAM(unmanaged);

  SubmitApplicationRequest submitRequest =
      recordFactory.newRecordInstance(SubmitApplicationRequest.class);
  submitRequest.setApplicationSubmissionContext(submissionContext);
  return submitRequest;
}
Example 3
Source File: SylphSparkYarnClient.java    From sylph with Apache License 2.0 5 votes vote down vote up 
@Override
public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext)
{
    final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext);
    appContext.setApplicationType("Sylph_SPARK");
    appContext.setApplicationTags(ImmutableSet.of("a1", "a2"));
    appContext.setQueue(yarnQueue);
    appContext.setMaxAppAttempts(2);
    return appContext;
}
Example 4
Source File: LaunchCluster.java    From TensorFlowOnYARN with Apache License 2.0 5 votes vote down vote up 
ApplicationId submitApplication(
    YarnClientApplication app,
    String appName,
    ContainerLaunchContext launchContext,
    Resource resource,
    String queue) throws Exception {
  ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
  appContext.setApplicationName(appName);
  appContext.setApplicationTags(new HashSet<>());
  appContext.setAMContainerSpec(launchContext);
  appContext.setResource(resource);
  appContext.setQueue(queue);

  return yarnClient.submitApplication(appContext);
}
Example 5
Source File: RMWebServices.java    From hadoop with Apache License 2.0 5 votes vote down vote up 
/**
 * Create the actual ApplicationSubmissionContext to be submitted to the RM
 * from the information provided by the user.
 * 
 * @param newApp
 *          the information provided by the user
 * @return returns the constructed ApplicationSubmissionContext
 * @throws IOException
 */
protected ApplicationSubmissionContext createAppSubmissionContext(
    ApplicationSubmissionContextInfo newApp) throws IOException {

  // create local resources and app submission context

  ApplicationId appid;
  String error =
      "Could not parse application id " + newApp.getApplicationId();
  try {
    appid =
        ConverterUtils.toApplicationId(recordFactory,
          newApp.getApplicationId());
  } catch (Exception e) {
    throw new BadRequestException(error);
  }
  ApplicationSubmissionContext appContext =
      ApplicationSubmissionContext.newInstance(appid,
        newApp.getApplicationName(), newApp.getQueue(),
        Priority.newInstance(newApp.getPriority()),
        createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
        newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
        createAppSubmissionContextResource(newApp),
        newApp.getApplicationType(),
        newApp.getKeepContainersAcrossApplicationAttempts(),
        newApp.getAppNodeLabelExpression(),
        newApp.getAMContainerNodeLabelExpression());
  appContext.setApplicationTags(newApp.getApplicationTags());

  return appContext;
}
Example 6
Source File: RMWebServices.java    From big-c with Apache License 2.0 5 votes vote down vote up 
/**
 * Create the actual ApplicationSubmissionContext to be submitted to the RM
 * from the information provided by the user.
 * 
 * @param newApp
 *          the information provided by the user
 * @return returns the constructed ApplicationSubmissionContext
 * @throws IOException
 */
protected ApplicationSubmissionContext createAppSubmissionContext(
    ApplicationSubmissionContextInfo newApp) throws IOException {

  // create local resources and app submission context

  ApplicationId appid;
  String error =
      "Could not parse application id " + newApp.getApplicationId();
  try {
    appid =
        ConverterUtils.toApplicationId(recordFactory,
          newApp.getApplicationId());
  } catch (Exception e) {
    throw new BadRequestException(error);
  }
  ApplicationSubmissionContext appContext =
      ApplicationSubmissionContext.newInstance(appid,
        newApp.getApplicationName(), newApp.getQueue(),
        Priority.newInstance(newApp.getPriority()),
        createContainerLaunchContext(newApp), newApp.getUnmanagedAM(),
        newApp.getCancelTokensWhenComplete(), newApp.getMaxAppAttempts(),
        createAppSubmissionContextResource(newApp),
        newApp.getApplicationType(),
        newApp.getKeepContainersAcrossApplicationAttempts(),
        newApp.getAppNodeLabelExpression(),
        newApp.getAMContainerNodeLabelExpression());
  appContext.setApplicationTags(newApp.getApplicationTags());

  return appContext;
}
Example 7
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up 
@Test(timeout=1000)
public void testSetApplicationTags() {
  TezConfiguration conf = new TezConfiguration(false);
  conf.set(TezConfiguration.TEZ_APPLICATION_TAGS, "foo,bar");
  AMConfiguration amconfig = new AMConfiguration(conf, null, null);
  ApplicationSubmissionContext appContext = Records
      .newRecord(ApplicationSubmissionContext.class);
  Collection<String> tagsFromConf =
      amconfig.getTezConfiguration().getTrimmedStringCollection(
      TezConfiguration.TEZ_APPLICATION_TAGS);
  appContext.setApplicationTags(new HashSet<String>(tagsFromConf));
  assertTrue(appContext.getApplicationTags().contains("foo"));
  assertTrue(appContext.getApplicationTags().contains("bar"));
}
Example 8
Source File: AthenaXYarnClusterDescriptor.java    From AthenaX with Apache License 2.0 4 votes vote down vote up 
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception {
  appContext.setMaxAppAttempts(MAX_ATTEMPT);

  Map<String, LocalResource> localResources = new HashMap<>();
  Set<Path> shippedPaths = new HashSet<>();
  collectLocalResources(localResources, shippedPaths);
  final ContainerLaunchContext amContainer = setupApplicationMasterContainer(
      this.getYarnSessionClusterEntrypoint(),
      false,
      true,
      false, (int) job.taskManagerMemoryMb());

  amContainer.setLocalResources(localResources);

  final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator));
  final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString())
      .collect(Collectors.joining(","));

  // Setup CLASSPATH and environment variables for ApplicationMaster
  ApplicationId appId = appContext.getApplicationId();
  final Map<String, String> appMasterEnv = setUpAmEnvironment(
      appId,
      classPath,
      shippedFiles,
      getDynamicPropertiesEncoded()
  );

  amContainer.setEnvironment(appMasterEnv);

  // Set up resource type requirements for ApplicationMaster
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(getFlinkConfiguration()
      .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY));
  capability.setVirtualCores(1);

  appContext.setApplicationName(job.name());
  appContext.setApplicationType(ATHENAX_APPLICATION_TYPE);
  appContext.setAMContainerSpec(amContainer);
  appContext.setResource(capability);
  appContext.setApplicationTags(Collections.singleton(job.metadata().serialize()));
  if (job.queue() != null) {
    appContext.setQueue(job.queue());
  }

  LOG.info("Submitting application master {}", appId);
  yarnClient.submitApplication(appContext);

  PollDeploymentStatus poll = new PollDeploymentStatus(appId);
  YARN_POLL_EXECUTOR.submit(poll);
  try {
    return poll.result.get();
  } catch (ExecutionException e) {
    LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause());
    yarnClient.killApplication(appId);
    throw (Exception) e.getCause();
  }
}