Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationType()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setApplicationType() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BuilderUtils.java From hadoop with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 2
Source File: TestClientRMService.java From hadoop with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
Example 3
Source File: BuilderUtils.java From big-c with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 4
Source File: TestClientRMService.java From big-c with Apache License 2.0 | 6 votes |
@SuppressWarnings("deprecation") private SubmitApplicationRequest mockSubmitAppRequest(ApplicationId appId, String name, String queue, Set<String> tags, boolean unmanaged) { ContainerLaunchContext amContainerSpec = mock(ContainerLaunchContext.class); Resource resource = Resources.createResource( YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); ApplicationSubmissionContext submissionContext = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); submissionContext.setAMContainerSpec(amContainerSpec); submissionContext.setApplicationName(name); submissionContext.setQueue(queue); submissionContext.setApplicationId(appId); submissionContext.setResource(resource); submissionContext.setApplicationType(appType); submissionContext.setApplicationTags(tags); submissionContext.setUnmanagedAM(unmanaged); SubmitApplicationRequest submitRequest = recordFactory.newRecordInstance(SubmitApplicationRequest.class); submitRequest.setApplicationSubmissionContext(submissionContext); return submitRequest; }
Example 5
Source File: YarnRemoteInterpreterProcess.java From zeppelin with Apache License 2.0 | 6 votes |
private ApplicationSubmissionContext createApplicationSubmissionContext( ApplicationSubmissionContext appContext) throws Exception { setResources(appContext); setPriority(appContext); setQueue(appContext); appContext.setApplicationId(appId); setApplicationName(appContext); appContext.setApplicationType("ZEPPELIN INTERPRETER"); appContext.setMaxAppAttempts(1); ContainerLaunchContext amContainer = setUpAMLaunchContext(); appContext.setAMContainerSpec(amContainer); appContext.setCancelTokensWhenComplete(true); return appContext; }
Example 6
Source File: SylphSparkYarnClient.java From sylph with Apache License 2.0 | 5 votes |
@Override public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext) { final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext); appContext.setApplicationType("Sylph_SPARK"); appContext.setApplicationTags(ImmutableSet.of("a1", "a2")); appContext.setQueue(yarnQueue); appContext.setMaxAppAttempts(2); return appContext; }
Example 7
Source File: AthenaXYarnClusterDescriptor.java From AthenaX with Apache License 2.0 | 4 votes |
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception { appContext.setMaxAppAttempts(MAX_ATTEMPT); Map<String, LocalResource> localResources = new HashMap<>(); Set<Path> shippedPaths = new HashSet<>(); collectLocalResources(localResources, shippedPaths); final ContainerLaunchContext amContainer = setupApplicationMasterContainer( this.getYarnSessionClusterEntrypoint(), false, true, false, (int) job.taskManagerMemoryMb()); amContainer.setLocalResources(localResources); final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator)); final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString()) .collect(Collectors.joining(",")); // Setup CLASSPATH and environment variables for ApplicationMaster ApplicationId appId = appContext.getApplicationId(); final Map<String, String> appMasterEnv = setUpAmEnvironment( appId, classPath, shippedFiles, getDynamicPropertiesEncoded() ); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(getFlinkConfiguration() .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY)); capability.setVirtualCores(1); appContext.setApplicationName(job.name()); appContext.setApplicationType(ATHENAX_APPLICATION_TYPE); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setApplicationTags(Collections.singleton(job.metadata().serialize())); if (job.queue() != null) { appContext.setQueue(job.queue()); } LOG.info("Submitting application master {}", appId); yarnClient.submitApplication(appContext); PollDeploymentStatus poll = new PollDeploymentStatus(appId); YARN_POLL_EXECUTOR.submit(poll); try { return poll.result.get(); } catch (ExecutionException e) { LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause()); yarnClient.killApplication(appId); throw (Exception) e.getCause(); } }
Example 8
Source File: ClientRMService.java From hadoop with Apache License 2.0 | 4 votes |
@Override public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException { ApplicationSubmissionContext submissionContext = request .getApplicationSubmissionContext(); ApplicationId applicationId = submissionContext.getApplicationId(); // ApplicationSubmissionContext needs to be validated for safety - only // those fields that are independent of the RM's configuration will be // checked here, those that are dependent on RM configuration are validated // in RMAppManager. String user = null; try { // Safety user = UserGroupInformation.getCurrentUser().getShortUserName(); } catch (IOException ie) { LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", "Exception in submitting application", applicationId); throw RPCUtil.getRemoteException(ie); } // Check whether app has already been put into rmContext, // If it is, simply return the response if (rmContext.getRMApps().get(applicationId) != null) { LOG.info("This is an earlier submitted application: " + applicationId); return SubmitApplicationResponse.newInstance(); } if (submissionContext.getQueue() == null) { submissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); } if (submissionContext.getApplicationName() == null) { submissionContext.setApplicationName( YarnConfiguration.DEFAULT_APPLICATION_NAME); } if (submissionContext.getApplicationType() == null) { submissionContext .setApplicationType(YarnConfiguration.DEFAULT_APPLICATION_TYPE); } else { if (submissionContext.getApplicationType().length() > YarnConfiguration.APPLICATION_TYPE_LENGTH) { submissionContext.setApplicationType(submissionContext .getApplicationType().substring(0, YarnConfiguration.APPLICATION_TYPE_LENGTH)); } } try { // call RMAppManager to submit application directly rmAppManager.submitApplication(submissionContext, System.currentTimeMillis(), user); LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, "ClientRMService", applicationId); } catch (YarnException e) { LOG.info("Exception in submitting application with id " + applicationId.getId(), e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId); throw e; } SubmitApplicationResponse response = recordFactory .newRecordInstance(SubmitApplicationResponse.class); return response; }
Example 9
Source File: ClientRMService.java From big-c with Apache License 2.0 | 4 votes |
@Override public SubmitApplicationResponse submitApplication( SubmitApplicationRequest request) throws YarnException { ApplicationSubmissionContext submissionContext = request .getApplicationSubmissionContext(); ApplicationId applicationId = submissionContext.getApplicationId(); // ApplicationSubmissionContext needs to be validated for safety - only // those fields that are independent of the RM's configuration will be // checked here, those that are dependent on RM configuration are validated // in RMAppManager. String user = null; try { // Safety user = UserGroupInformation.getCurrentUser().getShortUserName(); } catch (IOException ie) { LOG.warn("Unable to get the current user.", ie); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, ie.getMessage(), "ClientRMService", "Exception in submitting application", applicationId); throw RPCUtil.getRemoteException(ie); } // Check whether app has already been put into rmContext, // If it is, simply return the response if (rmContext.getRMApps().get(applicationId) != null) { LOG.info("This is an earlier submitted application: " + applicationId); return SubmitApplicationResponse.newInstance(); } if (submissionContext.getQueue() == null) { submissionContext.setQueue(YarnConfiguration.DEFAULT_QUEUE_NAME); } if (submissionContext.getApplicationName() == null) { submissionContext.setApplicationName( YarnConfiguration.DEFAULT_APPLICATION_NAME); } if (submissionContext.getApplicationType() == null) { submissionContext .setApplicationType(YarnConfiguration.DEFAULT_APPLICATION_TYPE); } else { if (submissionContext.getApplicationType().length() > YarnConfiguration.APPLICATION_TYPE_LENGTH) { submissionContext.setApplicationType(submissionContext .getApplicationType().substring(0, YarnConfiguration.APPLICATION_TYPE_LENGTH)); } } try { // call RMAppManager to submit application directly rmAppManager.submitApplication(submissionContext, System.currentTimeMillis(), user); LOG.info("Application with id " + applicationId.getId() + " submitted by user " + user); RMAuditLogger.logSuccess(user, AuditConstants.SUBMIT_APP_REQUEST, "ClientRMService", applicationId); } catch (YarnException e) { LOG.info("Exception in submitting application with id " + applicationId.getId(), e); RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST, e.getMessage(), "ClientRMService", "Exception in submitting application", applicationId); throw e; } SubmitApplicationResponse response = recordFactory .newRecordInstance(SubmitApplicationResponse.class); return response; }
Example 10
Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0 | 4 votes |
/** * Setup and submit the Gobblin Yarn application. * * @throws IOException if there's anything wrong setting up and submitting the Yarn application * @throws YarnException if there's anything wrong setting up and submitting the Yarn application */ @VisibleForTesting ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY)); ApplicationId applicationId = appSubmissionContext.getApplicationId(); GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse(); // Set up resource type requirements for ApplicationMaster Resource resource = prepareContainerResource(newApplicationResponse); // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId); ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory()))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); amContainerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); } // Setup the application submission context appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); // Also setup container local resources by copying local jars and files the container need to HDFS addContainerLocalResources(applicationId); // Submit the application LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString())); this.yarnClient.submitApplication(appSubmissionContext); LOGGER.info("Application successfully submitted and accepted"); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue()); return applicationId; }