Java Code Examples for org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setMaxAppAttempts()
The following examples show how to use
org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext#setMaxAppAttempts() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: BuilderUtils.java From hadoop with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 2
Source File: BuilderUtils.java From big-c with Apache License 2.0 | 6 votes |
public static ApplicationSubmissionContext newApplicationSubmissionContext( ApplicationId applicationId, String applicationName, String queue, Priority priority, ContainerLaunchContext amContainer, boolean isUnmanagedAM, boolean cancelTokensWhenComplete, int maxAppAttempts, Resource resource, String applicationType) { ApplicationSubmissionContext context = recordFactory.newRecordInstance(ApplicationSubmissionContext.class); context.setApplicationId(applicationId); context.setApplicationName(applicationName); context.setQueue(queue); context.setPriority(priority); context.setAMContainerSpec(amContainer); context.setUnmanagedAM(isUnmanagedAM); context.setCancelTokensWhenComplete(cancelTokensWhenComplete); context.setMaxAppAttempts(maxAppAttempts); context.setResource(resource); context.setApplicationType(applicationType); return context; }
Example 3
Source File: YarnRemoteInterpreterProcess.java From zeppelin with Apache License 2.0 | 6 votes |
private ApplicationSubmissionContext createApplicationSubmissionContext( ApplicationSubmissionContext appContext) throws Exception { setResources(appContext); setPriority(appContext); setQueue(appContext); appContext.setApplicationId(appId); setApplicationName(appContext); appContext.setApplicationType("ZEPPELIN INTERPRETER"); appContext.setMaxAppAttempts(1); ContainerLaunchContext amContainer = setUpAMLaunchContext(); appContext.setAMContainerSpec(amContainer); appContext.setCancelTokensWhenComplete(true); return appContext; }
Example 4
Source File: TezClient.java From tez with Apache License 2.0 | 6 votes |
private ApplicationSubmissionContext setupApplicationContext() throws IOException, YarnException { TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, amConfig.getTezConfiguration()); Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials); // Add session token for shuffle TezClientUtils.createSessionToken(sessionAppId.toString(), jobTokenSecretManager, sessionCredentials); ApplicationSubmissionContext appContext = TezClientUtils.createApplicationSubmissionContext( sessionAppId, null, clientName, amConfig, tezJarResources, sessionCredentials, usingTezArchiveDeploy, apiVersionInfo, servicePluginsDescriptor, javaOptsChecker); // Set Tez Sessions to not retry on AM crashes if recovery is disabled if (!amConfig.getTezConfiguration().getBoolean( TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { appContext.setMaxAppAttempts(1); } return appContext; }
Example 5
Source File: SylphSparkYarnClient.java From sylph with Apache License 2.0 | 5 votes |
@Override public ApplicationSubmissionContext createApplicationSubmissionContext(YarnClientApplication newApp, ContainerLaunchContext containerContext) { final ApplicationSubmissionContext appContext = super.createApplicationSubmissionContext(newApp, containerContext); appContext.setApplicationType("Sylph_SPARK"); appContext.setApplicationTags(ImmutableSet.of("a1", "a2")); appContext.setQueue(yarnQueue); appContext.setMaxAppAttempts(2); return appContext; }
Example 6
Source File: DeployerApplicationYarnClient.java From spring-cloud-deployer-yarn with Apache License 2.0 | 5 votes |
@Override protected ApplicationSubmissionContext getSubmissionContext(ApplicationId applicationId) { ApplicationSubmissionContext submissionContext = super.getSubmissionContext(applicationId); // force app fail after first appmaster failure regardles // what's set on yarn level. // TODO: should make configurable in spring yarn if possible submissionContext.setMaxAppAttempts(1); return submissionContext; }
Example 7
Source File: Hadoop21YarnAppClient.java From twill with Apache License 2.0 | 5 votes |
/** * Updates the {@link ApplicationSubmissionContext} based on configuration. */ protected void configureAppSubmissionContext(ApplicationSubmissionContext context) { int maxAttempts = configuration.getInt(Configs.Keys.YARN_MAX_APP_ATTEMPTS, -1); if (maxAttempts > 0) { context.setMaxAppAttempts(maxAttempts); } else { // Preserve the old behavior context.setMaxAppAttempts(2); } }
Example 8
Source File: AthenaXYarnClusterDescriptor.java From AthenaX with Apache License 2.0 | 4 votes |
private ApplicationReport startAppMaster(ApplicationSubmissionContext appContext) throws Exception { appContext.setMaxAppAttempts(MAX_ATTEMPT); Map<String, LocalResource> localResources = new HashMap<>(); Set<Path> shippedPaths = new HashSet<>(); collectLocalResources(localResources, shippedPaths); final ContainerLaunchContext amContainer = setupApplicationMasterContainer( this.getYarnSessionClusterEntrypoint(), false, true, false, (int) job.taskManagerMemoryMb()); amContainer.setLocalResources(localResources); final String classPath = localResources.keySet().stream().collect(Collectors.joining(File.pathSeparator)); final String shippedFiles = shippedPaths.stream().map(x -> x.getName() + "=" + x.toString()) .collect(Collectors.joining(",")); // Setup CLASSPATH and environment variables for ApplicationMaster ApplicationId appId = appContext.getApplicationId(); final Map<String, String> appMasterEnv = setUpAmEnvironment( appId, classPath, shippedFiles, getDynamicPropertiesEncoded() ); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(getFlinkConfiguration() .getInteger(JobManagerOptions.JOB_MANAGER_HEAP_MEMORY)); capability.setVirtualCores(1); appContext.setApplicationName(job.name()); appContext.setApplicationType(ATHENAX_APPLICATION_TYPE); appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setApplicationTags(Collections.singleton(job.metadata().serialize())); if (job.queue() != null) { appContext.setQueue(job.queue()); } LOG.info("Submitting application master {}", appId); yarnClient.submitApplication(appContext); PollDeploymentStatus poll = new PollDeploymentStatus(appId); YARN_POLL_EXECUTOR.submit(poll); try { return poll.result.get(); } catch (ExecutionException e) { LOG.warn("Failed to deploy {}, cause: {}", appId.toString(), e.getCause()); yarnClient.killApplication(appId); throw (Exception) e.getCause(); } }
Example 9
Source File: AMSimulator.java From hadoop with Apache License 2.0 | 4 votes |
private void submitApp() throws YarnException, InterruptedException, IOException { // ask for new application GetNewApplicationRequest newAppRequest = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse newAppResponse = rm.getClientRMService().getNewApplication(newAppRequest); appId = newAppResponse.getApplicationId(); // submit the application final SubmitApplicationRequest subAppRequest = Records.newRecord(SubmitApplicationRequest.class); ApplicationSubmissionContext appSubContext = Records.newRecord(ApplicationSubmissionContext.class); appSubContext.setApplicationId(appId); appSubContext.setMaxAppAttempts(1); appSubContext.setQueue(queue); appSubContext.setPriority(Priority.newInstance(0)); ContainerLaunchContext conLauContext = Records.newRecord(ContainerLaunchContext.class); conLauContext.setApplicationACLs( new HashMap<ApplicationAccessType, String>()); conLauContext.setCommands(new ArrayList<String>()); conLauContext.setEnvironment(new HashMap<String, String>()); conLauContext.setLocalResources(new HashMap<String, LocalResource>()); conLauContext.setServiceData(new HashMap<String, ByteBuffer>()); appSubContext.setAMContainerSpec(conLauContext); appSubContext.setUnmanagedAM(true); subAppRequest.setApplicationSubmissionContext(appSubContext); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws YarnException { rm.getClientRMService().submitApplication(subAppRequest); return null; } }); LOG.info(MessageFormat.format("Submit a new application {0}", appId)); // waiting until application ACCEPTED RMApp app = rm.getRMContext().getRMApps().get(appId); while(app.getState() != RMAppState.ACCEPTED) { Thread.sleep(10); } // Waiting until application attempt reach LAUNCHED // "Unmanaged AM must register after AM attempt reaches LAUNCHED state" this.appAttemptId = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt().getAppAttemptId(); RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt(); while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) { Thread.sleep(10); } }
Example 10
Source File: AMSimulator.java From big-c with Apache License 2.0 | 4 votes |
private void submitApp() throws YarnException, InterruptedException, IOException { // ask for new application GetNewApplicationRequest newAppRequest = Records.newRecord(GetNewApplicationRequest.class); GetNewApplicationResponse newAppResponse = rm.getClientRMService().getNewApplication(newAppRequest); appId = newAppResponse.getApplicationId(); // submit the application final SubmitApplicationRequest subAppRequest = Records.newRecord(SubmitApplicationRequest.class); ApplicationSubmissionContext appSubContext = Records.newRecord(ApplicationSubmissionContext.class); appSubContext.setApplicationId(appId); appSubContext.setMaxAppAttempts(1); appSubContext.setQueue(queue); appSubContext.setPriority(Priority.newInstance(0)); ContainerLaunchContext conLauContext = Records.newRecord(ContainerLaunchContext.class); conLauContext.setApplicationACLs( new HashMap<ApplicationAccessType, String>()); conLauContext.setCommands(new ArrayList<String>()); conLauContext.setEnvironment(new HashMap<String, String>()); conLauContext.setLocalResources(new HashMap<String, LocalResource>()); conLauContext.setServiceData(new HashMap<String, ByteBuffer>()); appSubContext.setAMContainerSpec(conLauContext); appSubContext.setUnmanagedAM(true); subAppRequest.setApplicationSubmissionContext(appSubContext); UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user); ugi.doAs(new PrivilegedExceptionAction<Object>() { @Override public Object run() throws YarnException { rm.getClientRMService().submitApplication(subAppRequest); return null; } }); LOG.info(MessageFormat.format("Submit a new application {0}", appId)); // waiting until application ACCEPTED RMApp app = rm.getRMContext().getRMApps().get(appId); while(app.getState() != RMAppState.ACCEPTED) { Thread.sleep(10); } // Waiting until application attempt reach LAUNCHED // "Unmanaged AM must register after AM attempt reaches LAUNCHED state" this.appAttemptId = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt().getAppAttemptId(); RMAppAttempt rmAppAttempt = rm.getRMContext().getRMApps().get(appId) .getCurrentAppAttempt(); while (rmAppAttempt.getAppAttemptState() != RMAppAttemptState.LAUNCHED) { Thread.sleep(10); } }
Example 11
Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0 | 4 votes |
/** * Setup and submit the Gobblin Yarn application. * * @throws IOException if there's anything wrong setting up and submitting the Yarn application * @throws YarnException if there's anything wrong setting up and submitting the Yarn application */ @VisibleForTesting ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY)); ApplicationId applicationId = appSubmissionContext.getApplicationId(); GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse(); // Set up resource type requirements for ApplicationMaster Resource resource = prepareContainerResource(newApplicationResponse); // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId); ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory()))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); amContainerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); } // Setup the application submission context appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); // Also setup container local resources by copying local jars and files the container need to HDFS addContainerLocalResources(applicationId); // Submit the application LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString())); this.yarnClient.submitApplication(appSubmissionContext); LOGGER.info("Application successfully submitted and accepted"); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue()); return applicationId; }
Example 12
Source File: TezClient.java From incubator-tez with Apache License 2.0 | 4 votes |
/** * Start the client. This establishes a connection to the YARN cluster. * In session mode, this start the App Master thats runs all the DAGs in the * session. * @throws TezException * @throws IOException */ public synchronized void start() throws TezException, IOException { amConfig.setYarnConfiguration(new YarnConfiguration(amConfig.getTezConfiguration())); yarnClient = createYarnClient(); yarnClient.init(amConfig.getYarnConfiguration()); yarnClient.start(); if (isSession) { LOG.info("Session mode. Starting session."); TezClientUtils.processTezLocalCredentialsFile(sessionCredentials, amConfig.getTezConfiguration()); Map<String, LocalResource> tezJarResources = getTezJarResources(sessionCredentials); clientTimeout = amConfig.getTezConfiguration().getInt( TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS, TezConfiguration.TEZ_SESSION_CLIENT_TIMEOUT_SECS_DEFAULT); try { if (sessionAppId == null) { sessionAppId = createApplication(); } // Add session token for shuffle TezClientUtils.createSessionToken(sessionAppId.toString(), jobTokenSecretManager, sessionCredentials); ApplicationSubmissionContext appContext = TezClientUtils.createApplicationSubmissionContext( amConfig.getTezConfiguration(), sessionAppId, null, clientName, amConfig, tezJarResources, sessionCredentials); // Set Tez Sessions to not retry on AM crashes if recovery is disabled if (!amConfig.getTezConfiguration().getBoolean( TezConfiguration.DAG_RECOVERY_ENABLED, TezConfiguration.DAG_RECOVERY_ENABLED_DEFAULT)) { appContext.setMaxAppAttempts(1); } yarnClient.submitApplication(appContext); sessionStarted = true; } catch (YarnException e) { throw new TezException(e); } } }