Java Code Examples for org.apache.hadoop.yarn.client.api.YarnClientApplication#getApplicationSubmissionContext()
The following examples show how to use
org.apache.hadoop.yarn.client.api.YarnClientApplication#getApplicationSubmissionContext() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: LaunchCluster.java From TensorFlowOnYARN with Apache License 2.0 | 5 votes |
ApplicationId submitApplication( YarnClientApplication app, String appName, ContainerLaunchContext launchContext, Resource resource, String queue) throws Exception { ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(appName); appContext.setApplicationTags(new HashSet<>()); appContext.setAMContainerSpec(launchContext); appContext.setResource(resource); appContext.setQueue(queue); return yarnClient.submitApplication(appContext); }
Example 2
Source File: YarnSubmissionHelper.java From reef with Apache License 2.0 | 5 votes |
public YarnSubmissionHelper(final YarnConfiguration yarnConfiguration, final REEFFileNames fileNames, final ClasspathProvider classpath, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider, final boolean isUnmanaged, final List<String> commandPrefixList) throws IOException, YarnException { this.classpath = classpath; this.yarnProxyUser = yarnProxyUser; this.isUnmanaged = isUnmanaged; this.driverStdoutFilePath = ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStdoutFileName(); this.driverStderrFilePath = ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/" + fileNames.getDriverStderrFileName(); LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting Application ID from YARN."); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationResponse = yarnClientApplication.getNewApplicationResponse(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setUnmanagedAM(isUnmanaged); this.applicationId = this.applicationSubmissionContext.getApplicationId(); this.tokenProvider = tokenProvider; this.commandPrefixList = commandPrefixList; this.configurationFilePaths = Collections.singletonList(fileNames.getDriverConfigurationPath()); LOG.log(Level.INFO, "YARN Application ID: {0}", this.applicationId); }
Example 3
Source File: UnmanagedAmYarnSubmissionHelper.java From reef with Apache License 2.0 | 5 votes |
UnmanagedAmYarnSubmissionHelper( final YarnConfiguration yarnConfiguration, final YarnProxyUser yarnProxyUser, final SecurityTokenProvider tokenProvider) throws IOException, YarnException { this.tokenProvider = tokenProvider; this.yarnProxyUser = yarnProxyUser; LOG.log(Level.FINE, "Initializing YARN Client"); this.yarnClient = YarnClient.createYarnClient(); this.yarnClient.init(yarnConfiguration); this.yarnClient.start(); LOG.log(Level.FINE, "Initialized YARN Client"); LOG.log(Level.FINE, "Requesting UNMANAGED Application ID from YARN."); final ContainerLaunchContext launchContext = YarnTypes.getContainerLaunchContext( Collections.<String>emptyList(), Collections.<String, LocalResource>emptyMap(), tokenProvider.getTokens()); final YarnClientApplication yarnClientApplication = this.yarnClient.createApplication(); this.applicationSubmissionContext = yarnClientApplication.getApplicationSubmissionContext(); this.applicationSubmissionContext.setAMContainerSpec(launchContext); this.applicationSubmissionContext.setUnmanagedAM(true); this.applicationId = this.applicationSubmissionContext.getApplicationId(); LOG.log(Level.INFO, "YARN UNMANAGED Application ID: {0}", this.applicationId); }
Example 4
Source File: AppClient.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
public void run(VMConfig vmConfig, Configuration conf) throws Exception { try { vmConfig.overrideHadoopConfiguration(conf); System.out.println("Create YarnClient") ; YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); System.out.println("Create YarnClientApplication via YarnClient") ; YarnClientApplication app = yarnClient.createApplication(); String appId = app.getApplicationSubmissionContext().getApplicationId().toString() ; System.out.println("Application Id = " + appId) ; System.out.println("Set up the container launch context for the application master") ; ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); StringBuilder sb = new StringBuilder(); List<String> commands = Collections.singletonList( sb.append(vmConfig.buildCommand()). append(" 1> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stdout"). append(" 2> ").append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append("/stderr") .toString() ); amContainer.setCommands(commands) ; System.out.println("Setup the app classpath and resources") ; if(vmConfig.getVmResources().size() > 0) { amContainer.setLocalResources(new VMResources(conf, vmConfig)); } System.out.println("Setup the classpath for ApplicationMaster, environment = " + vmConfig.getEnvironment()) ; Map<String, String> appMasterEnv = new HashMap<String, String>(); boolean jvmEnv = vmConfig.getEnvironment() != VMConfig.Environment.YARN; Util.setupAppMasterEnv(jvmEnv , conf, appMasterEnv); amContainer.setEnvironment(appMasterEnv); System.out.println("Set up resource type requirements for ApplicationMaster") ; Resource resource = Records.newRecord(Resource.class); resource.setMemory(256); resource.setVirtualCores(1); System.out.println("Finally, set-up ApplicationSubmissionContext for the application"); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName(vmConfig.getName()); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(resource); appContext.setQueue("default"); // queue // Submit application ApplicationId applicationId = appContext.getApplicationId(); System.out.println("Submitting application " + applicationId); yarnClient.submitApplication(appContext); } catch(Exception ex) { ex.printStackTrace(); throw ex ; } }
Example 5
Source File: Client.java From hadoop-mini-clusters with Apache License 2.0 | 4 votes |
public void run(String[] args) throws Exception { final String command = args[0]; final int n = Integer.valueOf(args[1]); final Path jarPath = new Path(args[2]); final String resourceManagerAddress = args[3]; final String resourceManagerHostname = args[4]; final String resourceManagerSchedulerAddress = args[5]; final String resourceManagerResourceTrackerAddress = args[6]; // Create yarnClient YarnConfiguration conf = new YarnConfiguration(); conf.set("yarn.resourcemanager.address", resourceManagerAddress); conf.set("yarn.resourcemanager.hostname", resourceManagerHostname); conf.set("yarn.resourcemanager.scheduler.address", resourceManagerSchedulerAddress); conf.set("yarn.resourcemanager.resource-tracker.address", resourceManagerResourceTrackerAddress); YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(conf); yarnClient.start(); // Create application via yarnClient YarnClientApplication app = yarnClient.createApplication(); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); amContainer.setCommands( Collections.singletonList( "$JAVA_HOME/bin/java" + " -Xmx256M" + " com.hortonworks.simpleyarnapp.ApplicationMaster" + " " + command + " " + String.valueOf(n) + " " + resourceManagerAddress + " " + resourceManagerHostname + " " + resourceManagerSchedulerAddress + " " + resourceManagerResourceTrackerAddress + " 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" + " 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr" ) ); // Setup jar for ApplicationMaster LocalResource appMasterJar = Records.newRecord(LocalResource.class); setupAppMasterJar(jarPath, appMasterJar); amContainer.setLocalResources( Collections.singletonMap("simple-yarn-app-1.1.0.jar", appMasterJar)); // Setup CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource type requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(256); capability.setVirtualCores(1); // Finally, set-up ApplicationSubmissionContext for the application ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); appContext.setApplicationName("simple-yarn-app"); // application name appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // queue // Submit application ApplicationId appId = appContext.getApplicationId(); System.out.println("Submitting application " + appId); yarnClient.submitApplication(appContext); ApplicationReport appReport = yarnClient.getApplicationReport(appId); YarnApplicationState appState = appReport.getYarnApplicationState(); while (appState != YarnApplicationState.FINISHED && appState != YarnApplicationState.KILLED && appState != YarnApplicationState.FAILED) { Thread.sleep(100); appReport = yarnClient.getApplicationReport(appId); appState = appReport.getYarnApplicationState(); } System.out.println( "Application " + appId + " finished with" + " state " + appState + " at " + appReport.getFinishTime()); }
Example 6
Source File: GobblinYarnAppLauncher.java From incubator-gobblin with Apache License 2.0 | 4 votes |
/** * Setup and submit the Gobblin Yarn application. * * @throws IOException if there's anything wrong setting up and submitting the Yarn application * @throws YarnException if there's anything wrong setting up and submitting the Yarn application */ @VisibleForTesting ApplicationId setupAndSubmitApplication() throws IOException, YarnException { YarnClientApplication gobblinYarnApp = this.yarnClient.createApplication(); ApplicationSubmissionContext appSubmissionContext = gobblinYarnApp.getApplicationSubmissionContext(); appSubmissionContext.setApplicationType(GOBBLIN_YARN_APPLICATION_TYPE); appSubmissionContext.setMaxAppAttempts(ConfigUtils.getInt(config, GobblinYarnConfigurationKeys.APP_MASTER_MAX_ATTEMPTS_KEY, GobblinYarnConfigurationKeys.DEFAULT_APP_MASTER_MAX_ATTEMPTS_KEY)); ApplicationId applicationId = appSubmissionContext.getApplicationId(); GetNewApplicationResponse newApplicationResponse = gobblinYarnApp.getNewApplicationResponse(); // Set up resource type requirements for ApplicationMaster Resource resource = prepareContainerResource(newApplicationResponse); // Add lib jars, and jars and files that the ApplicationMaster need as LocalResources Map<String, LocalResource> appMasterLocalResources = addAppMasterLocalResources(applicationId); ContainerLaunchContext amContainerLaunchContext = Records.newRecord(ContainerLaunchContext.class); amContainerLaunchContext.setLocalResources(appMasterLocalResources); amContainerLaunchContext.setEnvironment(YarnHelixUtils.getEnvironmentVariables(this.yarnConfiguration)); amContainerLaunchContext.setCommands(Lists.newArrayList(buildApplicationMasterCommand(applicationId.toString(), resource.getMemory()))); Map<ApplicationAccessType, String> acls = new HashMap<>(1); acls.put(ApplicationAccessType.VIEW_APP, this.appViewAcl); amContainerLaunchContext.setApplicationACLs(acls); if (UserGroupInformation.isSecurityEnabled()) { setupSecurityTokens(amContainerLaunchContext); } // Setup the application submission context appSubmissionContext.setApplicationName(this.applicationName); appSubmissionContext.setResource(resource); appSubmissionContext.setQueue(this.appQueueName); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(amContainerLaunchContext); // Also setup container local resources by copying local jars and files the container need to HDFS addContainerLocalResources(applicationId); // Submit the application LOGGER.info("Submitting application " + sanitizeApplicationId(applicationId.toString())); this.yarnClient.submitApplication(appSubmissionContext); LOGGER.info("Application successfully submitted and accepted"); ApplicationReport applicationReport = this.yarnClient.getApplicationReport(applicationId); LOGGER.info("Application Name: " + applicationReport.getName()); LOGGER.info("Application Tracking URL: " + applicationReport.getTrackingUrl()); LOGGER.info("Application User: " + applicationReport.getUser() + " Queue: " + applicationReport.getQueue()); return applicationId; }
Example 7
Source File: MyClient.java From yarn-beginners-examples with Apache License 2.0 | 4 votes |
/** * Main run function for the client * @return true if application completed successfully * @throws java.io.IOException * @throws org.apache.hadoop.yarn.exceptions.YarnException */ public boolean run() throws IOException, YarnException { LOG.info("Running Client"); yarnClient.start(); // Get a new application id YarnClientApplication app = yarnClient.createApplication(); GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); int maxMem = appResponse.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); // A resource ask cannot exceed the max. if (amMemory > maxMem) { LOG.info("AM memory specified above max threshold of cluster. Using max value." + ", specified=" + amMemory + ", max=" + maxMem); amMemory = maxMem; } int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); if (amVCores > maxVCores) { LOG.info("AM virtual cores specified above max threshold of cluster. " + "Using max value." + ", specified=" + amVCores + ", max=" + maxVCores); amVCores = maxVCores; } // set the application name ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); appContext.setApplicationName(appName); // Set up resource type requirements // For now, both memory and vcores are supported, so we set memory and // vcores requirements Resource capability = Records.newRecord(Resource.class); capability.setMemory(amMemory); capability.setVirtualCores(amVCores); appContext.setResource(capability); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(amPriority); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue(amQueue); // Set the ContainerLaunchContext to describe the Container ith which the ApplicationMaster is launched. appContext.setAMContainerSpec(getAMContainerSpec(appId.getId())); // Submit the application to the applications manager // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); // Ignore the response as either a valid response object is returned on success // or an exception thrown to denote some form of a failure LOG.info("Submitting application to ASM"); yarnClient.submitApplication(appContext); // Monitor the application return monitorApplication(appId); }
Example 8
Source File: Client.java From Scribengin with GNU Affero General Public License v3.0 | 4 votes |
public ApplicationId run() throws IOException, YarnException { LOG.info("calling run."); yarnClient.start(); // YarnClientApplication is used to populate: // 1. GetNewApplication Response // 2. ApplicationSubmissionContext YarnClientApplication app = yarnClient.createApplication(); // GetNewApplicationResponse can be used to determined resources available. GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); this.appId = appContext.getApplicationId(); appContext.setApplicationName(this.appname); // Set up the container launch context for AM. ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); LocalResource appMasterJar; FileSystem fs = FileSystem.get(this.conf); amContainer.setLocalResources( Collections.singletonMap("master.jar", Util.newYarnAppResource(fs, new Path(this.hdfsJar), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION))); // Set up CLASSPATH for ApplicationMaster Map<String, String> appMasterEnv = new HashMap<String, String>(); setupAppMasterEnv(appMasterEnv); amContainer.setEnvironment(appMasterEnv); // Set up resource requirements for ApplicationMaster Resource capability = Records.newRecord(Resource.class); capability.setMemory(this.applicationMasterMem); capability.setVirtualCores(1); //TODO: Can we really setVirtualCores ? amContainer.setCommands(Collections.singletonList(this.getAppMasterCommand())); // put everything together. appContext.setAMContainerSpec(amContainer); appContext.setResource(capability); appContext.setQueue("default"); // TODO: Need to investigate more on queuing an scheduling. // Submit application yarnClient.submitApplication(appContext); LOG.info("APPID: "+this.appId.toString()); return this.appId; //return this.monitorApplication(appId); }