Java Code Examples for org.apache.hadoop.yarn.server.utils.BuilderUtils#newContainerLaunchContext()
The following examples show how to use
org.apache.hadoop.yarn.server.utils.BuilderUtils#newContainerLaunchContext() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: YarnServiceTest.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private void startApp() throws Exception { // submit a dummy app ApplicationSubmissionContext appSubmissionContext = yarnClient.createApplication().getApplicationSubmissionContext(); this.applicationId = appSubmissionContext.getApplicationId(); ContainerLaunchContext containerLaunchContext = BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap()); // Setup the application submission context appSubmissionContext.setApplicationName("TestApp"); appSubmissionContext.setResource(Resource.newInstance(128, 1)); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(containerLaunchContext); this.yarnClient.submitApplication(appSubmissionContext); // wait for application to be accepted int i; RMAppAttempt attempt = null; for (i = 0; i < 120; i++) { ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { this.applicationAttemptId = appReport.getCurrentApplicationAttemptId(); attempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt(); break; } Thread.sleep(1000); } Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state"); // Set the AM-RM token in the UGI for access during testing UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser() .getUserName())); UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken()); }
Example 2
Source File: YarnServiceTestWithExpiration.java From incubator-gobblin with Apache License 2.0 | 5 votes |
private void startApp() throws Exception { // submit a dummy app ApplicationSubmissionContext appSubmissionContext = yarnClient.createApplication().getApplicationSubmissionContext(); this.applicationId = appSubmissionContext.getApplicationId(); ContainerLaunchContext containerLaunchContext = BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "100"), Collections.emptyMap(), null, Collections.emptyMap()); // Setup the application submission context appSubmissionContext.setApplicationName("TestApp"); appSubmissionContext.setResource(Resource.newInstance(128, 1)); appSubmissionContext.setPriority(Priority.newInstance(0)); appSubmissionContext.setAMContainerSpec(containerLaunchContext); this.yarnClient.submitApplication(appSubmissionContext); // wait for application to be accepted int i; RMAppAttempt attempt = null; for (i = 0; i < 120; i++) { ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { this.applicationAttemptId = appReport.getCurrentApplicationAttemptId(); attempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(appReport.getCurrentApplicationAttemptId().getApplicationId()).getCurrentAppAttempt(); break; } Thread.sleep(1000); } Assert.assertTrue(i < 120, "timed out waiting for ACCEPTED state"); // Set the AM-RM token in the UGI for access during testing UserGroupInformation.setLoginUser(UserGroupInformation.createRemoteUser(UserGroupInformation.getCurrentUser() .getUserName())); UserGroupInformation.getCurrentUser().addToken(attempt.getAMRMToken()); }
Example 3
Source File: YarnServiceTestWithExpiration.java From incubator-gobblin with Apache License 2.0 | 5 votes |
protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName) throws IOException { try { Thread.sleep(1000); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } return BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "60000"), Collections.emptyMap(), null, Collections.emptyMap()); }
Example 4
Source File: TestRMAppAttemptTransitions.java From hadoop with Apache License 2.0 | 4 votes |
@SuppressWarnings("deprecation") @Before public void setUp() throws Exception { AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); InlineDispatcher rmDispatcher = new InlineDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); amLivelinessMonitor = mock(AMLivelinessMonitor.class); amFinishingMonitor = mock(AMLivelinessMonitor.class); writer = mock(RMApplicationHistoryWriter.class); MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey(); when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData); rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager, writer); store = mock(RMStateStore.class); ((RMContextImpl) rmContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); applicationMasterLauncher = mock(ApplicationMasterLauncher.class); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher()); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher()); rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher()); rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher()); rmnodeEventHandler = mock(RMNodeImpl.class); rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler); rmDispatcher.init(conf); rmDispatcher.start(); ApplicationId applicationId = MockApps.newAppID(appId++); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); resourceScheduler = mock(ResourceScheduler.class); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); when(appResUsgRpt.getGcoreSeconds()).thenReturn(0L); when(resourceScheduler .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1, 1); ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); unmanagedAM = false; application = mock(RMAppImpl.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); spyRMContext.getRMApps().put(application.getApplicationId(), application); testAppAttemptNewState(); }
Example 5
Source File: TestAMRMClient.java From hadoop with Apache License 2.0 | 4 votes |
@Before public void startApp() throws Exception { // submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( Collections.<String, LocalResource> emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1, 1)); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records .newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start RMAppAttempt appAttempt = null; while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); // emulate RM setup of AMRM token in credentials by adding the token // *before* setting the token service UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf)); }
Example 6
Source File: TestRMAppAttemptTransitions.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings("deprecation") @Before public void setUp() throws Exception { AuthenticationMethod authMethod = AuthenticationMethod.SIMPLE; if (isSecurityEnabled) { authMethod = AuthenticationMethod.KERBEROS; } SecurityUtil.setAuthenticationMethod(authMethod, conf); UserGroupInformation.setConfiguration(conf); InlineDispatcher rmDispatcher = new InlineDispatcher(); ContainerAllocationExpirer containerAllocationExpirer = mock(ContainerAllocationExpirer.class); amLivelinessMonitor = mock(AMLivelinessMonitor.class); amFinishingMonitor = mock(AMLivelinessMonitor.class); writer = mock(RMApplicationHistoryWriter.class); MasterKeyData masterKeyData = amRMTokenManager.createNewMasterKey(); when(amRMTokenManager.getMasterKey()).thenReturn(masterKeyData); rmContext = new RMContextImpl(rmDispatcher, containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor, null, amRMTokenManager, new RMContainerTokenSecretManager(conf), nmTokenManager, clientToAMTokenManager, writer); store = mock(RMStateStore.class); ((RMContextImpl) rmContext).setStateStore(store); publisher = mock(SystemMetricsPublisher.class); ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher); scheduler = mock(YarnScheduler.class); masterService = mock(ApplicationMasterService.class); applicationMasterLauncher = mock(ApplicationMasterLauncher.class); rmDispatcher.register(RMAppAttemptEventType.class, new TestApplicationAttemptEventDispatcher()); rmDispatcher.register(RMAppEventType.class, new TestApplicationEventDispatcher()); rmDispatcher.register(SchedulerEventType.class, new TestSchedulerEventDispatcher()); rmDispatcher.register(AMLauncherEventType.class, new TestAMLauncherEventDispatcher()); rmnodeEventHandler = mock(RMNodeImpl.class); rmDispatcher.register(RMNodeEventType.class, rmnodeEventHandler); rmDispatcher.init(conf); rmDispatcher.start(); ApplicationId applicationId = MockApps.newAppID(appId++); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0); resourceScheduler = mock(ResourceScheduler.class); ApplicationResourceUsageReport appResUsgRpt = mock(ApplicationResourceUsageReport.class); when(appResUsgRpt.getMemorySeconds()).thenReturn(0L); when(appResUsgRpt.getVcoreSeconds()).thenReturn(0L); when(resourceScheduler .getAppResourceUsageReport((ApplicationAttemptId)Matchers.any())) .thenReturn(appResUsgRpt); spyRMContext = spy(rmContext); Mockito.doReturn(resourceScheduler).when(spyRMContext).getScheduler(); final String user = MockApps.newUserName(); final String queue = MockApps.newQueue(); submissionContext = mock(ApplicationSubmissionContext.class); when(submissionContext.getQueue()).thenReturn(queue); Resource resource = BuilderUtils.newResource(1536, 1); ContainerLaunchContext amContainerSpec = BuilderUtils.newContainerLaunchContext(null, null, null, null, null, null); when(submissionContext.getAMContainerSpec()).thenReturn(amContainerSpec); when(submissionContext.getResource()).thenReturn(resource); unmanagedAM = false; application = mock(RMAppImpl.class); applicationAttempt = new RMAppAttemptImpl(applicationAttemptId, spyRMContext, scheduler, masterService, submissionContext, new Configuration(), false, BuilderUtils.newResourceRequest( RMAppAttemptImpl.AM_CONTAINER_PRIORITY, ResourceRequest.ANY, submissionContext.getResource(), 1)); when(application.getCurrentAppAttempt()).thenReturn(applicationAttempt); when(application.getApplicationId()).thenReturn(applicationId); spyRMContext.getRMApps().put(application.getApplicationId(), application); testAppAttemptNewState(); }
Example 7
Source File: TestAMRMClient.java From big-c with Apache License 2.0 | 4 votes |
@Before public void startApp() throws Exception { // submit new app ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); ApplicationId appId = appContext.getApplicationId(); // set the application name appContext.setApplicationName("Test"); // Set the priority for the application master Priority pri = Records.newRecord(Priority.class); pri.setPriority(0); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); // Set up the container launch context for the application master ContainerLaunchContext amContainer = BuilderUtils.newContainerLaunchContext( Collections.<String, LocalResource> emptyMap(), new HashMap<String, String>(), Arrays.asList("sleep", "100"), new HashMap<String, ByteBuffer>(), null, new HashMap<ApplicationAccessType, String>()); appContext.setAMContainerSpec(amContainer); appContext.setResource(Resource.newInstance(1024, 1)); // Create the request to send to the applications manager SubmitApplicationRequest appRequest = Records .newRecord(SubmitApplicationRequest.class); appRequest.setApplicationSubmissionContext(appContext); // Submit the application to the applications manager yarnClient.submitApplication(appContext); // wait for app to start RMAppAttempt appAttempt = null; while (true) { ApplicationReport appReport = yarnClient.getApplicationReport(appId); if (appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED) { attemptId = appReport.getCurrentApplicationAttemptId(); appAttempt = yarnCluster.getResourceManager().getRMContext().getRMApps() .get(attemptId.getApplicationId()).getCurrentAppAttempt(); while (true) { if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) { break; } } break; } } // Just dig into the ResourceManager and get the AMRMToken just for the sake // of testing. UserGroupInformation.setLoginUser(UserGroupInformation .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName())); // emulate RM setup of AMRM token in credentials by adding the token // *before* setting the token service UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken()); appAttempt.getAMRMToken().setService(ClientRMProxy.getAMRMTokenService(conf)); }
Example 8
Source File: YarnContainerProxy.java From incubator-tajo with Apache License 2.0 | 4 votes |
public ContainerLaunchContext createContainerLaunchContext(ContainerLaunchContext commonContainerLaunchContext) { // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); // Duplicate the ByteBuffers for access by multiple containers. Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); for (Map.Entry<String, ByteBuffer> entry : commonContainerLaunchContext.getServiceData().entrySet()) { myServiceData.put(entry.getKey(), entry.getValue().duplicate()); } //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size vargs.add("-Xmx2000m"); // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name //vargs.add(getRunnerClass()); vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("tr"); //workerMode vargs.add(getId()); // subqueryId vargs.add(containerMgrAddress); // nodeId vargs.add(containerID.toString()); // containerId Vector<CharSequence> taskParams = getTaskParams(); if(taskParams != null) { vargs.addAll(taskParams); } vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up TaskRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); return BuilderUtils.newContainerLaunchContext(commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>()); }
Example 9
Source File: YarnTajoResourceManager.java From incubator-tajo with Apache License 2.0 | 4 votes |
private ApplicationAttemptId allocateAndLaunchQueryMaster(QueryInProgress queryInProgress) throws IOException, YarnException { QueryId queryId = queryInProgress.getQueryId(); ApplicationId appId = ApplicationIdUtils.queryIdToAppId(queryId); LOG.info("Allocate and launch ApplicationMaster for QueryMaster: queryId=" + queryId + ", appId=" + appId); ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class); // set the application id appContext.setApplicationId(appId); // set the application name appContext.setApplicationName("Tajo"); Priority pri = Records.newRecord(Priority.class); pri.setPriority(5); appContext.setPriority(pri); // Set the queue to which this application is to be submitted in the RM appContext.setQueue("default"); ContainerLaunchContext commonContainerLaunchContext = YarnContainerProxy.createCommonContainerLaunchContext(masterContext.getConf(), queryId.toString(), true); // Setup environment by cloning from common env. Map<String, String> env = commonContainerLaunchContext.getEnvironment(); Map<String, String> myEnv = new HashMap<String, String>(env.size()); myEnv.putAll(env); //////////////////////////////////////////////////////////////////////////// // Set the local resources //////////////////////////////////////////////////////////////////////////// // Set the necessary command to execute the application master Vector<CharSequence> vargs = new Vector<CharSequence>(30); // Set java executable command //LOG.info("Setting up app master command"); vargs.add("${JAVA_HOME}" + "/bin/java"); // Set Xmx based on am memory size String jvmOptions = masterContext.getConf().get("tajo.rm.yarn.querymaster.jvm.option", "-Xmx2000m"); for(String eachToken: jvmOptions.split((" "))) { vargs.add(eachToken); } // Set Remote Debugging //if (!context.getQuery().getSubQuery(event.getExecutionBlockId()).isLeafQuery()) { //vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=5005"); //} // Set class name vargs.add(TajoWorker.class.getCanonicalName()); vargs.add("qm"); vargs.add(queryId.toString()); // queryId vargs.add(masterContext.getTajoMasterService().getBindAddress().getHostName() + ":" + masterContext.getTajoMasterService().getBindAddress().getPort()); vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); // Get final commmand StringBuilder command = new StringBuilder(); for (CharSequence str : vargs) { command.append(str).append(" "); } LOG.info("Completed setting up QueryMasterRunner command " + command.toString()); List<String> commands = new ArrayList<String>(); commands.add(command.toString()); final Resource resource = Records.newRecord(Resource.class); // TODO - get default value from conf resource.setMemory(2000); resource.setVirtualCores(1); Map<String, ByteBuffer> myServiceData = new HashMap<String, ByteBuffer>(); ContainerLaunchContext masterContainerContext = BuilderUtils.newContainerLaunchContext( commonContainerLaunchContext.getLocalResources(), myEnv, commands, myServiceData, null, new HashMap<ApplicationAccessType, String>(2) ); appContext.setAMContainerSpec(masterContainerContext); LOG.info("Submitting QueryMaster to ResourceManager"); yarnClient.submitApplication(appContext); ApplicationReport appReport = monitorApplication(appId, EnumSet.of(YarnApplicationState.ACCEPTED)); ApplicationAttemptId attemptId = appReport.getCurrentApplicationAttemptId(); LOG.info("Launching QueryMaster with appAttemptId: " + attemptId); return attemptId; }
Example 10
Source File: YarnServiceTest.java From incubator-gobblin with Apache License 2.0 | 4 votes |
protected ContainerLaunchContext newContainerLaunchContext(Container container, String helixInstanceName) throws IOException { return BuilderUtils.newContainerLaunchContext(Collections.emptyMap(), Collections.emptyMap(), Arrays.asList("sleep", "60000"), Collections.emptyMap(), null, Collections.emptyMap()); }