Java Code Examples for org.apache.hadoop.yarn.client.api.async.AMRMClientAsync#createAMRMClientAsync()
The following examples show how to use
org.apache.hadoop.yarn.client.api.async.AMRMClientAsync#createAMRMClientAsync() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestAMRMClientAsync.java From hadoop with Apache License 2.0 | 6 votes |
@Test (timeout = 10000) public void testAMRMClientAsyncShutDown() throws Exception { Configuration conf = new Configuration(); TestCallbackHandler callbackHandler = new TestCallbackHandler(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); createAllocateResponse(new ArrayList<ContainerStatus>(), new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenThrow( new ApplicationAttemptNotFoundException("app not found, shut down")); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler); asyncClient.init(conf); asyncClient.start(); asyncClient.registerApplicationMaster("localhost", 1234, null); Thread.sleep(50); verify(client, times(1)).allocate(anyFloat()); asyncClient.stop(); }
Example 2
Source File: YarnResourceManager.java From flink with Apache License 2.0 | 5 votes |
protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient( YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, @Nullable String webInterfaceUrl) throws Exception { AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync( yarnHeartbeatIntervalMillis, this); resourceManagerClient.init(yarnConfiguration); resourceManagerClient.start(); //TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address Tuple2<String, Integer> hostPort = parseHostPort(getAddress()); final int restPort; if (webInterfaceUrl != null) { final int lastColon = webInterfaceUrl.lastIndexOf(':'); if (lastColon == -1) { restPort = -1; } else { restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1)); } } else { restPort = -1; } final RegisterApplicationMasterResponse registerApplicationMasterResponse = resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl); getContainersFromPreviousAttempts(registerApplicationMasterResponse); return resourceManagerClient; }
Example 3
Source File: YarnContainerManager.java From reef with Apache License 2.0 | 5 votes |
@Inject private YarnContainerManager( @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod, @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory, final YarnConfiguration yarnConf, final YarnProxyUser yarnProxyUser, final REEFEventHandlers reefEventHandlers, final Containers containers, final ApplicationMasterRegistration registration, final ContainerRequestCounter containerRequestCounter, final DriverStatusManager driverStatusManager, final REEFFileNames reefFileNames, final TrackingURLProvider trackingURLProvider, final LocalAddressProvider addressProvider, final RackNameFormatter rackNameFormatter, final InjectionFuture<ProgressProvider> progressProvider) throws IOException { this.reefEventHandlers = reefEventHandlers; this.driverStatusManager = driverStatusManager; this.containers = containers; this.registration = registration; this.containerRequestCounter = containerRequestCounter; this.yarnConf = yarnConf; this.yarnProxyUser = yarnProxyUser; this.rackNameFormatter = rackNameFormatter; this.trackingUrl = trackingURLProvider.getTrackingUrl(); this.amRegistrationHost = addressProvider.getLocalAddress(); this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this); this.nodeManager = new NMClientAsyncImpl(this); this.jobSubmissionDirectory = jobSubmissionDirectory; this.reefFileNames = reefFileNames; this.progressProvider = progressProvider; LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.", new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory}); }
Example 4
Source File: YarnResourceManager.java From flink with Apache License 2.0 | 5 votes |
protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient( YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, @Nullable String webInterfaceUrl) throws Exception { AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync( yarnHeartbeatIntervalMillis, this); resourceManagerClient.init(yarnConfiguration); resourceManagerClient.start(); //TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address Tuple2<String, Integer> hostPort = parseHostPort(getAddress()); final int restPort; if (webInterfaceUrl != null) { final int lastColon = webInterfaceUrl.lastIndexOf(':'); if (lastColon == -1) { restPort = -1; } else { restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1)); } } else { restPort = -1; } final RegisterApplicationMasterResponse registerApplicationMasterResponse = resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl); getContainersFromPreviousAttempts(registerApplicationMasterResponse); updateMatchingStrategy(registerApplicationMasterResponse); return resourceManagerClient; }
Example 5
Source File: OlapServerMaster.java From spliceengine with GNU Affero General Public License v3.0 | 5 votes |
private AMRMClientAsync<AMRMClient.ContainerRequest> initClient(Configuration conf) throws YarnException, IOException { AMRMClientAsync.CallbackHandler allocListener = new AMRMClientAsync.CallbackHandler() { @Override public void onContainersCompleted(List<ContainerStatus> statuses) { } @Override public void onContainersAllocated(List<Container> containers) { } @Override public void onShutdownRequest() { LOG.warn("Shutting down"); end.set(true); } @Override public void onNodesUpdated(List<NodeReport> updatedNodes) { } @Override public float getProgress() { return 0; } @Override public void onError(Throwable e) { LOG.error("Unexpected error", e); end.set(true); } }; AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); rmClient.init(conf); rmClient.start(); // Register with ResourceManager rmClient.registerApplicationMaster(Utils.localHostName(), 0, ""); return rmClient; }
Example 6
Source File: TestAMRMClientAsync.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 5000) public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor() throws YarnException, IOException, InterruptedException { Configuration conf = new Configuration(); final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); List<ContainerStatus> completed = Arrays.asList( ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenReturn(response); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); callbackHandler.asynClient = asyncClient; asyncClient.init(conf); asyncClient.start(); Supplier<Boolean> checker = new Supplier<Boolean>() { @Override public Boolean get() { return callbackHandler.notify; } }; asyncClient.registerApplicationMaster("localhost", 1234, null); asyncClient.waitFor(checker); Assert.assertTrue(checker.get()); }
Example 7
Source File: TestAMRMClientAsync.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 5000) public void testCallAMRMClientAsyncStopFromCallbackHandler() throws YarnException, IOException, InterruptedException { Configuration conf = new Configuration(); TestCallbackHandler2 callbackHandler = new TestCallbackHandler2(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); List<ContainerStatus> completed = Arrays.asList( ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenReturn(response); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); callbackHandler.asynClient = asyncClient; asyncClient.init(conf); asyncClient.start(); synchronized (callbackHandler.notifier) { asyncClient.registerApplicationMaster("localhost", 1234, null); while(callbackHandler.notify == false) { try { callbackHandler.notifier.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } }
Example 8
Source File: TestAMRMClientAsync.java From big-c with Apache License 2.0 | 5 votes |
@Test (timeout = 10000) public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception { Configuration conf = new Configuration(); final TestCallbackHandler callbackHandler = new TestCallbackHandler(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); when(client.allocate(anyFloat())).thenThrow( new ApplicationAttemptNotFoundException("app not found, shut down")); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler); asyncClient.init(conf); asyncClient.start(); Supplier<Boolean> checker = new Supplier<Boolean>() { @Override public Boolean get() { return callbackHandler.reboot; } }; asyncClient.registerApplicationMaster("localhost", 1234, null); asyncClient.waitFor(checker); asyncClient.stop(); // stopping should have joined all threads and completed all callbacks Assert.assertTrue(callbackHandler.callbackCount == 0); verify(client, times(1)).allocate(anyFloat()); asyncClient.stop(); }
Example 9
Source File: YarnManager.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
@Inject public void onInit(VMConfig vmConfig) throws Exception { logger.info("Start init(VMConfig vmConfig)"); this.vmConfig = vmConfig; try { this.yarnConfig = vmConfig.getHadoopProperties(); conf = new YarnConfiguration() ; vmConfig.overrideHadoopConfiguration(conf); amrmClient = AMRMClient.createAMRMClient(); amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler()); amrmClientAsync.init(conf); amrmClientAsync.start(); nmClient = NMClient.createNMClient(); nmClient.init(conf); nmClient.start(); // Register with RM String appHostName = InetAddress.getLocalHost().getHostAddress() ; RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, ""); System.out.println("amrmClientAsync.registerApplicationMaster"); } catch(Throwable t) { logger.error("Error: " , t); t.printStackTrace(); } logger.info("Finish init(VMConfig vmConfig)"); }
Example 10
Source File: SolrMaster.java From yarn-proto with Apache License 2.0 | 5 votes |
public void run() throws Exception { int virtualCores = Integer.parseInt(cli.getOptionValue("virtualCores", "1")); AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(100, this); rmClient.init(getConfiguration()); rmClient.start(); // Register with ResourceManager rmClient.registerApplicationMaster("", 0, ""); // Priority for worker containers - priorities are intra-application Priority priority = Records.newRecord(Priority.class); priority.setPriority(0); // Resource requirements for worker containers Resource capability = Records.newRecord(Resource.class); capability.setMemory(memory); capability.setVirtualCores(virtualCores); // Make container requests to ResourceManager for (int i = 0; i < numContainersToWaitFor; ++i) rmClient.addContainerRequest(new ContainerRequest(capability, null, null, priority)); log.info("Waiting for " + numContainersToWaitFor + " containers to finish"); while (!doneWithContainers()) Thread.sleep(10000); log.info("SolrMaster application shutdown."); // Un-register with ResourceManager try { rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); } catch (Exception exc) { // safe to ignore ... this usually fails anyway } }
Example 11
Source File: AbstractApplicationMaster.java From Scribengin with GNU Affero General Public License v3.0 | 5 votes |
public boolean run() throws IOException, YarnException { // Initialize clients to RM and NMs. LOG.info("ApplicationMaster::run"); AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler(); resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener); resourceManager.init(conf); resourceManager.start(); nodeManager = NMClient.createNMClient(); nodeManager.init(conf); nodeManager.start(); // Register with RM resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); Log.info("total container count: "+Integer.toString(totalContainerCount)); // Ask RM to give us a bunch of containers //for (int i = 0; i < totalContainerCount; i++) { ContainerRequest containerReq = setupContainerReqForRM(); resourceManager.addContainerRequest(containerReq); //} requestedContainerCount.addAndGet(totalContainerCount); while (!done) { try { Thread.sleep(200); } catch (InterruptedException ex) { } }// while // Un-register with ResourceManager resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", ""); return true; }
Example 12
Source File: TestAMRMClientAsync.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 5000) public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor() throws YarnException, IOException, InterruptedException { Configuration conf = new Configuration(); final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); List<ContainerStatus> completed = Arrays.asList( ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenReturn(response); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); callbackHandler.asynClient = asyncClient; asyncClient.init(conf); asyncClient.start(); Supplier<Boolean> checker = new Supplier<Boolean>() { @Override public Boolean get() { return callbackHandler.notify; } }; asyncClient.registerApplicationMaster("localhost", 1234, null); asyncClient.waitFor(checker); Assert.assertTrue(checker.get()); }
Example 13
Source File: TestAMRMClientAsync.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 5000) public void testCallAMRMClientAsyncStopFromCallbackHandler() throws YarnException, IOException, InterruptedException { Configuration conf = new Configuration(); TestCallbackHandler2 callbackHandler = new TestCallbackHandler2(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); List<ContainerStatus> completed = Arrays.asList( ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenReturn(response); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); callbackHandler.asynClient = asyncClient; asyncClient.init(conf); asyncClient.start(); synchronized (callbackHandler.notifier) { asyncClient.registerApplicationMaster("localhost", 1234, null); while(callbackHandler.notify == false) { try { callbackHandler.notifier.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } }
Example 14
Source File: TestAMRMClientAsync.java From hadoop with Apache License 2.0 | 5 votes |
@Test (timeout = 10000) public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception { Configuration conf = new Configuration(); final TestCallbackHandler callbackHandler = new TestCallbackHandler(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); when(client.allocate(anyFloat())).thenThrow( new ApplicationAttemptNotFoundException("app not found, shut down")); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler); asyncClient.init(conf); asyncClient.start(); Supplier<Boolean> checker = new Supplier<Boolean>() { @Override public Boolean get() { return callbackHandler.reboot; } }; asyncClient.registerApplicationMaster("localhost", 1234, null); asyncClient.waitFor(checker); asyncClient.stop(); // stopping should have joined all threads and completed all callbacks Assert.assertTrue(callbackHandler.callbackCount == 0); verify(client, times(1)).allocate(anyFloat()); asyncClient.stop(); }
Example 15
Source File: TestAMRMClientAsync.java From hadoop with Apache License 2.0 | 5 votes |
private void runHeartBeatThrowOutException(Exception ex) throws Exception{ Configuration conf = new Configuration(); TestCallbackHandler callbackHandler = new TestCallbackHandler(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); when(client.allocate(anyFloat())).thenThrow(ex); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); asyncClient.init(conf); asyncClient.start(); synchronized (callbackHandler.notifier) { asyncClient.registerApplicationMaster("localhost", 1234, null); while(callbackHandler.savedException == null) { try { callbackHandler.notifier.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } Assert.assertTrue(callbackHandler.savedException.getMessage().contains( ex.getMessage())); asyncClient.stop(); // stopping should have joined all threads and completed all callbacks Assert.assertTrue(callbackHandler.callbackCount == 0); }
Example 16
Source File: ApplicationMaster.java From TensorFlowOnYARN with Apache License 2.0 | 5 votes |
private RegisterApplicationMasterResponse setupRMConnection(String hostname, int rpcPort) throws Exception { AMRMClientAsync.AbstractCallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); // Register self with ResourceManager // This will start heartbeating to the RM return amRMClient.registerApplicationMaster(hostname, rpcPort, ""); }
Example 17
Source File: YarnResourceManager.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient( YarnConfiguration yarnConfiguration, int yarnHeartbeatIntervalMillis, @Nullable String webInterfaceUrl) throws Exception { AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync( yarnHeartbeatIntervalMillis, this); resourceManagerClient.init(yarnConfiguration); resourceManagerClient.start(); //TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address Tuple2<String, Integer> hostPort = parseHostPort(getAddress()); final int restPort; if (webInterfaceUrl != null) { final int lastColon = webInterfaceUrl.lastIndexOf(':'); if (lastColon == -1) { restPort = -1; } else { restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1)); } } else { restPort = -1; } final RegisterApplicationMasterResponse registerApplicationMasterResponse = resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl); getContainersFromPreviousAttempts(registerApplicationMasterResponse); return resourceManagerClient; }
Example 18
Source File: TestAMRMClientAsync.java From big-c with Apache License 2.0 | 4 votes |
void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws InterruptedException, YarnException, IOException { Configuration conf = new Configuration(); @SuppressWarnings("unchecked") AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class); List<ContainerStatus> completed = Arrays.asList( ContainerStatus.newInstance(newContainerId(0, 0, 0, 0), ContainerState.COMPLETE, "", 0)); final AllocateResponse response = createAllocateResponse(completed, new ArrayList<Container>(), null); when(client.allocate(anyFloat())).thenReturn(response); AMRMClientAsync<ContainerRequest> asyncClient = AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler); callbackHandler.asynClient = asyncClient; callbackHandler.throwOutException = true; asyncClient.init(conf); asyncClient.start(); // call register and wait for error callback and stop synchronized (callbackHandler.notifier) { asyncClient.registerApplicationMaster("localhost", 1234, null); while(callbackHandler.notify == false) { try { callbackHandler.notifier.wait(); } catch (InterruptedException e) { e.printStackTrace(); } } } // verify error invoked verify(callbackHandler, times(0)).getProgress(); verify(callbackHandler, times(1)).onError(any(Exception.class)); // sleep to wait for a few heartbeat calls that can trigger callbacks Thread.sleep(50); // verify no more invocations after the first one. // ie. callback thread has stopped verify(callbackHandler, times(0)).getProgress(); verify(callbackHandler, times(1)).onError(any(Exception.class)); }
Example 19
Source File: ApplicationMaster.java From big-c with Apache License 2.0 | 4 votes |
@SuppressWarnings({ "unchecked" }) public void run() throws YarnException, IOException, InterruptedException { LOG.info("Starting ApplicationMaster"); // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class // are marked as LimitedPrivate Credentials credentials = UserGroupInformation.getCurrentUser().getCredentials(); DataOutputBuffer dob = new DataOutputBuffer(); credentials.writeTokenStorageToStream(dob); // Now remove the AM->RM token so that containers cannot access it. Iterator<Token<?>> iter = credentials.getAllTokens().iterator(); LOG.info("Executing with tokens:"); while (iter.hasNext()) { Token<?> token = iter.next(); LOG.info(token); if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { iter.remove(); } } allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); // Create appSubmitterUgi and add original tokens to it String appSubmitterUserName = System.getenv(ApplicationConstants.Environment.USER.name()); appSubmitterUgi = UserGroupInformation.createRemoteUser(appSubmitterUserName); appSubmitterUgi.addCredentials(credentials); AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); amRMClient.init(conf); amRMClient.start(); containerListener = createNMCallbackHandler(); nmClientAsync = new NMClientAsyncImpl(containerListener); nmClientAsync.init(conf); nmClientAsync.start(); startTimelineClient(conf); if(timelineClient != null) { publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi); } // Setup local RPC Server to accept status requests directly from clients // TODO need to setup a protocol for client to be able to communicate to // the RPC server // TODO use the rpc port info to register with the RM for the client to // send requests to this app master // Register self with ResourceManager // This will start heartbeating to the RM appMasterHostname = NetUtils.getHostname(); RegisterApplicationMasterResponse response = amRMClient .registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); // Dump out information about cluster capability as seen by the // resource manager int maxMem = response.getMaximumResourceCapability().getMemory(); LOG.info("Max mem capabililty of resources in this cluster " + maxMem); int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); // A resource ask cannot exceed the max. if (containerMemory > maxMem) { LOG.info("Container memory specified above max threshold of cluster." + " Using max value." + ", specified=" + containerMemory + ", max=" + maxMem); containerMemory = maxMem; } if (containerVirtualCores > maxVCores) { LOG.info("Container virtual cores specified above max threshold of cluster." + " Using max value." + ", specified=" + containerVirtualCores + ", max=" + maxVCores); containerVirtualCores = maxVCores; } List<Container> previousAMRunningContainers = response.getContainersFromPreviousAttempts(); LOG.info(appAttemptID + " received " + previousAMRunningContainers.size() + " previous attempts' running containers on AM registration."); numAllocatedContainers.addAndGet(previousAMRunningContainers.size()); int numTotalContainersToRequest = numTotalContainers - previousAMRunningContainers.size(); // Setup ask for containers from RM // Send request for containers to RM // Until we get our fully allocated quota, we keep on polling RM for // containers // Keep looping until all the containers are launched and shell script // executed on them ( regardless of success/failure). for (int i = 0; i < numTotalContainersToRequest; ++i) { ContainerRequest containerAsk = setupContainerAskForRM(); amRMClient.addContainerRequest(containerAsk); } numRequestedContainers.set(numTotalContainers); }
Example 20
Source File: UnmanagedAmTest.java From reef with Apache License 2.0 | 2 votes |
@Test public void testAmShutdown() throws IOException, YarnException { Assume.assumeTrue( "This test requires a YARN Resource Manager to connect to", Boolean.parseBoolean(System.getenv("REEF_TEST_YARN"))); final YarnConfiguration yarnConfig = new YarnConfiguration(); // Start YARN client and register the application final YarnClient yarnClient = YarnClient.createYarnClient(); yarnClient.init(yarnConfig); yarnClient.start(); final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class); containerContext.setCommands(Collections.<String>emptyList()); containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap()); containerContext.setEnvironment(Collections.<String, String>emptyMap()); containerContext.setTokens(getTokens()); final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext(); appContext.setApplicationName("REEF_Unmanaged_AM_Test"); appContext.setAMContainerSpec(containerContext); appContext.setUnmanagedAM(true); appContext.setQueue("default"); final ApplicationId applicationId = appContext.getApplicationId(); LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId); yarnClient.submitApplication(appContext); LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId); addToken(yarnClient.getAMRMToken(applicationId)); // Start the AM final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this); rmClient.init(yarnConfig); rmClient.start(); final NMClientAsync nmClient = new NMClientAsyncImpl(this); nmClient.init(yarnConfig); nmClient.start(); final RegisterApplicationMasterResponse registration = rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null); LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null); LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState()); // Shutdown the AM rmClient.stop(); nmClient.stop(); // Get the final application report final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId); final YarnApplicationState appState = appReport.getYarnApplicationState(); final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus(); LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] { applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus}); Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState); Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus); // Shutdown YARN client yarnClient.stop(); }