Java Code Examples for org.apache.hadoop.yarn.client.api.async.AMRMClientAsync#createAMRMClientAsync()

@Test (timeout = 10000)
public void testAMRMClientAsyncShutDown() throws Exception {
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  createAllocateResponse(new ArrayList<ContainerStatus>(),
    new ArrayList<Container>(), null);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  asyncClient.registerApplicationMaster("localhost", 1234, null);

  Thread.sleep(50);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient(
		YarnConfiguration yarnConfiguration,
		int yarnHeartbeatIntervalMillis,
		@Nullable String webInterfaceUrl) throws Exception {
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync(
		yarnHeartbeatIntervalMillis,
		this);

	resourceManagerClient.init(yarnConfiguration);
	resourceManagerClient.start();

	//TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address
	Tuple2<String, Integer> hostPort = parseHostPort(getAddress());

	final int restPort;

	if (webInterfaceUrl != null) {
		final int lastColon = webInterfaceUrl.lastIndexOf(':');

		if (lastColon == -1) {
			restPort = -1;
		} else {
			restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1));
		}
	} else {
		restPort = -1;
	}

	final RegisterApplicationMasterResponse registerApplicationMasterResponse =
		resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl);
	getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	return resourceManagerClient;
}
 

@Inject
private YarnContainerManager(
    @Parameter(YarnHeartbeatPeriod.class) final int yarnRMHeartbeatPeriod,
    @Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
    final YarnConfiguration yarnConf,
    final YarnProxyUser yarnProxyUser,
    final REEFEventHandlers reefEventHandlers,
    final Containers containers,
    final ApplicationMasterRegistration registration,
    final ContainerRequestCounter containerRequestCounter,
    final DriverStatusManager driverStatusManager,
    final REEFFileNames reefFileNames,
    final TrackingURLProvider trackingURLProvider,
    final LocalAddressProvider addressProvider,
    final RackNameFormatter rackNameFormatter,
    final InjectionFuture<ProgressProvider> progressProvider) throws IOException {

  this.reefEventHandlers = reefEventHandlers;
  this.driverStatusManager = driverStatusManager;

  this.containers = containers;
  this.registration = registration;
  this.containerRequestCounter = containerRequestCounter;
  this.yarnConf = yarnConf;
  this.yarnProxyUser = yarnProxyUser;
  this.rackNameFormatter = rackNameFormatter;

  this.trackingUrl = trackingURLProvider.getTrackingUrl();
  this.amRegistrationHost = addressProvider.getLocalAddress();

  this.resourceManager = AMRMClientAsync.createAMRMClientAsync(yarnRMHeartbeatPeriod, this);
  this.nodeManager = new NMClientAsyncImpl(this);

  this.jobSubmissionDirectory = jobSubmissionDirectory;
  this.reefFileNames = reefFileNames;
  this.progressProvider = progressProvider;

  LOG.log(Level.INFO, "Instantiated YarnContainerManager: {0} {1}, trackingUrl: {3}, jobSubmissionDirectory: {4}.",
      new Object[] {this.registration, this.yarnProxyUser, this.trackingUrl, this.jobSubmissionDirectory});
}
 

protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient(
		YarnConfiguration yarnConfiguration,
		int yarnHeartbeatIntervalMillis,
		@Nullable String webInterfaceUrl) throws Exception {
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync(
		yarnHeartbeatIntervalMillis,
		this);

	resourceManagerClient.init(yarnConfiguration);
	resourceManagerClient.start();

	//TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address
	Tuple2<String, Integer> hostPort = parseHostPort(getAddress());

	final int restPort;

	if (webInterfaceUrl != null) {
		final int lastColon = webInterfaceUrl.lastIndexOf(':');

		if (lastColon == -1) {
			restPort = -1;
		} else {
			restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1));
		}
	} else {
		restPort = -1;
	}

	final RegisterApplicationMasterResponse registerApplicationMasterResponse =
		resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl);
	getContainersFromPreviousAttempts(registerApplicationMasterResponse);
	updateMatchingStrategy(registerApplicationMasterResponse);

	return resourceManagerClient;
}
 

private AMRMClientAsync<AMRMClient.ContainerRequest> initClient(Configuration conf) throws YarnException, IOException {
    AMRMClientAsync.CallbackHandler allocListener = new AMRMClientAsync.CallbackHandler() {
        @Override
        public void onContainersCompleted(List<ContainerStatus> statuses) {
        }

        @Override
        public void onContainersAllocated(List<Container> containers) {
        }

        @Override
        public void onShutdownRequest() {
            LOG.warn("Shutting down");
            end.set(true);
        }

        @Override
        public void onNodesUpdated(List<NodeReport> updatedNodes) {
        }

        @Override
        public float getProgress() {
            return 0;
        }

        @Override
        public void onError(Throwable e) {
            LOG.error("Unexpected error", e);
            end.set(true);
        }
    };
    AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
    rmClient.init(conf);
    rmClient.start();

    // Register with ResourceManager
    rmClient.registerApplicationMaster(Utils.localHostName(), 0, "");

    return rmClient;
}
 

@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 

@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 

@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 

public void run() throws Exception {
  int virtualCores = Integer.parseInt(cli.getOptionValue("virtualCores", "1"));

  AMRMClientAsync<ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(100, this);
  rmClient.init(getConfiguration());
  rmClient.start();

  // Register with ResourceManager
  rmClient.registerApplicationMaster("", 0, "");

  // Priority for worker containers - priorities are intra-application
  Priority priority = Records.newRecord(Priority.class);
  priority.setPriority(0);

  // Resource requirements for worker containers
  Resource capability = Records.newRecord(Resource.class);
  capability.setMemory(memory);
  capability.setVirtualCores(virtualCores);

  // Make container requests to ResourceManager
  for (int i = 0; i < numContainersToWaitFor; ++i)
    rmClient.addContainerRequest(new ContainerRequest(capability, null, null, priority));

  log.info("Waiting for " + numContainersToWaitFor + " containers to finish");
  while (!doneWithContainers())
    Thread.sleep(10000);

  log.info("SolrMaster application shutdown.");

  // Un-register with ResourceManager
  try {
    rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  } catch (Exception exc) {
    // safe to ignore ... this usually fails anyway
  }
}
 

public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 

@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandlerWithWaitFor()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  final TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.notify;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);
  Assert.assertTrue(checker.get());
}
 

@Test (timeout = 5000)
public void testCallAMRMClientAsyncStopFromCallbackHandler()
    throws YarnException, IOException, InterruptedException {
  Configuration conf = new Configuration();
  TestCallbackHandler2 callbackHandler = new TestCallbackHandler2();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  asyncClient.init(conf);
  asyncClient.start();

  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
}
 

@Test (timeout = 10000)
public void testAMRMClientAsyncShutDownWithWaitFor() throws Exception {
  Configuration conf = new Configuration();
  final TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(
    new ApplicationAttemptNotFoundException("app not found, shut down"));

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 10, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();

  Supplier<Boolean> checker = new Supplier<Boolean>() {
    @Override
    public Boolean get() {
      return callbackHandler.reboot;
    }
  };

  asyncClient.registerApplicationMaster("localhost", 1234, null);
  asyncClient.waitFor(checker);

  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);

  verify(client, times(1)).allocate(anyFloat());
  asyncClient.stop();
}
 

private void runHeartBeatThrowOutException(Exception ex) throws Exception{
  Configuration conf = new Configuration();
  TestCallbackHandler callbackHandler = new TestCallbackHandler();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);
  when(client.allocate(anyFloat())).thenThrow(ex);

  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  asyncClient.init(conf);
  asyncClient.start();
  
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.savedException == null) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  Assert.assertTrue(callbackHandler.savedException.getMessage().contains(
      ex.getMessage()));
  
  asyncClient.stop();
  // stopping should have joined all threads and completed all callbacks
  Assert.assertTrue(callbackHandler.callbackCount == 0);
}
 

private RegisterApplicationMasterResponse setupRMConnection(String hostname, int rpcPort) throws Exception {
  AMRMClientAsync.AbstractCallbackHandler allocListener =
      new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();
  // Register self with ResourceManager
  // This will start heartbeating to the RM
  return amRMClient.registerApplicationMaster(hostname, rpcPort, "");
}
 

protected AMRMClientAsync<AMRMClient.ContainerRequest> createAndStartResourceManagerClient(
		YarnConfiguration yarnConfiguration,
		int yarnHeartbeatIntervalMillis,
		@Nullable String webInterfaceUrl) throws Exception {
	AMRMClientAsync<AMRMClient.ContainerRequest> resourceManagerClient = AMRMClientAsync.createAMRMClientAsync(
		yarnHeartbeatIntervalMillis,
		this);

	resourceManagerClient.init(yarnConfiguration);
	resourceManagerClient.start();

	//TODO: change akka address to tcp host and port, the getAddress() interface should return a standard tcp address
	Tuple2<String, Integer> hostPort = parseHostPort(getAddress());

	final int restPort;

	if (webInterfaceUrl != null) {
		final int lastColon = webInterfaceUrl.lastIndexOf(':');

		if (lastColon == -1) {
			restPort = -1;
		} else {
			restPort = Integer.valueOf(webInterfaceUrl.substring(lastColon + 1));
		}
	} else {
		restPort = -1;
	}

	final RegisterApplicationMasterResponse registerApplicationMasterResponse =
		resourceManagerClient.registerApplicationMaster(hostPort.f0, restPort, webInterfaceUrl);
	getContainersFromPreviousAttempts(registerApplicationMasterResponse);

	return resourceManagerClient;
}
 

void runCallBackThrowOutException(TestCallbackHandler2 callbackHandler) throws
      InterruptedException, YarnException, IOException {
  Configuration conf = new Configuration();
  @SuppressWarnings("unchecked")
  AMRMClient<ContainerRequest> client = mock(AMRMClientImpl.class);

  List<ContainerStatus> completed = Arrays.asList(
      ContainerStatus.newInstance(newContainerId(0, 0, 0, 0),
          ContainerState.COMPLETE, "", 0));
  final AllocateResponse response = createAllocateResponse(completed,
      new ArrayList<Container>(), null);

  when(client.allocate(anyFloat())).thenReturn(response);
  AMRMClientAsync<ContainerRequest> asyncClient =
      AMRMClientAsync.createAMRMClientAsync(client, 20, callbackHandler);
  callbackHandler.asynClient = asyncClient;
  callbackHandler.throwOutException = true;
  asyncClient.init(conf);
  asyncClient.start();

  // call register and wait for error callback and stop
  synchronized (callbackHandler.notifier) {
    asyncClient.registerApplicationMaster("localhost", 1234, null);
    while(callbackHandler.notify == false) {
      try {
        callbackHandler.notifier.wait();
      } catch (InterruptedException e) {
        e.printStackTrace();
      }
    }
  }
  // verify error invoked
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
  // sleep to wait for a few heartbeat calls that can trigger callbacks
  Thread.sleep(50);
  // verify no more invocations after the first one.
  // ie. callback thread has stopped
  verify(callbackHandler, times(0)).getProgress();
  verify(callbackHandler, times(1)).onError(any(Exception.class));
}
 

@SuppressWarnings({ "unchecked" })
public void run() throws YarnException, IOException, InterruptedException {
  LOG.info("Starting ApplicationMaster");

  // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
  // are marked as LimitedPrivate
  Credentials credentials =
      UserGroupInformation.getCurrentUser().getCredentials();
  DataOutputBuffer dob = new DataOutputBuffer();
  credentials.writeTokenStorageToStream(dob);
  // Now remove the AM->RM token so that containers cannot access it.
  Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
  LOG.info("Executing with tokens:");
  while (iter.hasNext()) {
    Token<?> token = iter.next();
    LOG.info(token);
    if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
      iter.remove();
    }
  }
  allTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());

  // Create appSubmitterUgi and add original tokens to it
  String appSubmitterUserName =
      System.getenv(ApplicationConstants.Environment.USER.name());
  appSubmitterUgi =
      UserGroupInformation.createRemoteUser(appSubmitterUserName);
  appSubmitterUgi.addCredentials(credentials);


  AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
  amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
  amRMClient.init(conf);
  amRMClient.start();

  containerListener = createNMCallbackHandler();
  nmClientAsync = new NMClientAsyncImpl(containerListener);
  nmClientAsync.init(conf);
  nmClientAsync.start();

  startTimelineClient(conf);
  if(timelineClient != null) {
    publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
        DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
  }

  // Setup local RPC Server to accept status requests directly from clients
  // TODO need to setup a protocol for client to be able to communicate to
  // the RPC server
  // TODO use the rpc port info to register with the RM for the client to
  // send requests to this app master

  // Register self with ResourceManager
  // This will start heartbeating to the RM
  appMasterHostname = NetUtils.getHostname();
  RegisterApplicationMasterResponse response = amRMClient
      .registerApplicationMaster(appMasterHostname, appMasterRpcPort,
          appMasterTrackingUrl);
  // Dump out information about cluster capability as seen by the
  // resource manager
  int maxMem = response.getMaximumResourceCapability().getMemory();
  LOG.info("Max mem capabililty of resources in this cluster " + maxMem);
  
  int maxVCores = response.getMaximumResourceCapability().getVirtualCores();
  LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores);

  // A resource ask cannot exceed the max.
  if (containerMemory > maxMem) {
    LOG.info("Container memory specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerMemory + ", max="
        + maxMem);
    containerMemory = maxMem;
  }

  if (containerVirtualCores > maxVCores) {
    LOG.info("Container virtual cores specified above max threshold of cluster."
        + " Using max value." + ", specified=" + containerVirtualCores + ", max="
        + maxVCores);
    containerVirtualCores = maxVCores;
  }

  List<Container> previousAMRunningContainers =
      response.getContainersFromPreviousAttempts();
  LOG.info(appAttemptID + " received " + previousAMRunningContainers.size()
    + " previous attempts' running containers on AM registration.");
  numAllocatedContainers.addAndGet(previousAMRunningContainers.size());

  int numTotalContainersToRequest =
      numTotalContainers - previousAMRunningContainers.size();
  // Setup ask for containers from RM
  // Send request for containers to RM
  // Until we get our fully allocated quota, we keep on polling RM for
  // containers
  // Keep looping until all the containers are launched and shell script
  // executed on them ( regardless of success/failure).
  for (int i = 0; i < numTotalContainersToRequest; ++i) {
    ContainerRequest containerAsk = setupContainerAskForRM();
    amRMClient.addContainerRequest(containerAsk);
  }
  numRequestedContainers.set(numTotalContainers);
}
 

@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}