Java Code Examples for org.apache.hadoop.yarn.client.api.NMClient

The following examples show how to use org.apache.hadoop.yarn.client.api.NMClient. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: yarn-proto   Source File: SolrMaster.java    License: Apache License 2.0 6 votes vote down vote up
public SolrMaster(CommandLine cli) throws Exception {
  this.cli = cli;
  Configuration hadoopConf = new Configuration();
  if (cli.hasOption("conf")) {
    hadoopConf.addResource(new Path(cli.getOptionValue("conf")));
    hadoopConf.reloadConfiguration();
  }
  conf = new YarnConfiguration(hadoopConf);

  nmClient = NMClient.createNMClient();
  nmClient.init(conf);
  nmClient.start();
  numContainersToWaitFor = Integer.parseInt(cli.getOptionValue("nodes"));
  memory = Integer.parseInt(cli.getOptionValue("memory", "512"));
  port = Integer.parseInt(cli.getOptionValue("port"));
  nextPort = port;

  SecureRandom random = new SecureRandom();
  this.randomStopKey = new BigInteger(130, random).toString(32);

  this.inetAddresses = getMyInetAddresses();
}
 
Example 2
Source Project: Flink-CEPplus   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
 
Example 3
Source Project: Flink-CEPplus   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
 
Example 4
Source Project: flink   Source File: YarnResourceManager.java    License: Apache License 2.0 5 votes vote down vote up
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClient nodeManagerClient = NMClient.createNMClient();
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	nodeManagerClient.cleanupRunningContainersOnStop(true);
	return nodeManagerClient;
}
 
Example 5
Source Project: flink   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
TestingYarnResourceManager(
		RpcService rpcService,
		String resourceManagerEndpointId,
		ResourceID resourceId,
		Configuration flinkConfig,
		Map<String, String> env,
		HighAvailabilityServices highAvailabilityServices,
		HeartbeatServices heartbeatServices,
		SlotManager slotManager,
		MetricRegistry metricRegistry,
		JobLeaderIdService jobLeaderIdService,
		ClusterInformation clusterInformation,
		FatalErrorHandler fatalErrorHandler,
		@Nullable String webInterfaceUrl,
		AMRMClientAsync<AMRMClient.ContainerRequest> mockResourceManagerClient,
		NMClient mockNMClient,
		JobManagerMetricGroup jobManagerMetricGroup) {
	super(
		rpcService,
		resourceManagerEndpointId,
		resourceId,
		flinkConfig,
		env,
		highAvailabilityServices,
		heartbeatServices,
		slotManager,
		metricRegistry,
		jobLeaderIdService,
		clusterInformation,
		fatalErrorHandler,
		webInterfaceUrl,
		jobManagerMetricGroup);
	this.mockNMClient = mockNMClient;
	this.mockResourceManagerClient = mockResourceManagerClient;
}
 
Example 6
Source Project: hadoop   Source File: NMClientAsyncImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
 
Example 7
Source Project: hadoop   Source File: NMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
 
Example 8
Source Project: hadoop   Source File: TestNMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
 
Example 9
Source Project: big-c   Source File: NMClientAsyncImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected NMClientAsyncImpl(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name, client, callbackHandler);
  this.client = client;
  this.callbackHandler = callbackHandler;
}
 
Example 10
Source Project: big-c   Source File: NMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
@Private
@VisibleForTesting
protected NMClientAsync(String name, NMClient client,
    CallbackHandler callbackHandler) {
  super(name);
  this.setClient(client);
  this.setCallbackHandler(callbackHandler);
}
 
Example 11
Source Project: big-c   Source File: TestNMClientAsync.java    License: Apache License 2.0 5 votes vote down vote up
private NMClient mockNMClient(int mode)
    throws YarnException, IOException {
  NMClient client = mock(NMClient.class);
  switch (mode) {
    case 0:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doNothing().when(client).stopContainer(any(ContainerId.class),
          any(NodeId.class));
      break;
    case 1:
      doThrow(RPCUtil.getRemoteException("Start Exception")).when(client)
          .startContainer(any(Container.class),
              any(ContainerLaunchContext.class));
      doThrow(RPCUtil.getRemoteException("Query Exception")).when(client)
          .getContainerStatus(any(ContainerId.class), any(NodeId.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
      break;
    case 2:
      when(client.startContainer(any(Container.class),
          any(ContainerLaunchContext.class))).thenReturn(
              Collections.<String, ByteBuffer>emptyMap());
      when(client.getContainerStatus(any(ContainerId.class),
          any(NodeId.class))).thenReturn(
              recordFactory.newRecordInstance(ContainerStatus.class));
      doThrow(RPCUtil.getRemoteException("Stop Exception")).when(client)
          .stopContainer(any(ContainerId.class), any(NodeId.class));
  }
  return client;
}
 
Example 12
public boolean run() throws IOException, YarnException {
  // Initialize clients to RM and NMs.
  LOG.info("ApplicationMaster::run");
  AMRMClientAsync.CallbackHandler rmListener = new RMCallbackHandler();
  resourceManager = AMRMClientAsync.createAMRMClientAsync(1000, rmListener);
  resourceManager.init(conf);
  resourceManager.start();
  
  nodeManager = NMClient.createNMClient();
  nodeManager.init(conf);
  nodeManager.start();
  
  // Register with RM
  resourceManager.registerApplicationMaster(appMasterHostname, appMasterRpcPort,
      appMasterTrackingUrl);

  Log.info("total container count: "+Integer.toString(totalContainerCount));
  
  // Ask RM to give us a bunch of containers
  //for (int i = 0; i < totalContainerCount; i++) {
    ContainerRequest containerReq = setupContainerReqForRM();
    resourceManager.addContainerRequest(containerReq);
  //}
  requestedContainerCount.addAndGet(totalContainerCount);

  while (!done) {
    try {
      Thread.sleep(200);
    } catch (InterruptedException ex) {
    }
  }// while

  // Un-register with ResourceManager
  resourceManager.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "", "");
  return true;
}
 
Example 13
@Inject
public void onInit(VMConfig vmConfig) throws Exception {
  logger.info("Start init(VMConfig vmConfig)");
  this.vmConfig = vmConfig;
  try {
    this.yarnConfig = vmConfig.getHadoopProperties();
    conf = new YarnConfiguration() ;
    vmConfig.overrideHadoopConfiguration(conf);
    
    amrmClient = AMRMClient.createAMRMClient();
    amrmClientAsync = AMRMClientAsync.createAMRMClientAsync(amrmClient, 1000, new AMRMCallbackHandler());
    amrmClientAsync.init(conf);
    amrmClientAsync.start();
    
    nmClient = NMClient.createNMClient();
    nmClient.init(conf);
    nmClient.start();
    // Register with RM
    String appHostName = InetAddress.getLocalHost().getHostAddress()  ;

    RegisterApplicationMasterResponse registerResponse = amrmClientAsync.registerApplicationMaster(appHostName, 0, "");
    System.out.println("amrmClientAsync.registerApplicationMaster");
  } catch(Throwable t) {
    logger.error("Error: " , t);
    t.printStackTrace();
  }
  logger.info("Finish init(VMConfig vmConfig)");
}
 
Example 14
Source Project: Flink-CEPplus   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return mockNMClient;
}
 
Example 15
Source Project: flink   Source File: YarnResourceManagerTest.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected NMClient createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return mockNMClient;
}
 
Example 16
Source Project: hadoop   Source File: NMClientAsync.java    License: Apache License 2.0 4 votes vote down vote up
public NMClient getClient() {
  return client;
}
 
Example 17
Source Project: hadoop   Source File: NMClientAsync.java    License: Apache License 2.0 4 votes vote down vote up
public void setClient(NMClient client) {
  this.client = client;
}
 
Example 18
Source Project: hadoop   Source File: TestNMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example 19
Source Project: twill   Source File: Hadoop21YarnNMClient.java    License: Apache License 2.0 4 votes vote down vote up
public Hadoop21YarnNMClient(Configuration configuration) {
  this.nmClient = NMClient.createNMClient();
  nmClient.init(configuration);
}
 
Example 20
Source Project: twill   Source File: Hadoop21YarnNMClient.java    License: Apache License 2.0 4 votes vote down vote up
private ContainerTerminator(Container container, NMClient nmClient) {
  this.container = container;
  this.nmClient = nmClient;
}
 
Example 21
Source Project: big-c   Source File: NMClientAsync.java    License: Apache License 2.0 4 votes vote down vote up
public NMClient getClient() {
  return client;
}
 
Example 22
Source Project: big-c   Source File: NMClientAsync.java    License: Apache License 2.0 4 votes vote down vote up
public void setClient(NMClient client) {
  this.client = client;
}
 
Example 23
Source Project: big-c   Source File: TestNMClient.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws YarnException, IOException {
  // start minicluster
  conf = new YarnConfiguration();
  yarnCluster =
      new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
  yarnCluster.init(conf);
  yarnCluster.start();
  assertNotNull(yarnCluster);
  assertEquals(STATE.STARTED, yarnCluster.getServiceState());

  // start rm client
  yarnClient = (YarnClientImpl) YarnClient.createYarnClient();
  yarnClient.init(conf);
  yarnClient.start();
  assertNotNull(yarnClient);
  assertEquals(STATE.STARTED, yarnClient.getServiceState());

  // get node info
  nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);

  // submit new app
  ApplicationSubmissionContext appContext = 
      yarnClient.createApplication().getApplicationSubmissionContext();
  ApplicationId appId = appContext.getApplicationId();
  // set the application name
  appContext.setApplicationName("Test");
  // Set the priority for the application master
  Priority pri = Priority.newInstance(0);
  appContext.setPriority(pri);
  // Set the queue to which this application is to be submitted in the RM
  appContext.setQueue("default");
  // Set up the container launch context for the application master
  ContainerLaunchContext amContainer = Records
      .newRecord(ContainerLaunchContext.class);
  appContext.setAMContainerSpec(amContainer);
  // unmanaged AM
  appContext.setUnmanagedAM(true);
  // Create the request to send to the applications manager
  SubmitApplicationRequest appRequest = Records
      .newRecord(SubmitApplicationRequest.class);
  appRequest.setApplicationSubmissionContext(appContext);
  // Submit the application to the applications manager
  yarnClient.submitApplication(appContext);

  // wait for app to start
  int iterationsLeft = 30;
  RMAppAttempt appAttempt = null;
  while (iterationsLeft > 0) {
    ApplicationReport appReport = yarnClient.getApplicationReport(appId);
    if (appReport.getYarnApplicationState() ==
        YarnApplicationState.ACCEPTED) {
      attemptId = appReport.getCurrentApplicationAttemptId();
      appAttempt =
          yarnCluster.getResourceManager().getRMContext().getRMApps()
            .get(attemptId.getApplicationId()).getCurrentAppAttempt();
      while (true) {
        if (appAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
          break;
        }
      }
      break;
    }
    sleep(1000);
    --iterationsLeft;
  }
  if (iterationsLeft == 0) {
    fail("Application hasn't bee started");
  }

  // Just dig into the ResourceManager and get the AMRMToken just for the sake
  // of testing.
  UserGroupInformation.setLoginUser(UserGroupInformation
    .createRemoteUser(UserGroupInformation.getCurrentUser().getUserName()));
  UserGroupInformation.getCurrentUser().addToken(appAttempt.getAMRMToken());

  //creating an instance NMTokenCase
  nmTokenCache = new NMTokenCache();
  
  // start am rm client
  rmClient =
      (AMRMClientImpl<ContainerRequest>) AMRMClient
        .<ContainerRequest> createAMRMClient();

  //setting an instance NMTokenCase
  rmClient.setNMTokenCache(nmTokenCache);
  rmClient.init(conf);
  rmClient.start();
  assertNotNull(rmClient);
  assertEquals(STATE.STARTED, rmClient.getServiceState());

  // start am nm client
  nmClient = (NMClientImpl) NMClient.createNMClient();
  
  //propagating the AMRMClient NMTokenCache instance
  nmClient.setNMTokenCache(rmClient.getNMTokenCache());
  nmClient.init(conf);
  nmClient.start();
  assertNotNull(nmClient);
  assertEquals(STATE.STARTED, nmClient.getServiceState());
}
 
Example 24
Source Project: ignite   Source File: ApplicationMaster.java    License: Apache License 2.0 2 votes vote down vote up
/**
 * Sets NMClient.
 *
 * @param nmClient NMClient.
 */
public void setNmClient(NMClient nmClient) {
    this.nmClient = nmClient;
}
 
Example 25
public NMClient getNMClient() { return this.nmClient ; }