org.apache.hadoop.yarn.client.api.async.NMClientAsync Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.async.NMClientAsync. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestYarnClusterResourceManager.java    From samza with Apache License 2.0 6 votes vote down vote up
@Test
public void testAllocatedResourceExpiryForYarn() {
  YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class);
  SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class);
  Config config = mock(Config.class);
  AMRMClientAsync asyncClient = mock(AMRMClientAsync.class);
  YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081);
  SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class);
  SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class);
  NMClientAsync asyncNMClient = mock(NMClientAsync.class);
  ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);

  // start the cluster manager
  YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient,
      callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);

  SamzaResource allocatedResource = mock(SamzaResource.class);
  when(allocatedResource.getTimestamp()).thenReturn(System.currentTimeMillis() - Duration.ofMinutes(10).toMillis());

  Assert.assertTrue(yarnClusterResourceManager.isResourceExpired(allocatedResource));
}
 
Example #2
Source File: LaunchContainer.java    From metron with Apache License 2.0 6 votes vote down vote up
public LaunchContainer( Configuration conf
        , String zkQuorum
        , String zkRoot
        , NMClientAsync nmClientAsync
        , ModelRequest request
        , Container container
        , ByteBuffer allTokens
        , Path appJarLocation
)
{
  this.allTokens = allTokens;
  this.zkQuorum = zkQuorum;
  this.zkRoot = zkRoot;
  this.request = request;
  this.container = container;
  this.conf = conf;
  this.nmClientAsync = nmClientAsync;
  this.appJarLocation = appJarLocation;
}
 
Example #3
Source File: StreamingAppMasterService.java    From Bats with Apache License 2.0 5 votes vote down vote up
public void stop(NMClientAsync nmClient)
{
  if (stop == Long.MAX_VALUE) {
    nmClient.stopContainerAsync(container.getId(), container.getNodeId());
    LOG.info("Requested stop container {}", container.getId().toString());
    stop = System.currentTimeMillis();
  }
}
 
Example #4
Source File: NMClientAsyncImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public StatefulContainer(NMClientAsync client, ContainerId containerId) {
  this.nmClientAsync = client;
  this.containerId = containerId;
  stateMachine = stateMachineFactory.make(this);
  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  readLock = lock.readLock();
  writeLock = lock.writeLock();
}
 
Example #5
Source File: NMClientAsyncImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
public StatefulContainer(NMClientAsync client, ContainerId containerId) {
  this.nmClientAsync = client;
  this.containerId = containerId;
  stateMachine = stateMachineFactory.make(this);
  ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
  readLock = lock.readLock();
  writeLock = lock.writeLock();
}
 
Example #6
Source File: LaunchContainerRunnable.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
/**
 * @param lcontainer Allocated container
 * @param nmClient
 * @param sca
 * @param tokens
 */
public LaunchContainerRunnable(Container lcontainer, NMClientAsync nmClient, StreamingContainerAgent sca, ByteBuffer tokens)
{
  this.container = lcontainer;
  this.nmClient = nmClient;
  this.dag = sca.getContainer().getPlan().getLogicalPlan();
  this.tokens = tokens;
  this.sca = sca;
}
 
Example #7
Source File: StreamingAppMasterService.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
public void stop(NMClientAsync nmClient)
{
  if (stop == Long.MAX_VALUE) {
    nmClient.stopContainerAsync(container.getId(), container.getNodeId());
    LOG.info("Requested stop container {}", container.getId().toString());
    stop = System.currentTimeMillis();
  }
}
 
Example #8
Source File: YarnClusterResourceManager.java    From samza with Apache License 2.0 5 votes vote down vote up
YarnClusterResourceManager(AMRMClientAsync amClientAsync, NMClientAsync nmClientAsync, Callback callback,
    YarnAppState yarnAppState, SamzaYarnAppMasterLifecycle lifecycle, SamzaYarnAppMasterService service,
    SamzaAppMasterMetrics metrics, YarnConfiguration yarnConfiguration, Config config) {
  super(callback);
  this.yarnConfiguration  = yarnConfiguration;
  this.metrics = metrics;
  this.yarnConfig = new YarnConfig(config);
  this.config = config;
  this.amClient = amClientAsync;
  this.state = yarnAppState;
  this.lifecycle = lifecycle;
  this.service = service;
  this.nmClientAsync = nmClientAsync;
}
 
Example #9
Source File: LaunchContainerRunnable.java    From Bats with Apache License 2.0 5 votes vote down vote up
/**
 * @param lcontainer Allocated container
 * @param nmClient
 * @param sca
 * @param tokens
 */
public LaunchContainerRunnable(Container lcontainer, NMClientAsync nmClient, StreamingContainerAgent sca, ByteBuffer tokens)
{
  this.container = lcontainer;
  this.nmClient = nmClient;
  this.dag = sca.getContainer().getPlan().getLogicalPlan();
  this.tokens = tokens;
  this.sca = sca;
}
 
Example #10
Source File: TestYarnClusterResourceManager.java    From samza with Apache License 2.0 5 votes vote down vote up
@Test
public void testErrorInStartContainerShouldUpdateState() {
  // create mocks
  final int samzaContainerId = 1;
  YarnConfiguration yarnConfiguration = mock(YarnConfiguration.class);
  SamzaAppMasterMetrics metrics = mock(SamzaAppMasterMetrics.class);
  Config config = mock(Config.class);
  AMRMClientAsync asyncClient = mock(AMRMClientAsync.class);
  YarnAppState yarnAppState = new YarnAppState(0, mock(ContainerId.class), "host", 8080, 8081);
  SamzaYarnAppMasterLifecycle lifecycle = mock(SamzaYarnAppMasterLifecycle.class);
  SamzaYarnAppMasterService service = mock(SamzaYarnAppMasterService.class);
  NMClientAsync asyncNMClient = mock(NMClientAsync.class);
  ClusterResourceManager.Callback callback = mock(ClusterResourceManager.Callback.class);

  // start the cluster manager
  YarnClusterResourceManager yarnClusterResourceManager = new YarnClusterResourceManager(asyncClient, asyncNMClient,
      callback, yarnAppState, lifecycle, service, metrics, yarnConfiguration, config);

  yarnAppState.pendingProcessors.put(String.valueOf(samzaContainerId),
      new YarnContainer(Container.newInstance(
          ContainerId.newContainerId(
              ApplicationAttemptId.newInstance(
                  ApplicationId.newInstance(10000L, 1), 1), 1),
          NodeId.newInstance("host1", 8088), "http://host1",
          Resource.newInstance(1024, 1), Priority.newInstance(1),
          Token.newInstance("id".getBytes(), "read", "password".getBytes(), "service"))));

  yarnClusterResourceManager.start();
  assertEquals(1, yarnAppState.pendingProcessors.size());

  yarnClusterResourceManager.onStartContainerError(ContainerId.newContainerId(
      ApplicationAttemptId.newInstance(
          ApplicationId.newInstance(10000L, 1), 1), 1),
      new Exception());

  assertEquals(0, yarnAppState.pendingProcessors.size());
  verify(callback, times(1)).onStreamProcessorLaunchFailure(anyObject(), any(Exception.class));
}
 
Example #11
Source File: ContainerRequestListener.java    From metron with Apache License 2.0 5 votes vote down vote up
public void initialize(AMRMClientAsync<AMRMClient.ContainerRequest> amRMClient
                      , NMClientAsync nmClient
                      , ServiceDiscoverer serviceDiscoverer
                      )
{
  this.nmClient = nmClient;
  this.amRMClient = amRMClient;
  this.serviceDiscoverer = serviceDiscoverer;
}
 
Example #12
Source File: YarnResourceManager.java    From flink with Apache License 2.0 5 votes vote down vote up
protected NMClientAsync createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	// create the client to communicate with the node managers
	NMClientAsync nodeManagerClient = NMClientAsync.createNMClientAsync(this);
	nodeManagerClient.init(yarnConfiguration);
	nodeManagerClient.start();
	return nodeManagerClient;
}
 
Example #13
Source File: YarnClusterResourceManager.java    From samza with Apache License 2.0 4 votes vote down vote up
/**
 * Creates an YarnClusterResourceManager from config, a jobModelReader and a callback.
 * @param config to instantiate the cluster manager with
 * @param jobModelManager the jobModel manager to get the job model (mostly for the UI)
 * @param callback the callback to receive events from Yarn.
 * @param samzaAppState samza app state for display in the UI
 */
public YarnClusterResourceManager(Config config, JobModelManager jobModelManager,
    ClusterResourceManager.Callback callback, SamzaApplicationState samzaAppState) {
  super(callback);
  yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  // Use the Samza job config "fs.<scheme>.impl" and "fs.<scheme>.impl.*" for YarnConfiguration
  FileSystemImplConfig fsImplConfig = new FileSystemImplConfig(config);
  fsImplConfig.getSchemes().forEach(
    scheme -> {
      fsImplConfig.getSchemeConfig(scheme).forEach(
        (confKey, confValue) -> yarnConfiguration.set(confKey, confValue)
      );
    }
  );

  MetricsRegistryMap registry = new MetricsRegistryMap();
  metrics = new SamzaAppMasterMetrics(config, samzaAppState, registry);

  // parse configs from the Yarn environment
  String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.toString());
  ContainerId containerId = ConverterUtils.toContainerId(containerIdStr);
  String nodeHostString = System.getenv(ApplicationConstants.Environment.NM_HOST.toString());
  String nodePortString = System.getenv(ApplicationConstants.Environment.NM_PORT.toString());
  String nodeHttpPortString = System.getenv(ApplicationConstants.Environment.NM_HTTP_PORT.toString());

  int nodePort = Integer.parseInt(nodePortString);
  int nodeHttpPort = Integer.parseInt(nodeHttpPortString);
  YarnConfig yarnConfig = new YarnConfig(config);
  this.yarnConfig = yarnConfig;
  this.config = config;
  int interval = yarnConfig.getAMPollIntervalMs();

  //Instantiate the AM Client.
  this.amClient = AMRMClientAsync.createAMRMClientAsync(interval, this);

  this.state = new YarnAppState(-1, containerId, nodeHostString, nodePort, nodeHttpPort);

  log.info("Initialized YarnAppState: {}", state.toString());
  this.service = new SamzaYarnAppMasterService(config, samzaAppState, this.state, registry, yarnConfiguration);

  log.info("Container ID: {}, Nodehost:  {} , Nodeport : {} , NodeHttpport: {}", containerIdStr, nodeHostString, nodePort, nodeHttpPort);
  ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config);
  this.lifecycle = new SamzaYarnAppMasterLifecycle(
      clusterManagerConfig.getContainerMemoryMb(),
      clusterManagerConfig.getNumCores(),
      samzaAppState,
      state,
      amClient
  );
  this.nmClientAsync = NMClientAsync.createNMClientAsync(this);

}
 
Example #14
Source File: WorkflowDriver.java    From Hi-WAY with Apache License 2.0 4 votes vote down vote up
public NMClientAsync getNmClientAsync() {
	return nmClientAsync;
}
 
Example #15
Source File: YarnService.java    From incubator-gobblin with Apache License 2.0 4 votes vote down vote up
public YarnService(Config config, String applicationName, String applicationId, YarnConfiguration yarnConfiguration,
    FileSystem fs, EventBus eventBus, HelixManager helixManager) throws Exception {
  this.applicationName = applicationName;
  this.applicationId = applicationId;

  this.config = config;

  this.eventBus = eventBus;

  this.helixManager = helixManager;

  this.gobblinMetrics = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildGobblinMetrics()) : Optional.<GobblinMetrics>absent();

  this.eventSubmitter = config.getBoolean(ConfigurationKeys.METRICS_ENABLED_KEY) ?
      Optional.of(buildEventSubmitter()) : Optional.<EventSubmitter>absent();

  this.yarnConfiguration = yarnConfiguration;
  this.fs = fs;

  this.amrmClientAsync = closer.register(
      AMRMClientAsync.createAMRMClientAsync(1000, new AMRMClientCallbackHandler()));
  this.amrmClientAsync.init(this.yarnConfiguration);
  this.nmClientAsync = closer.register(NMClientAsync.createNMClientAsync(getNMClientCallbackHandler()));
  this.nmClientAsync.init(this.yarnConfiguration);

  this.initialContainers = config.getInt(GobblinYarnConfigurationKeys.INITIAL_CONTAINERS_KEY);
  this.requestedContainerMemoryMbs = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY);
  this.requestedContainerCores = config.getInt(GobblinYarnConfigurationKeys.CONTAINER_CORES_KEY);
  this.containerHostAffinityEnabled = config.getBoolean(GobblinYarnConfigurationKeys.CONTAINER_HOST_AFFINITY_ENABLED);

  this.helixInstanceMaxRetries = config.getInt(GobblinYarnConfigurationKeys.HELIX_INSTANCE_MAX_RETRIES);
  this.helixInstanceTags = ConfigUtils.getString(config, GobblinClusterConfigurationKeys.HELIX_INSTANCE_TAGS_KEY, null);

  this.containerJvmArgs = config.hasPath(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY) ?
      Optional.of(config.getString(GobblinYarnConfigurationKeys.CONTAINER_JVM_ARGS_KEY)) :
      Optional.<String>absent();

  this.containerLaunchExecutor = ScalingThreadPoolExecutor.newScalingThreadPool(5, Integer.MAX_VALUE, 0L,
      ExecutorsUtils.newThreadFactory(Optional.of(LOGGER), Optional.of("ContainerLaunchExecutor")));

  this.tokens = getSecurityTokens();

  this.releasedContainerCache = CacheBuilder.newBuilder().expireAfterAccess(ConfigUtils.getInt(config,
      GobblinYarnConfigurationKeys.RELEASED_CONTAINERS_CACHE_EXPIRY_SECS,
      GobblinYarnConfigurationKeys.DEFAULT_RELEASED_CONTAINERS_CACHE_EXPIRY_SECS), TimeUnit.SECONDS).build();

  this.jvmMemoryXmxRatio = ConfigUtils.getDouble(this.config,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY,
      GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_XMX_RATIO);

  Preconditions.checkArgument(this.jvmMemoryXmxRatio >= 0 && this.jvmMemoryXmxRatio <= 1,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY + " must be between 0 and 1 inclusive");

  this.jvmMemoryOverheadMbs = ConfigUtils.getInt(this.config,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY,
      GobblinYarnConfigurationKeys.DEFAULT_CONTAINER_JVM_MEMORY_OVERHEAD_MBS);

  Preconditions.checkArgument(this.jvmMemoryOverheadMbs < this.requestedContainerMemoryMbs * this.jvmMemoryXmxRatio,
      GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_OVERHEAD_MBS_KEY + " cannot be more than "
          + GobblinYarnConfigurationKeys.CONTAINER_MEMORY_MBS_KEY + " * "
          + GobblinYarnConfigurationKeys.CONTAINER_JVM_MEMORY_XMX_RATIO_KEY);

  this.appViewAcl = ConfigUtils.getString(this.config, GobblinYarnConfigurationKeys.APP_VIEW_ACL,
      GobblinYarnConfigurationKeys.DEFAULT_APP_VIEW_ACL);
  this.containerTimezone = ConfigUtils.getString(this.config, GobblinYarnConfigurationKeys.GOBBLIN_YARN_CONTAINER_TIMEZONE,
      GobblinYarnConfigurationKeys.DEFAULT_GOBBLIN_YARN_CONTAINER_TIMEZONE);
}
 
Example #16
Source File: YarnResourceManagerTest.java    From flink with Apache License 2.0 4 votes vote down vote up
@Override
protected NMClientAsync createAndStartNodeManagerClient(YarnConfiguration yarnConfiguration) {
	return testingYarnNMClientAsync;
}
 
Example #17
Source File: NMClientAsyncImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public NMClientAsyncImpl(CallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 
Example #18
Source File: NMClientAsyncImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public NMClientAsyncImpl(CallbackHandler callbackHandler) {
  this(NMClientAsync.class.getName(), callbackHandler);
}
 
Example #19
Source File: ApplicationMaster.java    From TensorFlowOnYARN with Apache License 2.0 4 votes vote down vote up
NMClientAsync getNMClientAsync() {
  return nmClientAsync;
}
 
Example #20
Source File: UnmanagedAmTest.java    From reef with Apache License 2.0 2 votes vote down vote up
@Test
public void testAmShutdown() throws IOException, YarnException {

  Assume.assumeTrue(
      "This test requires a YARN Resource Manager to connect to",
      Boolean.parseBoolean(System.getenv("REEF_TEST_YARN")));

  final YarnConfiguration yarnConfig = new YarnConfiguration();

  // Start YARN client and register the application

  final YarnClient yarnClient = YarnClient.createYarnClient();
  yarnClient.init(yarnConfig);
  yarnClient.start();

  final ContainerLaunchContext containerContext = Records.newRecord(ContainerLaunchContext.class);
  containerContext.setCommands(Collections.<String>emptyList());
  containerContext.setLocalResources(Collections.<String, LocalResource>emptyMap());
  containerContext.setEnvironment(Collections.<String, String>emptyMap());
  containerContext.setTokens(getTokens());

  final ApplicationSubmissionContext appContext = yarnClient.createApplication().getApplicationSubmissionContext();
  appContext.setApplicationName("REEF_Unmanaged_AM_Test");
  appContext.setAMContainerSpec(containerContext);
  appContext.setUnmanagedAM(true);
  appContext.setQueue("default");

  final ApplicationId applicationId = appContext.getApplicationId();
  LOG.log(Level.INFO, "Registered YARN application: {0}", applicationId);

  yarnClient.submitApplication(appContext);

  LOG.log(Level.INFO, "YARN application submitted: {0}", applicationId);

  addToken(yarnClient.getAMRMToken(applicationId));

  // Start the AM

  final AMRMClientAsync<AMRMClient.ContainerRequest> rmClient = AMRMClientAsync.createAMRMClientAsync(1000, this);
  rmClient.init(yarnConfig);
  rmClient.start();

  final NMClientAsync nmClient = new NMClientAsyncImpl(this);
  nmClient.init(yarnConfig);
  nmClient.start();

  final RegisterApplicationMasterResponse registration =
      rmClient.registerApplicationMaster(NetUtils.getHostname(), -1, null);

  LOG.log(Level.INFO, "Unmanaged AM is running: {0}", registration);

  rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, "Success!", null);

  LOG.log(Level.INFO, "Unregistering AM: state {0}", rmClient.getServiceState());

  // Shutdown the AM

  rmClient.stop();
  nmClient.stop();

  // Get the final application report

  final ApplicationReport appReport = yarnClient.getApplicationReport(applicationId);
  final YarnApplicationState appState = appReport.getYarnApplicationState();
  final FinalApplicationStatus finalAttemptStatus = appReport.getFinalApplicationStatus();

  LOG.log(Level.INFO, "Application {0} final attempt {1} status: {2}/{3}", new Object[] {
      applicationId, appReport.getCurrentApplicationAttemptId(), appState, finalAttemptStatus});

  Assert.assertEquals("Application must be in FINISHED state", YarnApplicationState.FINISHED, appState);
  Assert.assertEquals("Final status must be SUCCEEDED", FinalApplicationStatus.SUCCEEDED, finalAttemptStatus);

  // Shutdown YARN client

  yarnClient.stop();
}