org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy Java Examples

The following examples show how to use org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ContainerLauncherImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.limitOnPoolSize = conf.getInt(
      MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
      MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
  LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);

  this.initialPoolSize = conf.getInt(
      MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE,
      MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE);
  LOG.info("The thread pool initial size is " + this.initialPoolSize);

  super.serviceInit(conf);
  cmProxy = new ContainerManagementProtocolProxy(conf);
}
 
Example #2
Source File: TestContainerLauncher.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
protected ContainerLauncher
    createContainerLauncher(final AppContext context) {
  return new ContainerLauncherImpl(context) {

    @Override
    public ContainerManagementProtocolProxyData getCMProxy(
        String containerMgrBindAddr, ContainerId containerId)
        throws IOException {
      InetSocketAddress addr = NetUtils.getConnectAddress(server);
      String containerManagerBindAddr =
          addr.getHostName() + ":" + addr.getPort();
      Token token =
          tokenSecretManager.createNMToken(
            containerId.getApplicationAttemptId(),
            NodeId.newInstance(addr.getHostName(), addr.getPort()), "user");
      ContainerManagementProtocolProxy cmProxy =
          new ContainerManagementProtocolProxy(conf);
      ContainerManagementProtocolProxyData proxy =
          cmProxy.new ContainerManagementProtocolProxyData(
            YarnRPC.create(conf), containerManagerBindAddr, containerId,
            token);
      return proxy;
    }
  };

}
 
Example #3
Source File: ContainerLauncherImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected void serviceInit(Configuration conf) throws Exception {
  this.limitOnPoolSize = conf.getInt(
      MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
      MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
  LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);

  this.initialPoolSize = conf.getInt(
      MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE,
      MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE);
  LOG.info("The thread pool initial size is " + this.initialPoolSize);

  super.serviceInit(conf);
  cmProxy = new ContainerManagementProtocolProxy(conf);
}
 
Example #4
Source File: TestContainerLauncher.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
protected ContainerLauncher
    createContainerLauncher(final AppContext context) {
  return new ContainerLauncherImpl(context) {

    @Override
    public ContainerManagementProtocolProxyData getCMProxy(
        String containerMgrBindAddr, ContainerId containerId)
        throws IOException {
      InetSocketAddress addr = NetUtils.getConnectAddress(server);
      String containerManagerBindAddr =
          addr.getHostName() + ":" + addr.getPort();
      Token token =
          tokenSecretManager.createNMToken(
            containerId.getApplicationAttemptId(),
            NodeId.newInstance(addr.getHostName(), addr.getPort()), "user");
      ContainerManagementProtocolProxy cmProxy =
          new ContainerManagementProtocolProxy(conf);
      ContainerManagementProtocolProxyData proxy =
          cmProxy.new ContainerManagementProtocolProxyData(
            YarnRPC.create(conf), containerManagerBindAddr, containerId,
            token);
      return proxy;
    }
  };

}
 
Example #5
Source File: ContainerLauncherImpl.java    From hadoop with Apache License 2.0 4 votes vote down vote up
public ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData
    getCMProxy(String containerMgrBindAddr, ContainerId containerId)
        throws IOException {
  return cmProxy.getProxy(containerMgrBindAddr, containerId);
}
 
Example #6
Source File: ContainerLauncherImpl.java    From big-c with Apache License 2.0 4 votes vote down vote up
public ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData
    getCMProxy(String containerMgrBindAddr, ContainerId containerId)
        throws IOException {
  return cmProxy.getProxy(containerMgrBindAddr, containerId);
}
 
Example #7
Source File: ContainerLauncherImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
@Override
public void serviceStart() {
  cmProxy =
      new ContainerManagementProtocolProxy(getConfig());

  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "ContainerLauncher #%d").setDaemon(true).build();

  // Start with a default core-pool size of 10 and change it dynamically.
  launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
      Integer.MAX_VALUE, 1, TimeUnit.HOURS,
      new LinkedBlockingQueue<Runnable>(),
      tf);
  eventHandlingThread = new Thread() {
    @Override
    public void run() {
      NMCommunicatorEvent event = null;
      while (!Thread.currentThread().isInterrupted()) {
        try {
          event = eventQueue.take();
        } catch (InterruptedException e) {
          if(!serviceStopped.get()) {
            LOG.error("Returning, interrupted : " + e);
          }
          return;
        }
        int poolSize = launcherPool.getCorePoolSize();

        // See if we need up the pool size only if haven't reached the
        // maximum limit yet.
        if (poolSize != limitOnPoolSize) {

          // nodes where containers will run at *this* point of time. This is
          // *not* the cluster size and doesn't need to be.
          int numNodes = context.getAllNodes().size();
          int idealPoolSize = Math.min(limitOnPoolSize, numNodes);

          if (poolSize < idealPoolSize) {
            // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
            // later is just a buffer so we are not always increasing the
            // pool-size
            int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
                + INITIAL_POOL_SIZE);
            LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
                + " as number-of-nodes to talk to is " + numNodes);
            launcherPool.setCorePoolSize(newPoolSize);
          }
        }

        // the events from the queue are handled in parallel
        // using a thread pool
        launcherPool.execute(createEventProcessor(event));

        // TODO: Group launching of multiple containers to a single
        // NodeManager into a single connection
      }
    }
  };
  eventHandlingThread.setName("ContainerLauncher Event Handler");
  eventHandlingThread.start();
}
 
Example #8
Source File: ContainerLauncherImpl.java    From incubator-tez with Apache License 2.0 4 votes vote down vote up
protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
    ContainerId containerID, final String containerManagerBindAddr,
    Token containerToken) throws IOException {
  return cmProxy.getProxy(containerManagerBindAddr, containerID);
}
 
Example #9
Source File: TezContainerLauncherImpl.java    From tez with Apache License 2.0 4 votes vote down vote up
@Override
public void start() throws TezException {
  // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed
  cmProxy =
      new ContainerManagementProtocolProxy(conf);

  ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat(
      "ContainerLauncher #%d").setDaemon(true).build();

  // Start with a default core-pool size of 10 and change it dynamically.
  launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE,
      Integer.MAX_VALUE, 1, TimeUnit.HOURS,
      new LinkedBlockingQueue<Runnable>(),
      tf, new CustomizedRejectedExecutionHandler());
  eventHandlingThread = new Thread() {
    @Override
    public void run() {
      ContainerOp event = null;
      while (!Thread.currentThread().isInterrupted()) {
        try {
          event = eventQueue.take();
        } catch (InterruptedException e) {
          if(!serviceStopped.get()) {
            LOG.error("Returning, interrupted : " + e);
          }
          return;
        }
        int poolSize = launcherPool.getCorePoolSize();

        // See if we need up the pool size only if haven't reached the
        // maximum limit yet.
        if (poolSize != limitOnPoolSize) {

          // nodes where containers will run at *this* point of time. This is
          // *not* the cluster size and doesn't need to be.
          int numNodes =
              getContext().getNumNodes(TezConstants.getTezYarnServicePluginName());
          int idealPoolSize = Math.min(limitOnPoolSize, numNodes);

          if (poolSize < idealPoolSize) {
            // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the
            // later is just a buffer so we are not always increasing the
            // pool-size
            int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize
                + INITIAL_POOL_SIZE);
            LOG.info("Setting ContainerLauncher pool size to " + newPoolSize
                + " as number-of-nodes to talk to is " + numNodes);
            launcherPool.setCorePoolSize(newPoolSize);
          }
        }

        // the events from the queue are handled in parallel
        // using a thread pool
        launcherPool.execute(createEventProcessor(event));

        // TODO: Group launching of multiple containers to a single
        // NodeManager into a single connection
      }
    }
  };
  eventHandlingThread.setName("ContainerLauncher Event Handler");
  eventHandlingThread.start();
  boolean cleanupDagDataOnComplete = ShuffleUtils.isTezShuffleHandler(conf)
      && conf.getBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION,
      TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT);
  if (cleanupDagDataOnComplete) {
    String deletionTrackerClassName = conf.get(TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS,
        TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS_DEFAULT);
    deletionTracker = ReflectionUtils.createClazzInstance(
        deletionTrackerClassName, new Class[]{Configuration.class}, new Object[]{conf});
  }
}
 
Example #10
Source File: TezContainerLauncherImpl.java    From tez with Apache License 2.0 4 votes vote down vote up
protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(
    ContainerId containerID, final String containerManagerBindAddr,
    Token containerToken) throws IOException {
  return cmProxy.getProxy(containerManagerBindAddr, containerID);
}