org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy Java Examples
The following examples show how to use
org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: ContainerLauncherImpl.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected void serviceInit(Configuration conf) throws Exception { this.limitOnPoolSize = conf.getInt( MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT); LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize); this.initialPoolSize = conf.getInt( MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE); LOG.info("The thread pool initial size is " + this.initialPoolSize); super.serviceInit(conf); cmProxy = new ContainerManagementProtocolProxy(conf); }
Example #2
Source File: TestContainerLauncher.java From hadoop with Apache License 2.0 | 5 votes |
@Override protected ContainerLauncher createContainerLauncher(final AppContext context) { return new ContainerLauncherImpl(context) { @Override public ContainerManagementProtocolProxyData getCMProxy( String containerMgrBindAddr, ContainerId containerId) throws IOException { InetSocketAddress addr = NetUtils.getConnectAddress(server); String containerManagerBindAddr = addr.getHostName() + ":" + addr.getPort(); Token token = tokenSecretManager.createNMToken( containerId.getApplicationAttemptId(), NodeId.newInstance(addr.getHostName(), addr.getPort()), "user"); ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(conf); ContainerManagementProtocolProxyData proxy = cmProxy.new ContainerManagementProtocolProxyData( YarnRPC.create(conf), containerManagerBindAddr, containerId, token); return proxy; } }; }
Example #3
Source File: ContainerLauncherImpl.java From big-c with Apache License 2.0 | 5 votes |
@Override protected void serviceInit(Configuration conf) throws Exception { this.limitOnPoolSize = conf.getInt( MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT); LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize); this.initialPoolSize = conf.getInt( MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE, MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREADPOOL_INITIAL_SIZE); LOG.info("The thread pool initial size is " + this.initialPoolSize); super.serviceInit(conf); cmProxy = new ContainerManagementProtocolProxy(conf); }
Example #4
Source File: TestContainerLauncher.java From big-c with Apache License 2.0 | 5 votes |
@Override protected ContainerLauncher createContainerLauncher(final AppContext context) { return new ContainerLauncherImpl(context) { @Override public ContainerManagementProtocolProxyData getCMProxy( String containerMgrBindAddr, ContainerId containerId) throws IOException { InetSocketAddress addr = NetUtils.getConnectAddress(server); String containerManagerBindAddr = addr.getHostName() + ":" + addr.getPort(); Token token = tokenSecretManager.createNMToken( containerId.getApplicationAttemptId(), NodeId.newInstance(addr.getHostName(), addr.getPort()), "user"); ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(conf); ContainerManagementProtocolProxyData proxy = cmProxy.new ContainerManagementProtocolProxyData( YarnRPC.create(conf), containerManagerBindAddr, containerId, token); return proxy; } }; }
Example #5
Source File: ContainerLauncherImpl.java From hadoop with Apache License 2.0 | 4 votes |
public ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(String containerMgrBindAddr, ContainerId containerId) throws IOException { return cmProxy.getProxy(containerMgrBindAddr, containerId); }
Example #6
Source File: ContainerLauncherImpl.java From big-c with Apache License 2.0 | 4 votes |
public ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy(String containerMgrBindAddr, ContainerId containerId) throws IOException { return cmProxy.getProxy(containerMgrBindAddr, containerId); }
Example #7
Source File: ContainerLauncherImpl.java From incubator-tez with Apache License 2.0 | 4 votes |
@Override public void serviceStart() { cmProxy = new ContainerManagementProtocolProxy(getConfig()); ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat( "ContainerLauncher #%d").setDaemon(true).build(); // Start with a default core-pool size of 10 and change it dynamically. launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE, Integer.MAX_VALUE, 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf); eventHandlingThread = new Thread() { @Override public void run() { NMCommunicatorEvent event = null; while (!Thread.currentThread().isInterrupted()) { try { event = eventQueue.take(); } catch (InterruptedException e) { if(!serviceStopped.get()) { LOG.error("Returning, interrupted : " + e); } return; } int poolSize = launcherPool.getCorePoolSize(); // See if we need up the pool size only if haven't reached the // maximum limit yet. if (poolSize != limitOnPoolSize) { // nodes where containers will run at *this* point of time. This is // *not* the cluster size and doesn't need to be. int numNodes = context.getAllNodes().size(); int idealPoolSize = Math.min(limitOnPoolSize, numNodes); if (poolSize < idealPoolSize) { // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the // later is just a buffer so we are not always increasing the // pool-size int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize + INITIAL_POOL_SIZE); LOG.info("Setting ContainerLauncher pool size to " + newPoolSize + " as number-of-nodes to talk to is " + numNodes); launcherPool.setCorePoolSize(newPoolSize); } } // the events from the queue are handled in parallel // using a thread pool launcherPool.execute(createEventProcessor(event)); // TODO: Group launching of multiple containers to a single // NodeManager into a single connection } } }; eventHandlingThread.setName("ContainerLauncher Event Handler"); eventHandlingThread.start(); }
Example #8
Source File: ContainerLauncherImpl.java From incubator-tez with Apache License 2.0 | 4 votes |
protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy( ContainerId containerID, final String containerManagerBindAddr, Token containerToken) throws IOException { return cmProxy.getProxy(containerManagerBindAddr, containerID); }
Example #9
Source File: TezContainerLauncherImpl.java From tez with Apache License 2.0 | 4 votes |
@Override public void start() throws TezException { // pass a copy of config to ContainerManagementProtocolProxy until YARN-3497 is fixed cmProxy = new ContainerManagementProtocolProxy(conf); ThreadFactory tf = new ThreadFactoryBuilder().setNameFormat( "ContainerLauncher #%d").setDaemon(true).build(); // Start with a default core-pool size of 10 and change it dynamically. launcherPool = new ThreadPoolExecutor(INITIAL_POOL_SIZE, Integer.MAX_VALUE, 1, TimeUnit.HOURS, new LinkedBlockingQueue<Runnable>(), tf, new CustomizedRejectedExecutionHandler()); eventHandlingThread = new Thread() { @Override public void run() { ContainerOp event = null; while (!Thread.currentThread().isInterrupted()) { try { event = eventQueue.take(); } catch (InterruptedException e) { if(!serviceStopped.get()) { LOG.error("Returning, interrupted : " + e); } return; } int poolSize = launcherPool.getCorePoolSize(); // See if we need up the pool size only if haven't reached the // maximum limit yet. if (poolSize != limitOnPoolSize) { // nodes where containers will run at *this* point of time. This is // *not* the cluster size and doesn't need to be. int numNodes = getContext().getNumNodes(TezConstants.getTezYarnServicePluginName()); int idealPoolSize = Math.min(limitOnPoolSize, numNodes); if (poolSize < idealPoolSize) { // Bump up the pool size to idealPoolSize+INITIAL_POOL_SIZE, the // later is just a buffer so we are not always increasing the // pool-size int newPoolSize = Math.min(limitOnPoolSize, idealPoolSize + INITIAL_POOL_SIZE); LOG.info("Setting ContainerLauncher pool size to " + newPoolSize + " as number-of-nodes to talk to is " + numNodes); launcherPool.setCorePoolSize(newPoolSize); } } // the events from the queue are handled in parallel // using a thread pool launcherPool.execute(createEventProcessor(event)); // TODO: Group launching of multiple containers to a single // NodeManager into a single connection } } }; eventHandlingThread.setName("ContainerLauncher Event Handler"); eventHandlingThread.start(); boolean cleanupDagDataOnComplete = ShuffleUtils.isTezShuffleHandler(conf) && conf.getBoolean(TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION, TezConfiguration.TEZ_AM_DAG_CLEANUP_ON_COMPLETION_DEFAULT); if (cleanupDagDataOnComplete) { String deletionTrackerClassName = conf.get(TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS, TezConfiguration.TEZ_AM_DELETION_TRACKER_CLASS_DEFAULT); deletionTracker = ReflectionUtils.createClazzInstance( deletionTrackerClassName, new Class[]{Configuration.class}, new Object[]{conf}); } }
Example #10
Source File: TezContainerLauncherImpl.java From tez with Apache License 2.0 | 4 votes |
protected ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData getCMProxy( ContainerId containerID, final String containerManagerBindAddr, Token containerToken) throws IOException { return cmProxy.getProxy(containerManagerBindAddr, containerID); }