org.apache.hadoop.yarn.server.nodemanager.NodeManager Java Examples
The following examples show how to use
org.apache.hadoop.yarn.server.nodemanager.NodeManager.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: Utils.java From spliceengine with GNU Affero General Public License v3.0 | 6 votes |
public static void waitForNMToRegister(NodeManager nm) throws Exception{ NMTokenSecretManagerInNM nmTokenSecretManagerNM = nm.getNMContext().getNMTokenSecretManager(); NMContainerTokenSecretManager containerTokenSecretManager = nm.getNMContext().getContainerTokenSecretManager(); int attempt = 60; while(attempt-- > 0) { try { if (nmTokenSecretManagerNM.getCurrentKey() != null && containerTokenSecretManager.getCurrentKey() != null) { break; } } catch (Exception e) { } Thread.sleep(2000); } }
Example #2
Source File: YarnTestBase.java From flink with Apache License 2.0 | 5 votes |
public static int getRunningContainers() { int count = 0; for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) { NodeManager nm = yarnCluster.getNodeManager(nmId); ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); count += containers.size(); } return count; }
Example #3
Source File: YarnTestBase.java From Flink-CEPplus with Apache License 2.0 | 5 votes |
public static int getRunningContainers() { int count = 0; for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) { NodeManager nm = yarnCluster.getNodeManager(nmId); ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); count += containers.size(); } return count; }
Example #4
Source File: TestContainerManagerSecurity.java From hadoop with Apache License 2.0 | 5 votes |
protected void waitForNMToReceiveNMTokenKey( NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm) throws InterruptedException { int attempt = 60; ContainerManagerImpl cm = ((ContainerManagerImpl) nm.getNMContext().getContainerManager()); while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM .getNodeId() == null) && attempt-- > 0) { Thread.sleep(2000); } }
Example #5
Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0 | 5 votes |
protected void waitForNMToReceiveNMTokenKey( NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm) throws InterruptedException { int attempt = 60; ContainerManagerImpl cm = ((ContainerManagerImpl) nm.getNMContext().getContainerManager()); while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM .getNodeId() == null) && attempt-- > 0) { Thread.sleep(2000); } }
Example #6
Source File: YarnTestBase.java From flink with Apache License 2.0 | 5 votes |
public static int getRunningContainers() { int count = 0; for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) { NodeManager nm = yarnCluster.getNodeManager(nmId); ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers(); count += containers.size(); } return count; }
Example #7
Source File: TestDistributedShellWithNodeLabels.java From big-c with Apache License 2.0 | 5 votes |
private void initializeNodeLabels() throws IOException { RMContext rmContext = distShellTest.yarnCluster.getResourceManager(0).getRMContext(); // Setup node labels RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); Set<String> labels = new HashSet<String>(); labels.add("x"); labelsMgr.addToCluserNodeLabels(labels); // Setup queue access to node labels distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x"); distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity", "100"); distShellTest.conf.set("yarn.scheduler.capacity.root.default.accessible-node-labels", "x"); distShellTest.conf.set( "yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity", "100"); rmContext.getScheduler().reinitialize(distShellTest.conf, rmContext); // Fetch node-ids from yarn cluster NodeId[] nodeIds = new NodeId[NUM_NMS]; for (int i = 0; i < NUM_NMS; i++) { NodeManager mgr = distShellTest.yarnCluster.getNodeManager(i); nodeIds[i] = mgr.getNMContext().getNodeId(); } // Set label x to NM[1] labelsMgr.addLabelsToNode(ImmutableMap.of(nodeIds[1], labels)); }
Example #8
Source File: TestDistributedShellWithNodeLabels.java From hadoop with Apache License 2.0 | 5 votes |
private void initializeNodeLabels() throws IOException { RMContext rmContext = distShellTest.yarnCluster.getResourceManager(0).getRMContext(); // Setup node labels RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager(); Set<String> labels = new HashSet<String>(); labels.add("x"); labelsMgr.addToCluserNodeLabels(labels); // Setup queue access to node labels distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x"); distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity", "100"); distShellTest.conf.set("yarn.scheduler.capacity.root.default.accessible-node-labels", "x"); distShellTest.conf.set( "yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity", "100"); rmContext.getScheduler().reinitialize(distShellTest.conf, rmContext); // Fetch node-ids from yarn cluster NodeId[] nodeIds = new NodeId[NUM_NMS]; for (int i = 0; i < NUM_NMS; i++) { NodeManager mgr = distShellTest.yarnCluster.getNodeManager(i); nodeIds[i] = mgr.getNMContext().getNodeId(); } // Set label x to NM[1] labelsMgr.addLabelsToNode(ImmutableMap.of(nodeIds[1], labels)); }
Example #9
Source File: BaseContainerManagerTest.java From hadoop with Apache License 2.0 | 5 votes |
@Before public void setup() throws IOException { localFS.delete(new Path(localDir.getAbsolutePath()), true); localFS.delete(new Path(tmpDir.getAbsolutePath()), true); localFS.delete(new Path(localLogDir.getAbsolutePath()), true); localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true); localDir.mkdir(); tmpDir.mkdir(); localLogDir.mkdir(); remoteLogDir.mkdir(); LOG.info("Created localDir in " + localDir.getAbsolutePath()); LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath()); String bindAddress = "0.0.0.0:12345"; conf.set(YarnConfiguration.NM_ADDRESS, bindAddress); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath()); conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1); // Default delSrvc delSrvc = createDeletionService(); delSrvc.init(conf); exec = createContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); nodeHealthChecker = new NodeHealthCheckerService( NodeManager.getNodeHealthScriptRunner(conf), dirsHandler); nodeHealthChecker.init(conf); containerManager = createContainerManager(delSrvc); ((NMContext)context).setContainerManager(containerManager); nodeStatusUpdater.init(conf); containerManager.init(conf); nodeStatusUpdater.start(); }
Example #10
Source File: ContainerManagerImpl.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void serviceStart() throws Exception { // Enqueue user dirs in deletion context Configuration conf = getConfig(); final InetSocketAddress initialAddress = conf.getSocketAddr( YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS, YarnConfiguration.DEFAULT_NM_PORT); boolean usingEphemeralPort = (initialAddress.getPort() == 0); if (context.getNMStateStore().canRecover() && usingEphemeralPort) { throw new IllegalArgumentException("Cannot support recovery with an " + "ephemeral server port. Check the setting of " + YarnConfiguration.NM_ADDRESS); } // If recovering then delay opening the RPC service until the recovery // of resources and containers have completed, otherwise requests from // clients during recovery can interfere with the recovery process. final boolean delayedRpcServerStart = context.getNMStateStore().canRecover(); Configuration serverConf = new Configuration(conf); // always enforce it to be token-based. serverConf.set( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, SaslRpcServer.AuthMethod.TOKEN.toString()); YarnRPC rpc = YarnRPC.create(conf); server = rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, serverConf, this.context.getNMTokenSecretManager(), conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT)); // Enable service authorization? if (conf.getBoolean( CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, false)) { refreshServiceAcls(conf, new NMPolicyProvider()); } LOG.info("Blocking new container-requests as container manager rpc" + " server is still starting."); this.setBlockNewContainerRequests(true); String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST); String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS); String hostOverride = null; if (bindHost != null && !bindHost.isEmpty() && nmAddress != null && !nmAddress.isEmpty()) { //a bind-host case with an address, to support overriding the first //hostname found when querying for our hostname with the specified //address, combine the specified address with the actual port listened //on by the server hostOverride = nmAddress.split(":")[0]; } // setup node ID InetSocketAddress connectAddress; if (delayedRpcServerStart) { connectAddress = NetUtils.getConnectAddress(initialAddress); } else { server.start(); connectAddress = NetUtils.getConnectAddress(server); } NodeId nodeId = buildNodeId(connectAddress, hostOverride); ((NodeManager.NMContext)context).setNodeId(nodeId); this.context.getNMTokenSecretManager().setNodeId(nodeId); this.context.getContainerTokenSecretManager().setNodeId(nodeId); // start remaining services super.serviceStart(); if (delayedRpcServerStart) { waitForRecoveredContainers(); server.start(); // check that the node ID is as previously advertised connectAddress = NetUtils.getConnectAddress(server); NodeId serverNode = buildNodeId(connectAddress, hostOverride); if (!serverNode.equals(nodeId)) { throw new IOException("Node mismatch after server started, expected '" + nodeId + "' but found '" + serverNode + "'"); } } LOG.info("ContainerManager started at " + connectAddress); LOG.info("ContainerManager bound to " + initialAddress); }
Example #11
Source File: SpliceTestYarnPlatform.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
private NodeManager getNodeManager() { return ((MiniYARNClusterSplice)yarnCluster).getNodeManager(0); }
Example #12
Source File: TestContainerLogsPage.java From big-c with Apache License 2.0 | 4 votes |
@Test(timeout=30000) public void testContainerLogDirs() throws IOException, YarnException { File absLogDir = new File("target", TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile(); String logdirwithFile = absLogDir.toURI().toString(); Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, new ApplicationACLsManager(conf), new NMNullStateStoreService(), null); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); String user = "nobody"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); Application app = mock(Application.class); when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); nmContext.getApplications().put(appId, app); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.setState(ContainerState.RUNNING); nmContext.getContainers().put(container1, container); List<File> files = null; files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); // After container is completed, it is removed from nmContext nmContext.getContainers().remove(container1); Assert.assertNull(nmContext.getContainers().get(container1)); files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler); // good log dirs are empty and nm log dir is in the full log dir list. when(dirsHandlerForFullDisk.getLogDirs()). thenReturn(new ArrayList<String>()); when(dirsHandlerForFullDisk.getLogDirsForRead()). thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()})); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager(conf), new NMNullStateStoreService(), null); nmContext.getApplications().put(appId, app); container.setState(ContainerState.RUNNING); nmContext.getContainers().put(container1, container); List<File> dirs = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); File containerLogDir = new File(absLogDir, appId + "/" + container1); Assert.assertTrue(dirs.contains(containerLogDir)); }
Example #13
Source File: TestNMWebServicesApps.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void configureServlets() { conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); dirsHandler = healthChecker.getDiskHandler(); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #14
Source File: TestNMWebServices.java From big-c with Apache License 2.0 | 4 votes |
@Override protected void configureServlets() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); dirsHandler = healthChecker.getDiskHandler(); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #15
Source File: TestNMWebServer.java From big-c with Apache License 2.0 | 4 votes |
private int startNMWebAppServer(String webAddr) { Context nmContext = new NodeManager.NMContext(null, null, null, null, null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { return 0; } @Override public long getPmemAllocatedForContainers() { return 0; } @Override public long getVCoresAllocatedForContainers() { return 0; } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler); try { server.init(conf); server.start(); return server.getPort(); } finally { server.stop(); healthChecker.stop(); } }
Example #16
Source File: TestNMWebServer.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testNMWebApp() throws IOException, YarnException { Context nmContext = new NodeManager.NMContext(null, null, null, null, null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { return 0; } @Override public long getPmemAllocatedForContainers() { return 0; } @Override public long getVCoresAllocatedForContainers() { return 0; } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler); server.init(conf); server.start(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); Dispatcher dispatcher = new AsyncDispatcher(); String user = "nobody"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); Application app = mock(Application.class); when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); nmContext.getApplications().put(appId, app); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); ContainerId container2 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1); NodeManagerMetrics metrics = mock(NodeManagerMetrics.class); NMStateStoreService stateStore = new NMNullStateStoreService(); for (ContainerId containerId : new ContainerId[] { container1, container2}) { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); Token containerToken = BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); Container container = new ContainerImpl(nmContext, conf, dispatcher, stateStore, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken), null, 0, user, false, null) { @Override public ContainerState getContainerState() { return ContainerState.RUNNING; }; }; nmContext.getContainers().put(containerId, container); //TODO: Gross hack. Fix in code. ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); nmContext.getApplications().get(applicationId).getContainers() .put(containerId, container); writeContainerLogs(nmContext, containerId, dirsHandler); } // TODO: Pull logs and test contents. // Thread.sleep(1000000); }
Example #17
Source File: TestNMWebServicesContainers.java From big-c with Apache License 2.0 | 4 votes |
protected void configureServlets() { resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.init(conf); dirsHandler = healthChecker.getDiskHandler(); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null, null) { public NodeId getNodeId() { return NodeId.newInstance("testhost.foo.com", 8042); }; public int getHttpPort() { return 1234; }; }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #18
Source File: MiniYARNClusterSplice.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
public NodeManager getNodeManager(int i) { return this.nodeManagers[i]; }
Example #19
Source File: SpliceTestYarnPlatform.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
public void start(int nodeCount) throws Exception { if (yarnCluster == null) { LOG.info("Starting up YARN cluster with "+nodeCount+" nodes. Server yarn-site.xml is: "+yarnSiteConfigURL); UserGroupInformation ugi; if (secure) ugi = UserGroupInformation.loginUserFromKeytabAndReturnUGI("yarn/[email protected]", keytab); else ugi = UserGroupInformation.createRemoteUser("yarn"); UserGroupInformation.setLoginUser(ugi); ugi.doAs(new PrivilegedExceptionAction<Void>() { @Override public Void run() throws Exception { yarnCluster = new MiniYARNClusterSplice(SpliceTestYarnPlatform.class.getSimpleName(), nodeCount, 1, 1); yarnCluster.init(conf); yarnCluster.start(); return null; } }); NodeManager nm = getNodeManager(); waitForNMToRegister(nm); // save the server config to classpath so yarn clients can read it Configuration yarnClusterConfig = yarnCluster.getConfig(); yarnClusterConfig.set("yarn.application.classpath", new File(yarnSiteConfigURL.getPath()).getParent()); yarnClusterConfig.set("fs.s3a.impl","com.splicemachine.fs.s3.PrestoS3FileSystem"); //write the document to a buffer (not directly to the file, as that //can cause the file being written to get read -which will then fail. ByteArrayOutputStream bytesOut = new ByteArrayOutputStream(); yarnClusterConfig.writeXml(bytesOut); bytesOut.close(); //write the bytes to the file in the classpath OutputStream os = new FileOutputStream(new File(yarnSiteConfigURL.getPath())); os.write(bytesOut.toByteArray()); os.close(); } LOG.info("YARN cluster started."); }
Example #20
Source File: SpliceTestYarnPlatform.java From spliceengine with GNU Affero General Public License v3.0 | 4 votes |
private static void waitForNMToRegister(NodeManager nm) throws Exception { Utils.waitForNMToRegister(nm); }
Example #21
Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0 | 4 votes |
/** * This tests whether a containerId is serialized/deserialized with epoch. * * @throws IOException * @throws InterruptedException * @throws YarnException */ private void testContainerTokenWithEpoch(Configuration conf) throws IOException, InterruptedException, YarnException { LOG.info("Running test for serializing/deserializing containerIds"); NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext() .getNMTokenSecretManager(); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L); NodeManager nm = yarnCluster.getNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager(); String user = "test"; waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.getNMContext().getNodeId(); // Both id should be equal. Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId()); // Creating a normal Container Token RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext(). getContainerTokenSecretManager(); Resource r = Resource.newInstance(1230, 2); Token containerToken = containerTokenSecretManager.createContainerToken(cId, nodeId, user, r, Priority.newInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(); byte[] tokenIdentifierContent = containerToken.getIdentifier().array(); DataInputBuffer dib = new DataInputBuffer(); dib.reset(tokenIdentifierContent, tokenIdentifierContent.length); containerTokenIdentifier.readFields(dib); Assert.assertEquals(cId, containerTokenIdentifier.getContainerID()); Assert.assertEquals( cId.toString(), containerTokenIdentifier.getContainerID().toString()); Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.create(conf); testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken, false); List<ContainerId> containerIds = new LinkedList<ContainerId>(); containerIds.add(cId); ContainerManagementProtocol proxy = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user); GetContainerStatusesResponse res = proxy.getContainerStatuses( GetContainerStatusesRequest.newInstance(containerIds)); Assert.assertNotNull(res.getContainerStatuses().get(0)); Assert.assertEquals( cId, res.getContainerStatuses().get(0).getContainerId()); Assert.assertEquals(cId.toString(), res.getContainerStatuses().get(0).getContainerId().toString()); }
Example #22
Source File: TestContainerManagerSecurity.java From big-c with Apache License 2.0 | 4 votes |
/** * This tests a malice user getting a proper token but then messing with it by * tampering with containerID/Resource etc.. His/her containers should be * rejected. * * @throws IOException * @throws InterruptedException * @throws YarnException */ private void testContainerToken(Configuration conf) throws IOException, InterruptedException, YarnException { LOG.info("Running test for malice user"); /* * We need to check for containerToken (authorization). * Here we will be assuming that we have valid NMToken * 1) ContainerToken used is expired. * 2) ContainerToken is tampered (resource is modified). */ NMTokenSecretManagerInRM nmTokenSecretManagerInRM = yarnCluster.getResourceManager().getRMContext() .getNMTokenSecretManager(); ApplicationId appId = ApplicationId.newInstance(1, 1); ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0); ContainerId cId = ContainerId.newContainerId(appAttemptId, 0); NodeManager nm = yarnCluster.getNodeManager(0); NMTokenSecretManagerInNM nmTokenSecretManagerInNM = nm.getNMContext().getNMTokenSecretManager(); String user = "test"; waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm); NodeId nodeId = nm.getNMContext().getNodeId(); // Both id should be equal. Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(), nmTokenSecretManagerInRM.getCurrentKey().getKeyId()); RMContainerTokenSecretManager containerTokenSecretManager = yarnCluster.getResourceManager().getRMContext(). getContainerTokenSecretManager(); Resource r = Resource.newInstance(1230, 2); Token containerToken = containerTokenSecretManager.createContainerToken( cId, nodeId, user, r, Priority.newInstance(0), 0); ContainerTokenIdentifier containerTokenIdentifier = getContainerTokenIdentifierFromToken(containerToken); // Verify new compatible version ContainerTokenIdentifier can work successfully. ContainerTokenIdentifierForTest newVersionTokenIdentifier = new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message"); byte[] password = containerTokenSecretManager.createPassword(newVersionTokenIdentifier); Token newContainerToken = BuilderUtils.newContainerToken( nodeId, password, newVersionTokenIdentifier); Token nmToken = nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user); YarnRPC rpc = YarnRPC.create(conf); Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId, newContainerToken, nmToken, false).isEmpty()); // Creating a tampered Container Token RMContainerTokenSecretManager tamperedContainerTokenSecretManager = new RMContainerTokenSecretManager(conf); tamperedContainerTokenSecretManager.rollMasterKey(); do { tamperedContainerTokenSecretManager.rollMasterKey(); tamperedContainerTokenSecretManager.activateNextMasterKey(); } while (containerTokenSecretManager.getCurrentKey().getKeyId() == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId()); ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1); // Creating modified containerToken Token containerToken2 = tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId, user, r, Priority.newInstance(0), 0); StringBuilder sb = new StringBuilder("Given Container "); sb.append(cId2); sb.append(" seems to have an illegally generated token."); Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId, containerToken2, nmToken, true).contains(sb.toString())); }
Example #23
Source File: MiniYARNCluster.java From big-c with Apache License 2.0 | 4 votes |
public NodeManager getNodeManager(int i) { return this.nodeManagers[i]; }
Example #24
Source File: TestNMWebServicesContainers.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected void configureServlets() { resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public long getGCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService( NodeManager.getNodeHealthScriptRunner(conf), dirsHandler); healthChecker.init(conf); dirsHandler = healthChecker.getDiskHandler(); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null) { public NodeId getNodeId() { return NodeId.newInstance("testhost.foo.com", 8042); }; public int getHttpPort() { return 1234; }; }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #25
Source File: TestNMWebServer.java From hadoop with Apache License 2.0 | 4 votes |
@Test public void testNMWebApp() throws IOException, YarnException { Context nmContext = new NodeManager.NMContext(null, null, null, null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { return 0; } @Override public long getPmemAllocatedForContainers() { return 0; } @Override public long getVCoresAllocatedForContainers() { return 0; } @Override public long getGCoresAllocatedForContainers() { return 0; } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler); server.init(conf); server.start(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); Dispatcher dispatcher = new AsyncDispatcher(); String user = "nobody"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); Application app = mock(Application.class); when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); nmContext.getApplications().put(appId, app); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); ContainerId container2 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1); NodeManagerMetrics metrics = mock(NodeManagerMetrics.class); NMStateStoreService stateStore = new NMNullStateStoreService(); for (ContainerId containerId : new ContainerId[] { container1, container2}) { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class); long currentTime = System.currentTimeMillis(); Token containerToken = BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime); Container container = new ContainerImpl(conf, dispatcher, stateStore, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken)) { @Override public ContainerState getContainerState() { return ContainerState.RUNNING; }; }; nmContext.getContainers().put(containerId, container); //TODO: Gross hack. Fix in code. ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId(); nmContext.getApplications().get(applicationId).getContainers() .put(containerId, container); writeContainerLogs(nmContext, containerId, dirsHandler); } // TODO: Pull logs and test contents. // Thread.sleep(1000000); }
Example #26
Source File: TestNMWebServer.java From hadoop with Apache License 2.0 | 4 votes |
private int startNMWebAppServer(String webAddr) { Context nmContext = new NodeManager.NMContext(null, null, null, null, null); ResourceView resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { return 0; } @Override public long getPmemAllocatedForContainers() { return 0; } @Override public long getVCoresAllocatedForContainers() { return 0; } @Override public long getGCoresAllocatedForContainers() { return 0; } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler); try { server.init(conf); server.start(); return server.getPort(); } finally { server.stop(); healthChecker.stop(); } }
Example #27
Source File: TestNMWebServer.java From hadoop with Apache License 2.0 | 4 votes |
private NodeHealthCheckerService createNodeHealthCheckerService(Configuration conf) { NodeHealthScriptRunner scriptRunner = NodeManager.getNodeHealthScriptRunner(conf); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); return new NodeHealthCheckerService(scriptRunner, dirsHandler); }
Example #28
Source File: TestNMWebServices.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected void configureServlets() { Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); dirsHandler = new LocalDirsHandlerService(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService( NodeManager.getNodeHealthScriptRunner(conf), dirsHandler); healthChecker.init(conf); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public long getGCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #29
Source File: TestNMWebServicesApps.java From hadoop with Apache License 2.0 | 4 votes |
@Override protected void configureServlets() { conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService( NodeManager.getNodeHealthScriptRunner(conf), dirsHandler); healthChecker.init(conf); dirsHandler = healthChecker.getDiskHandler(); aclsManager = new ApplicationACLsManager(conf); nmContext = new NodeManager.NMContext(null, null, dirsHandler, aclsManager, null); NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999); ((NodeManager.NMContext)nmContext).setNodeId(nodeId); resourceView = new ResourceView() { @Override public long getVmemAllocatedForContainers() { // 15.5G in bytes return new Long("16642998272"); } @Override public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } @Override public long getVCoresAllocatedForContainers() { return new Long("4000"); } @Override public long getGCoresAllocatedForContainers() { return new Long("4000"); } @Override public boolean isVmemCheckEnabled() { return true; } @Override public boolean isPmemCheckEnabled() { return true; } }; nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler); bind(JAXBContextResolver.class); bind(NMWebServices.class); bind(GenericExceptionHandler.class); bind(Context.class).toInstance(nmContext); bind(WebApp.class).toInstance(nmWebApp); bind(ResourceView.class).toInstance(resourceView); bind(ApplicationACLsManager.class).toInstance(aclsManager); bind(LocalDirsHandlerService.class).toInstance(dirsHandler); serve("/*").with(GuiceContainer.class); }
Example #30
Source File: TestContainerLogsPage.java From hadoop with Apache License 2.0 | 4 votes |
@Test(timeout=30000) public void testContainerLogDirs() throws IOException, YarnException { File absLogDir = new File("target", TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile(); String logdirwithFile = absLogDir.toURI().toString(); Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile); NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf); healthChecker.init(conf); LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler(); NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler, new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf); String user = "nobody"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1); Application app = mock(Application.class); when(app.getUser()).thenReturn(user); when(app.getAppId()).thenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId( appId, 1); ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0); nmContext.getApplications().put(appId, app); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.setState(ContainerState.RUNNING); nmContext.getContainers().put(container1, container); List<File> files = null; files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); // After container is completed, it is removed from nmContext nmContext.getContainers().remove(container1); Assert.assertNull(nmContext.getContainers().get(container1)); files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); Assert.assertTrue(!(files.get(0).toString().contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler); // good log dirs are empty and nm log dir is in the full log dir list. when(dirsHandlerForFullDisk.getLogDirs()). thenReturn(new ArrayList<String>()); when(dirsHandlerForFullDisk.getLogDirsForRead()). thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()})); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager(conf), new NMNullStateStoreService()); nmContext.getApplications().put(appId, app); container.setState(ContainerState.RUNNING); nmContext.getContainers().put(container1, container); List<File> dirs = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext); File containerLogDir = new File(absLogDir, appId + "/" + container1); Assert.assertTrue(dirs.contains(containerLogDir)); }