Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.NodeManager

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.NodeManager. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: spliceengine   Source File: Utils.java    License: GNU Affero General Public License v3.0 6 votes vote down vote up
public static void waitForNMToRegister(NodeManager nm) throws Exception{
    NMTokenSecretManagerInNM nmTokenSecretManagerNM =
            nm.getNMContext().getNMTokenSecretManager();
    NMContainerTokenSecretManager containerTokenSecretManager = nm.getNMContext().getContainerTokenSecretManager();
    int attempt = 60;
    while(attempt-- > 0) {
        try {
            if (nmTokenSecretManagerNM.getCurrentKey() != null && containerTokenSecretManager.getCurrentKey() != null) {
                break;
            }
        } catch (Exception e) {

        }
        Thread.sleep(2000);
    }
}
 
Example 2
Source Project: Flink-CEPplus   Source File: YarnTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static int getRunningContainers() {
	int count = 0;
	for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) {
		NodeManager nm = yarnCluster.getNodeManager(nmId);
		ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers();
		count += containers.size();
	}
	return count;
}
 
Example 3
Source Project: flink   Source File: YarnTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static int getRunningContainers() {
	int count = 0;
	for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) {
		NodeManager nm = yarnCluster.getNodeManager(nmId);
		ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers();
		count += containers.size();
	}
	return count;
}
 
Example 4
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 5 votes vote down vote up
protected void waitForNMToReceiveNMTokenKey(
    NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
    throws InterruptedException {
  int attempt = 60;
  ContainerManagerImpl cm =
      ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
  while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
      .getNodeId() == null) && attempt-- > 0) {
    Thread.sleep(2000);
  }
}
 
Example 5
Source Project: hadoop   Source File: BaseContainerManagerTest.java    License: Apache License 2.0 5 votes vote down vote up
@Before
public void setup() throws IOException {
  localFS.delete(new Path(localDir.getAbsolutePath()), true);
  localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
  localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
  localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
  localDir.mkdir();
  tmpDir.mkdir();
  localLogDir.mkdir();
  remoteLogDir.mkdir();
  LOG.info("Created localDir in " + localDir.getAbsolutePath());
  LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());

  String bindAddress = "0.0.0.0:12345";
  conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());

  conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
  // Default delSrvc
  delSrvc = createDeletionService();
  delSrvc.init(conf);

  exec = createContainerExecutor();
  dirsHandler = new LocalDirsHandlerService();
  nodeHealthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  nodeHealthChecker.init(conf);
  containerManager = createContainerManager(delSrvc);
  ((NMContext)context).setContainerManager(containerManager);
  nodeStatusUpdater.init(conf);
  containerManager.init(conf);
  nodeStatusUpdater.start();
}
 
Example 6
Source Project: hadoop   Source File: TestDistributedShellWithNodeLabels.java    License: Apache License 2.0 5 votes vote down vote up
private void initializeNodeLabels() throws IOException {
  RMContext rmContext = distShellTest.yarnCluster.getResourceManager(0).getRMContext();

  // Setup node labels
  RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
  Set<String> labels = new HashSet<String>();
  labels.add("x");
  labelsMgr.addToCluserNodeLabels(labels);

  // Setup queue access to node labels
  distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x");
  distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity",
      "100");
  distShellTest.conf.set("yarn.scheduler.capacity.root.default.accessible-node-labels", "x");
  distShellTest.conf.set(
      "yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity",
      "100");

  rmContext.getScheduler().reinitialize(distShellTest.conf, rmContext);

  // Fetch node-ids from yarn cluster
  NodeId[] nodeIds = new NodeId[NUM_NMS];
  for (int i = 0; i < NUM_NMS; i++) {
    NodeManager mgr = distShellTest.yarnCluster.getNodeManager(i);
    nodeIds[i] = mgr.getNMContext().getNodeId();
  }

  // Set label x to NM[1]
  labelsMgr.addLabelsToNode(ImmutableMap.of(nodeIds[1], labels));
}
 
Example 7
Source Project: big-c   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 5 votes vote down vote up
protected void waitForNMToReceiveNMTokenKey(
    NMTokenSecretManagerInNM nmTokenSecretManagerNM, NodeManager nm)
    throws InterruptedException {
  int attempt = 60;
  ContainerManagerImpl cm =
      ((ContainerManagerImpl) nm.getNMContext().getContainerManager());
  while ((cm.getBlockNewContainerRequestsStatus() || nmTokenSecretManagerNM
      .getNodeId() == null) && attempt-- > 0) {
    Thread.sleep(2000);
  }
}
 
Example 8
Source Project: big-c   Source File: TestDistributedShellWithNodeLabels.java    License: Apache License 2.0 5 votes vote down vote up
private void initializeNodeLabels() throws IOException {
  RMContext rmContext = distShellTest.yarnCluster.getResourceManager(0).getRMContext();

  // Setup node labels
  RMNodeLabelsManager labelsMgr = rmContext.getNodeLabelManager();
  Set<String> labels = new HashSet<String>();
  labels.add("x");
  labelsMgr.addToCluserNodeLabels(labels);

  // Setup queue access to node labels
  distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels", "x");
  distShellTest.conf.set("yarn.scheduler.capacity.root.accessible-node-labels.x.capacity",
      "100");
  distShellTest.conf.set("yarn.scheduler.capacity.root.default.accessible-node-labels", "x");
  distShellTest.conf.set(
      "yarn.scheduler.capacity.root.default.accessible-node-labels.x.capacity",
      "100");

  rmContext.getScheduler().reinitialize(distShellTest.conf, rmContext);

  // Fetch node-ids from yarn cluster
  NodeId[] nodeIds = new NodeId[NUM_NMS];
  for (int i = 0; i < NUM_NMS; i++) {
    NodeManager mgr = distShellTest.yarnCluster.getNodeManager(i);
    nodeIds[i] = mgr.getNMContext().getNodeId();
  }

  // Set label x to NM[1]
  labelsMgr.addLabelsToNode(ImmutableMap.of(nodeIds[1], labels));
}
 
Example 9
Source Project: flink   Source File: YarnTestBase.java    License: Apache License 2.0 5 votes vote down vote up
public static int getRunningContainers() {
	int count = 0;
	for (int nmId = 0; nmId < NUM_NODEMANAGERS; nmId++) {
		NodeManager nm = yarnCluster.getNodeManager(nmId);
		ConcurrentMap<ContainerId, Container> containers = nm.getNMContext().getContainers();
		count += containers.size();
	}
	return count;
}
 
Example 10
Source Project: hadoop   Source File: MiniYARNCluster.java    License: Apache License 2.0 4 votes vote down vote up
public NodeManager getNodeManager(int i) {
  return this.nodeManagers[i];
}
 
Example 11
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests a malice user getting a proper token but then messing with it by
 * tampering with containerID/Resource etc.. His/her containers should be
 * rejected.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerToken(Configuration conf) throws IOException,
    InterruptedException, YarnException {

  LOG.info("Running test for malice user");
  /*
   * We need to check for containerToken (authorization).
   * Here we will be assuming that we have valid NMToken  
   * 1) ContainerToken used is expired.
   * 2) ContainerToken is tampered (resource is modified).
   */
  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
        .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";
  
  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();
  
  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
  
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  
  Resource r = Resource.newInstance(1230, 2, 2);
  
  Token containerToken = 
      containerTokenSecretManager.createContainerToken(
          cId, nodeId, user, r, Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier = 
      getContainerTokenIdentifierFromToken(containerToken);
  
  // Verify new compatible version ContainerTokenIdentifier can work successfully.
  ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
      new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message");
  byte[] password = 
      containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
  
  Token newContainerToken = BuilderUtils.newContainerToken(
      nodeId, password, newVersionTokenIdentifier);
  
  Token nmToken =
          nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
  YarnRPC rpc = YarnRPC.create(conf);
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      newContainerToken, nmToken, false).isEmpty());
  
  // Creating a tampered Container Token
  RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  tamperedContainerTokenSecretManager.rollMasterKey();
  do {
    tamperedContainerTokenSecretManager.rollMasterKey();
    tamperedContainerTokenSecretManager.activateNextMasterKey();
  } while (containerTokenSecretManager.getCurrentKey().getKeyId()
      == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
  
  ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
  // Creating modified containerToken
  Token containerToken2 =
      tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
          user, r, Priority.newInstance(0), 0);
  
  StringBuilder sb = new StringBuilder("Given Container ");
  sb.append(cId2);
  sb.append(" seems to have an illegally generated token.");
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      containerToken2, nmToken, true).contains(sb.toString()));
}
 
Example 12
Source Project: hadoop   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests whether a containerId is serialized/deserialized with epoch.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerTokenWithEpoch(Configuration conf)
    throws IOException, InterruptedException, YarnException {

  LOG.info("Running test for serializing/deserializing containerIds");

  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
          .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";

  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();

  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());

  // Creating a normal Container Token
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  Resource r = Resource.newInstance(1230, 2);
  Token containerToken =
      containerTokenSecretManager.createContainerToken(cId, nodeId, user, r,
          Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier();
  byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
  containerTokenIdentifier.readFields(dib);
  
  
  Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
  Assert.assertEquals(
      cId.toString(), containerTokenIdentifier.getContainerID().toString());

  Token nmToken =
      nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);

  YarnRPC rpc = YarnRPC.create(conf);
  testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken,
      false);

  List<ContainerId> containerIds = new LinkedList<ContainerId>();
  containerIds.add(cId);
  ContainerManagementProtocol proxy
      = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
  GetContainerStatusesResponse res = proxy.getContainerStatuses(
      GetContainerStatusesRequest.newInstance(containerIds));
  Assert.assertNotNull(res.getContainerStatuses().get(0));
  Assert.assertEquals(
      cId, res.getContainerStatuses().get(0).getContainerId());
  Assert.assertEquals(cId.toString(),
      res.getContainerStatuses().get(0).getContainerId().toString());
}
 
Example 13
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceStart() throws Exception {

  // Enqueue user dirs in deletion context

  Configuration conf = getConfig();
  final InetSocketAddress initialAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_PORT);
  boolean usingEphemeralPort = (initialAddress.getPort() == 0);
  if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
    throw new IllegalArgumentException("Cannot support recovery with an "
        + "ephemeral server port. Check the setting of "
        + YarnConfiguration.NM_ADDRESS);
  }
  // If recovering then delay opening the RPC service until the recovery
  // of resources and containers have completed, otherwise requests from
  // clients during recovery can interfere with the recovery process.
  final boolean delayedRpcServerStart =
      context.getNMStateStore().canRecover();

  Configuration serverConf = new Configuration(conf);

  // always enforce it to be token-based.
  serverConf.set(
    CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    SaslRpcServer.AuthMethod.TOKEN.toString());
  
  YarnRPC rpc = YarnRPC.create(conf);

  server =
      rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
          serverConf, this.context.getNMTokenSecretManager(),
          conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new NMPolicyProvider());
  }
  
  LOG.info("Blocking new container-requests as container manager rpc" +
  		" server is still starting.");
  this.setBlockNewContainerRequests(true);

  String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
  String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
  String hostOverride = null;
  if (bindHost != null && !bindHost.isEmpty()
      && nmAddress != null && !nmAddress.isEmpty()) {
    //a bind-host case with an address, to support overriding the first
    //hostname found when querying for our hostname with the specified
    //address, combine the specified address with the actual port listened
    //on by the server
    hostOverride = nmAddress.split(":")[0];
  }

  // setup node ID
  InetSocketAddress connectAddress;
  if (delayedRpcServerStart) {
    connectAddress = NetUtils.getConnectAddress(initialAddress);
  } else {
    server.start();
    connectAddress = NetUtils.getConnectAddress(server);
  }
  NodeId nodeId = buildNodeId(connectAddress, hostOverride);
  ((NodeManager.NMContext)context).setNodeId(nodeId);
  this.context.getNMTokenSecretManager().setNodeId(nodeId);
  this.context.getContainerTokenSecretManager().setNodeId(nodeId);

  // start remaining services
  super.serviceStart();

  if (delayedRpcServerStart) {
    waitForRecoveredContainers();
    server.start();

    // check that the node ID is as previously advertised
    connectAddress = NetUtils.getConnectAddress(server);
    NodeId serverNode = buildNodeId(connectAddress, hostOverride);
    if (!serverNode.equals(nodeId)) {
      throw new IOException("Node mismatch after server started, expected '"
          + nodeId + "' but found '" + serverNode + "'");
    }
  }

  LOG.info("ContainerManager started at " + connectAddress);
  LOG.info("ContainerManager bound to " + initialAddress);
}
 
Example 14
Source Project: hadoop   Source File: TestContainerLogsPage.java    License: Apache License 2.0 4 votes vote down vote up
private NodeHealthCheckerService createNodeHealthCheckerService(Configuration conf) {
  NodeHealthScriptRunner scriptRunner = NodeManager.getNodeHealthScriptRunner(conf);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  return new NodeHealthCheckerService(scriptRunner, dirsHandler);
}
 
Example 15
Source Project: hadoop   Source File: TestContainerLogsPage.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=30000)
public void testContainerLogDirs() throws IOException, YarnException {
  File absLogDir = new File("target",
    TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
  String logdirwithFile = absLogDir.toURI().toString();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
  NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      new ApplicationACLsManager(conf), new NMNullStateStoreService());
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  String user = "nobody";
  long clusterTimeStamp = 1234;
  ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
      clusterTimeStamp, 1);
  Application app = mock(Application.class);
  when(app.getUser()).thenReturn(user);
  when(app.getAppId()).thenReturn(appId);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
      appAttemptId, 0);
  nmContext.getApplications().put(appId, app);

  MockContainer container =
      new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
          appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);   
  List<File> files = null;
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(container1);
  Assert.assertNull(nmContext.getContainers().get(container1));
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));

  // Create a new context to check if correct container log dirs are fetched
  // on full disk.
  LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler);
  // good log dirs are empty and nm log dir is in the full log dir list.
  when(dirsHandlerForFullDisk.getLogDirs()).
      thenReturn(new ArrayList<String>());
  when(dirsHandlerForFullDisk.getLogDirsForRead()).
      thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()}));
  nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk,
      new ApplicationACLsManager(conf), new NMNullStateStoreService());
  nmContext.getApplications().put(appId, app);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);
  List<File> dirs =
      ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  File containerLogDir = new File(absLogDir, appId + "/" + container1);
  Assert.assertTrue(dirs.contains(containerLogDir));
}
 
Example 16
Source Project: hadoop   Source File: TestNMWebServicesApps.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}
 
Example 17
Source Project: hadoop   Source File: TestNMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}
 
Example 18
Source Project: hadoop   Source File: TestNMWebServer.java    License: Apache License 2.0 4 votes vote down vote up
private NodeHealthCheckerService createNodeHealthCheckerService(Configuration conf) {
  NodeHealthScriptRunner scriptRunner = NodeManager.getNodeHealthScriptRunner(conf);
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  return new NodeHealthCheckerService(scriptRunner, dirsHandler);
}
 
Example 19
Source Project: hadoop   Source File: TestNMWebServer.java    License: Apache License 2.0 4 votes vote down vote up
private int startNMWebAppServer(String webAddr) {
  Context nmContext = new NodeManager.NMContext(null, null, null, null,
      null);
  ResourceView resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getPmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getGCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
  WebServer server = new WebServer(nmContext, resourceView,
      new ApplicationACLsManager(conf), dirsHandler);
  try {
    server.init(conf);
    server.start();
    return server.getPort();
  } finally {
    server.stop();
    healthChecker.stop();
  }
}
 
Example 20
Source Project: hadoop   Source File: TestNMWebServer.java    License: Apache License 2.0 4 votes vote down vote up
@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getGCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken)) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }
 
Example 21
Source Project: hadoop   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public long getGCoresAllocatedForContainers() {
      return new Long("4000");
    }

    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(
      NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null) {
    public NodeId getNodeId() {
      return NodeId.newInstance("testhost.foo.com", 8042);
    };

    public int getHttpPort() {
      return 1234;
    };
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}
 
Example 22
Source Project: big-c   Source File: MiniYARNCluster.java    License: Apache License 2.0 4 votes vote down vote up
public NodeManager getNodeManager(int i) {
  return this.nodeManagers[i];
}
 
Example 23
Source Project: big-c   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests a malice user getting a proper token but then messing with it by
 * tampering with containerID/Resource etc.. His/her containers should be
 * rejected.
 * 
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerToken(Configuration conf) throws IOException,
    InterruptedException, YarnException {

  LOG.info("Running test for malice user");
  /*
   * We need to check for containerToken (authorization).
   * Here we will be assuming that we have valid NMToken  
   * 1) ContainerToken used is expired.
   * 2) ContainerToken is tampered (resource is modified).
   */
  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
        .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";
  
  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();
  
  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());
  
  
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  
  Resource r = Resource.newInstance(1230, 2);
  
  Token containerToken = 
      containerTokenSecretManager.createContainerToken(
          cId, nodeId, user, r, Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier = 
      getContainerTokenIdentifierFromToken(containerToken);
  
  // Verify new compatible version ContainerTokenIdentifier can work successfully.
  ContainerTokenIdentifierForTest newVersionTokenIdentifier = 
      new ContainerTokenIdentifierForTest(containerTokenIdentifier, "message");
  byte[] password = 
      containerTokenSecretManager.createPassword(newVersionTokenIdentifier);
  
  Token newContainerToken = BuilderUtils.newContainerToken(
      nodeId, password, newVersionTokenIdentifier);
  
  Token nmToken =
          nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);
  YarnRPC rpc = YarnRPC.create(conf);
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      newContainerToken, nmToken, false).isEmpty());
  
  // Creating a tampered Container Token
  RMContainerTokenSecretManager tamperedContainerTokenSecretManager =
      new RMContainerTokenSecretManager(conf);
  tamperedContainerTokenSecretManager.rollMasterKey();
  do {
    tamperedContainerTokenSecretManager.rollMasterKey();
    tamperedContainerTokenSecretManager.activateNextMasterKey();
  } while (containerTokenSecretManager.getCurrentKey().getKeyId()
      == tamperedContainerTokenSecretManager.getCurrentKey().getKeyId());
  
  ContainerId cId2 = ContainerId.newContainerId(appAttemptId, 1);
  // Creating modified containerToken
  Token containerToken2 =
      tamperedContainerTokenSecretManager.createContainerToken(cId2, nodeId,
          user, r, Priority.newInstance(0), 0);
  
  StringBuilder sb = new StringBuilder("Given Container ");
  sb.append(cId2);
  sb.append(" seems to have an illegally generated token.");
  Assert.assertTrue(testStartContainer(rpc, appAttemptId, nodeId,
      containerToken2, nmToken, true).contains(sb.toString()));
}
 
Example 24
Source Project: big-c   Source File: TestContainerManagerSecurity.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * This tests whether a containerId is serialized/deserialized with epoch.
 *
 * @throws IOException
 * @throws InterruptedException
 * @throws YarnException
 */
private void testContainerTokenWithEpoch(Configuration conf)
    throws IOException, InterruptedException, YarnException {

  LOG.info("Running test for serializing/deserializing containerIds");

  NMTokenSecretManagerInRM nmTokenSecretManagerInRM =
      yarnCluster.getResourceManager().getRMContext()
          .getNMTokenSecretManager();
  ApplicationId appId = ApplicationId.newInstance(1, 1);
  ApplicationAttemptId appAttemptId =
      ApplicationAttemptId.newInstance(appId, 0);
  ContainerId cId = ContainerId.newContainerId(appAttemptId, (5L << 40) | 3L);
  NodeManager nm = yarnCluster.getNodeManager(0);
  NMTokenSecretManagerInNM nmTokenSecretManagerInNM =
      nm.getNMContext().getNMTokenSecretManager();
  String user = "test";

  waitForNMToReceiveNMTokenKey(nmTokenSecretManagerInNM, nm);

  NodeId nodeId = nm.getNMContext().getNodeId();

  // Both id should be equal.
  Assert.assertEquals(nmTokenSecretManagerInNM.getCurrentKey().getKeyId(),
      nmTokenSecretManagerInRM.getCurrentKey().getKeyId());

  // Creating a normal Container Token
  RMContainerTokenSecretManager containerTokenSecretManager =
      yarnCluster.getResourceManager().getRMContext().
          getContainerTokenSecretManager();
  Resource r = Resource.newInstance(1230, 2);
  Token containerToken =
      containerTokenSecretManager.createContainerToken(cId, nodeId, user, r,
          Priority.newInstance(0), 0);
  
  ContainerTokenIdentifier containerTokenIdentifier =
      new ContainerTokenIdentifier();
  byte[] tokenIdentifierContent = containerToken.getIdentifier().array();
  DataInputBuffer dib = new DataInputBuffer();
  dib.reset(tokenIdentifierContent, tokenIdentifierContent.length);
  containerTokenIdentifier.readFields(dib);
  
  
  Assert.assertEquals(cId, containerTokenIdentifier.getContainerID());
  Assert.assertEquals(
      cId.toString(), containerTokenIdentifier.getContainerID().toString());

  Token nmToken =
      nmTokenSecretManagerInRM.createNMToken(appAttemptId, nodeId, user);

  YarnRPC rpc = YarnRPC.create(conf);
  testStartContainer(rpc, appAttemptId, nodeId, containerToken, nmToken,
      false);

  List<ContainerId> containerIds = new LinkedList<ContainerId>();
  containerIds.add(cId);
  ContainerManagementProtocol proxy
      = getContainerManagementProtocolProxy(rpc, nmToken, nodeId, user);
  GetContainerStatusesResponse res = proxy.getContainerStatuses(
      GetContainerStatusesRequest.newInstance(containerIds));
  Assert.assertNotNull(res.getContainerStatuses().get(0));
  Assert.assertEquals(
      cId, res.getContainerStatuses().get(0).getContainerId());
  Assert.assertEquals(cId.toString(),
      res.getContainerStatuses().get(0).getContainerId().toString());
}
 
Example 25
Source Project: big-c   Source File: ContainerManagerImpl.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void serviceStart() throws Exception {

  // Enqueue user dirs in deletion context

  Configuration conf = getConfig();
  final InetSocketAddress initialAddress = conf.getSocketAddr(
      YarnConfiguration.NM_BIND_HOST,
      YarnConfiguration.NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_ADDRESS,
      YarnConfiguration.DEFAULT_NM_PORT);
  boolean usingEphemeralPort = (initialAddress.getPort() == 0);
  if (context.getNMStateStore().canRecover() && usingEphemeralPort) {
    throw new IllegalArgumentException("Cannot support recovery with an "
        + "ephemeral server port. Check the setting of "
        + YarnConfiguration.NM_ADDRESS);
  }
  // If recovering then delay opening the RPC service until the recovery
  // of resources and containers have completed, otherwise requests from
  // clients during recovery can interfere with the recovery process.
  final boolean delayedRpcServerStart =
      context.getNMStateStore().canRecover();

  Configuration serverConf = new Configuration(conf);

  // always enforce it to be token-based.
  serverConf.set(
    CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
    SaslRpcServer.AuthMethod.TOKEN.toString());
  
  YarnRPC rpc = YarnRPC.create(conf);

  server =
      rpc.getServer(ContainerManagementProtocol.class, this, initialAddress, 
          serverConf, this.context.getNMTokenSecretManager(),
          conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT, 
              YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
  
  // Enable service authorization?
  if (conf.getBoolean(
      CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, 
      false)) {
    refreshServiceAcls(conf, new NMPolicyProvider());
  }
  
  LOG.info("Blocking new container-requests as container manager rpc" +
  		" server is still starting.");
  this.setBlockNewContainerRequests(true);

  String bindHost = conf.get(YarnConfiguration.NM_BIND_HOST);
  String nmAddress = conf.getTrimmed(YarnConfiguration.NM_ADDRESS);
  String hostOverride = null;
  if (bindHost != null && !bindHost.isEmpty()
      && nmAddress != null && !nmAddress.isEmpty()) {
    //a bind-host case with an address, to support overriding the first
    //hostname found when querying for our hostname with the specified
    //address, combine the specified address with the actual port listened
    //on by the server
    hostOverride = nmAddress.split(":")[0];
  }

  // setup node ID
  InetSocketAddress connectAddress;
  if (delayedRpcServerStart) {
    connectAddress = NetUtils.getConnectAddress(initialAddress);
  } else {
    server.start();
    connectAddress = NetUtils.getConnectAddress(server);
  }
  NodeId nodeId = buildNodeId(connectAddress, hostOverride);
  ((NodeManager.NMContext)context).setNodeId(nodeId);
  this.context.getNMTokenSecretManager().setNodeId(nodeId);
  this.context.getContainerTokenSecretManager().setNodeId(nodeId);

  // start remaining services
  super.serviceStart();

  if (delayedRpcServerStart) {
    waitForRecoveredContainers();
    server.start();

    // check that the node ID is as previously advertised
    connectAddress = NetUtils.getConnectAddress(server);
    NodeId serverNode = buildNodeId(connectAddress, hostOverride);
    if (!serverNode.equals(nodeId)) {
      throw new IOException("Node mismatch after server started, expected '"
          + nodeId + "' but found '" + serverNode + "'");
    }
  }

  LOG.info("ContainerManager started at " + connectAddress);
  LOG.info("ContainerManager bound to " + initialAddress);
}
 
Example 26
Source Project: big-c   Source File: TestContainerLogsPage.java    License: Apache License 2.0 4 votes vote down vote up
@Test(timeout=30000)
public void testContainerLogDirs() throws IOException, YarnException {
  File absLogDir = new File("target",
    TestNMWebServer.class.getSimpleName() + "LogDir").getAbsoluteFile();
  String logdirwithFile = absLogDir.toURI().toString();
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOG_DIRS, logdirwithFile);
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  // Add an application and the corresponding containers
  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
  String user = "nobody";
  long clusterTimeStamp = 1234;
  ApplicationId appId = BuilderUtils.newApplicationId(recordFactory,
      clusterTimeStamp, 1);
  Application app = mock(Application.class);
  when(app.getUser()).thenReturn(user);
  when(app.getAppId()).thenReturn(appId);
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      appId, 1);
  ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId,
      appAttemptId, 0);
  nmContext.getApplications().put(appId, app);

  MockContainer container =
      new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user,
          appId, 1);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);   
  List<File> files = null;
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));
  
  // After container is completed, it is removed from nmContext
  nmContext.getContainers().remove(container1);
  Assert.assertNull(nmContext.getContainers().get(container1));
  files = ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  Assert.assertTrue(!(files.get(0).toString().contains("file:")));

  // Create a new context to check if correct container log dirs are fetched
  // on full disk.
  LocalDirsHandlerService dirsHandlerForFullDisk = spy(dirsHandler);
  // good log dirs are empty and nm log dir is in the full log dir list.
  when(dirsHandlerForFullDisk.getLogDirs()).
      thenReturn(new ArrayList<String>());
  when(dirsHandlerForFullDisk.getLogDirsForRead()).
      thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()}));
  nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk,
      new ApplicationACLsManager(conf), new NMNullStateStoreService(), null);
  nmContext.getApplications().put(appId, app);
  container.setState(ContainerState.RUNNING);
  nmContext.getContainers().put(container1, container);
  List<File> dirs =
      ContainerLogsUtils.getContainerLogDirs(container1, user, nmContext);
  File containerLogDir = new File(absLogDir, appId + "/" + container1);
  Assert.assertTrue(dirs.contains(containerLogDir));
}
 
Example 27
Source Project: big-c   Source File: TestNMWebServicesApps.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }

    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }


    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }

    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}
 
Example 28
Source Project: big-c   Source File: TestNMWebServices.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected void configureServlets() {
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  dirsHandler = healthChecker.getDiskHandler();
  aclsManager = new ApplicationACLsManager(conf);
  nmContext = new NodeManager.NMContext(null, null, dirsHandler,
      aclsManager, null, null);
  NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
  ((NodeManager.NMContext)nmContext).setNodeId(nodeId);
  resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      // 15.5G in bytes
      return new Long("16642998272");
    }

    @Override
    public long getPmemAllocatedForContainers() {
      // 16G in bytes
      return new Long("17179869184");
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return new Long("4000");
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  nmWebApp = new NMWebApp(resourceView, aclsManager, dirsHandler);
  bind(JAXBContextResolver.class);
  bind(NMWebServices.class);
  bind(GenericExceptionHandler.class);
  bind(Context.class).toInstance(nmContext);
  bind(WebApp.class).toInstance(nmWebApp);
  bind(ResourceView.class).toInstance(resourceView);
  bind(ApplicationACLsManager.class).toInstance(aclsManager);
  bind(LocalDirsHandlerService.class).toInstance(dirsHandler);

  serve("/*").with(GuiceContainer.class);
}
 
Example 29
Source Project: big-c   Source File: TestNMWebServer.java    License: Apache License 2.0 4 votes vote down vote up
private int startNMWebAppServer(String webAddr) {
  Context nmContext = new NodeManager.NMContext(null, null, null, null,
      null, null);
  ResourceView resourceView = new ResourceView() {
    @Override
    public long getVmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getPmemAllocatedForContainers() {
      return 0;
    }
    @Override
    public long getVCoresAllocatedForContainers() {
      return 0;
    }
    @Override
    public boolean isVmemCheckEnabled() {
      return true;
    }
    @Override
    public boolean isPmemCheckEnabled() {
      return true;
    }
  };
  Configuration conf = new Configuration();
  conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
  conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
  NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
  healthChecker.init(conf);
  LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
  conf.set(YarnConfiguration.NM_WEBAPP_ADDRESS, webAddr);
  WebServer server = new WebServer(nmContext, resourceView,
      new ApplicationACLsManager(conf), dirsHandler);
  try {
    server.init(conf);
    server.start();
    return server.getPort();
  } finally {
    server.stop();
    healthChecker.stop();
  }
}
 
Example 30
Source Project: big-c   Source File: TestNMWebServer.java    License: Apache License 2.0 4 votes vote down vote up
@Test
  public void testNMWebApp() throws IOException, YarnException {
    Context nmContext = new NodeManager.NMContext(null, null, null, null,
        null, null);
    ResourceView resourceView = new ResourceView() {
      @Override
      public long getVmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getPmemAllocatedForContainers() {
        return 0;
      }
      @Override
      public long getVCoresAllocatedForContainers() {
        return 0;
      }
      @Override
      public boolean isVmemCheckEnabled() {
        return true;
      }
      @Override
      public boolean isPmemCheckEnabled() {
        return true;
      }
    };
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();

    WebServer server = new WebServer(nmContext, resourceView,
        new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();

    // Add an application and the corresponding containers
    RecordFactory recordFactory =
        RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId =
        BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
        appId, 1);
    ContainerId container1 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 =
        BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1,
        container2}) {
      // TODO: Use builder utils
      ContainerLaunchContext launchContext =
          recordFactory.newRecordInstance(ContainerLaunchContext.class);
      long currentTime = System.currentTimeMillis();
      Token containerToken =
          BuilderUtils.newContainerToken(containerId, "127.0.0.1", 1234, user,
            BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123,
            "password".getBytes(), currentTime);
      Container container =
          new ContainerImpl(nmContext, conf, dispatcher, stateStore, launchContext,
            null, metrics,
            BuilderUtils.newContainerTokenIdentifier(containerToken), null, 0, user, false, null) {

            @Override
            public ContainerState getContainerState() {
              return ContainerState.RUNNING;
            };
          };
      nmContext.getContainers().put(containerId, container);
      //TODO: Gross hack. Fix in code.
      ApplicationId applicationId = 
          containerId.getApplicationAttemptId().getApplicationId();
      nmContext.getApplications().get(applicationId).getContainers()
          .put(containerId, container);
      writeContainerLogs(nmContext, containerId, dirsHandler);

    }
    // TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
  }