Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestLinuxContainerExecutor.java    License: Apache License 2.0 6 votes vote down vote up
private int runAndBlock(ContainerId cId, String ... cmd) throws IOException {
  String appId = "APP_"+getNextId();
  Container container = mock(Container.class);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);

  when(context.getEnvironment()).thenReturn(env);
  
  String script = writeScriptFile(cmd);

  Path scriptPath = new Path(script);
  Path tokensPath = new Path("/dev/null");
  Path workDir = new Path(workSpace.getAbsolutePath());
  Path pidFile = new Path(workDir, "pid.txt");

  exec.activateContainer(cId, pidFile);
  return exec.launchContainer(container, scriptPath, tokensPath,
      appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
      dirsHandler.getLogDirs());
}
 
Example 2
Source Project: hadoop   Source File: AllContainersPage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  TBODY<TABLE<BODY<Hamlet>>> tableBody = html.body()
    .table("#containers")
      .thead()
        .tr()
          .td()._("ContainerId")._()
          .td()._("ContainerState")._()
          .td()._("logs")._()
        ._()
      ._().tbody();
  for (Entry<ContainerId, Container> entry : this.nmContext
      .getContainers().entrySet()) {
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue());
    tableBody
      .tr()
        .td().a(url("container", info.getId()), info.getId())
        ._()
        .td()._(info.getState())._()
        .td()
            .a(url(info.getShortLogLink()), "logs")._()
      ._();
  }
  tableBody._()._()._();
}
 
Example 3
Source Project: hadoop   Source File: ContainerLogsUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Finds the local directories that logs for the given container are stored
 * on.
 */
public static List<File> getContainerLogDirs(ContainerId containerId,
    String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);

  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  // It is not required to have null check for container ( container == null )
  // and throw back exception.Because when container is completed, NodeManager
  // remove container information from its NMContext.Configuring log
  // aggregation to false, container log view request is forwarded to NM. NM
  // does not have completed container information,but still NM serve request for
  // reading container logs. 
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  return getContainerLogDirs(containerId, context.getLocalDirsHandler());
}
 
Example 4
Source Project: hadoop   Source File: NMWebServices.java    License: Apache License 2.0 6 votes vote down vote up
@GET
@Path("/containers")
@Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML })
public ContainersInfo getNodeContainers() {
  init();
  ContainersInfo allContainers = new ContainersInfo();
  for (Entry<ContainerId, Container> entry : this.nmContext.getContainers()
      .entrySet()) {
    if (entry.getValue() == null) {
      // just skip it
      continue;
    }
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue(),
        uriInfo.getBaseUri().toString(), webapp.name());
    allContainers.add(info);
  }
  return allContainers;
}
 
Example 5
Source Project: hadoop   Source File: ResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * For each of the requested resources for a container, determines the
 * appropriate {@link LocalResourcesTracker} and forwards a 
 * {@link LocalResourceRequest} to that tracker.
 */
private void handleInitContainerResources(
    ContainerLocalizationRequestEvent rsrcReqs) {
  Container c = rsrcReqs.getContainer();
  // create a loading cache for the file statuses
  LoadingCache<Path,Future<FileStatus>> statCache =
      CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
  LocalizerContext ctxt = new LocalizerContext(
      c.getUser(), c.getContainerId(), c.getCredentials(), statCache);
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
    rsrcReqs.getRequestedResources();
  for (Map.Entry<LocalResourceVisibility, Collection<LocalResourceRequest>> e :
       rsrcs.entrySet()) {
    LocalResourcesTracker tracker =
        getLocalResourcesTracker(e.getKey(), c.getUser(),
            c.getContainerId().getApplicationAttemptId()
                .getApplicationId());
    for (LocalResourceRequest req : e.getValue()) {
      tracker.handle(new ResourceRequestEvent(req, e.getKey(), ctxt));
    }
  }
}
 
Example 6
Source Project: big-c   Source File: DummyContainerManager.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@SuppressWarnings("unchecked")
protected ContainersLauncher createContainersLauncher(Context context,
    ContainerExecutor exec) {
  return new ContainersLauncher(context, super.dispatcher, exec,
                                super.dirsHandler, this) {
    @Override
    public void handle(ContainersLauncherEvent event) {
      Container container = event.getContainer();
      ContainerId containerId = container.getContainerId();
      switch (event.getType()) {
      case LAUNCH_CONTAINER:
        dispatcher.getEventHandler().handle(
            new ContainerEvent(containerId,
                ContainerEventType.CONTAINER_LAUNCHED));
        break;
      case CLEANUP_CONTAINER:
        dispatcher.getEventHandler().handle(
            new ContainerExitEvent(containerId,
                ContainerEventType.CONTAINER_KILLED_ON_REQUEST, 0,
                "Container exited with exit code 0."));
        break;
      }
    }
  };
}
 
Example 7
Source Project: big-c   Source File: NodeStatusUpdaterImpl.java    License: Apache License 2.0 6 votes vote down vote up
private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
  List<NMContainerStatus> containerStatuses =
      new ArrayList<NMContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    if (!this.context.getApplications().containsKey(applicationId)) {
      context.getContainers().remove(containerId);
      continue;
    }
    NMContainerStatus status =
        container.getNMContainerStatus();
    containerStatuses.add(status);
    if (status.getContainerState() == ContainerState.COMPLETE) {
      // Adding to finished containers cache. Cache will keep it around at
      // least for #durationToTrackStoppedContainers duration. In the
      // subsequent call to stop container it will get removed from cache.
      addCompletedContainer(containerId);
    }
  }
  LOG.info("Sending out " + containerStatuses.size()
    + " NM container statuses: " + containerStatuses);
  return containerStatuses;
}
 
Example 8
Source Project: hadoop   Source File: DockerLinuxContainerRuntime.java    License: Apache License 2.0 6 votes vote down vote up
private void executeDockerLoadCommand(DockerLoadCommand cmd,
    ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  String containerIdStr = container.getContainerId().toString();
  String commandFile = dockerClient.writeCommandToTempFile(cmd,
      containerIdStr);
  PrivilegedOperation launchOp = new PrivilegedOperation(
      PrivilegedOperation.OperationType.RUN_DOCKER_CMD);

  launchOp.appendArgs(commandFile);

  try {
    privilegedOperationExecutor.executePrivilegedOperation(null,
        launchOp, null, container.getLaunchContext().getEnvironment(),
        false);
  } catch (PrivilegedOperationException e) {
    LOG.warn("Docker load operation failed. Exception: ", e);
    throw new ContainerExecutionException("Docker load operation failed", e
        .getExitCode(), e.getOutput(), e.getErrorOutput());
  }
}
 
Example 9
Source Project: big-c   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example 10
Source Project: hadoop   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);

  this.olr = new OwnLocalResources();
}
 
Example 11
Source Project: hadoop   Source File: ContainerManagerImpl.java    License: Apache License 2.0 6 votes vote down vote up
private void waitForRecoveredContainers() throws InterruptedException {
  final int sleepMsec = 100;
  int waitIterations = 100;
  List<ContainerId> newContainers = new ArrayList<ContainerId>();
  while (--waitIterations >= 0) {
    newContainers.clear();
    for (Container container : context.getContainers().values()) {
      if (container.getContainerState() == org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.NEW) {
        newContainers.add(container.getContainerId());
      }
    }
    if (newContainers.isEmpty()) {
      break;
    }
    LOG.info("Waiting for containers: " + newContainers);
    Thread.sleep(sleepMsec);
  }
  if (waitIterations < 0) {
    LOG.warn("Timeout waiting for recovered containers");
  }
}
 
Example 12
Source Project: big-c   Source File: TestNMWebServicesApps.java    License: Apache License 2.0 6 votes vote down vote up
private HashMap<String, String> addAppContainers(Application app) 
    throws IOException {
  Dispatcher dispatcher = new AsyncDispatcher();
  ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
      app.getAppId(), 1);
  Container container1 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 1);
  Container container2 = new MockContainer(appAttemptId, dispatcher, conf,
      app.getUser(), app.getAppId(), 2);
  nmContext.getContainers()
      .put(container1.getContainerId(), container1);
  nmContext.getContainers()
      .put(container2.getContainerId(), container2);

  app.getContainers().put(container1.getContainerId(), container1);
  app.getContainers().put(container2.getContainerId(), container2);
  HashMap<String, String> hash = new HashMap<String, String>();
  hash.put(container1.getContainerId().toString(), container1
      .getContainerId().toString());
  hash.put(container2.getContainerId().toString(), container2
      .getContainerId().toString());
  return hash;
}
 
Example 13
Source Project: big-c   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 6 votes vote down vote up
public void verifyContainersInfoXML(NodeList nodes, Container cont)
    throws JSONException, Exception {
  for (int i = 0; i < nodes.getLength(); i++) {
    Element element = (Element) nodes.item(i);

    verifyNodeContainerInfoGeneric(cont,
        WebServicesTestUtils.getXmlString(element, "id"),
        WebServicesTestUtils.getXmlString(element, "state"),
        WebServicesTestUtils.getXmlString(element, "user"),
        WebServicesTestUtils.getXmlInt(element, "exitCode"),
        WebServicesTestUtils.getXmlString(element, "diagnostics"),
        WebServicesTestUtils.getXmlString(element, "nodeId"),
        WebServicesTestUtils.getXmlInt(element, "totalMemoryNeededMB"),
        WebServicesTestUtils.getXmlInt(element, "totalVCoresNeeded"),
        WebServicesTestUtils.getXmlString(element, "containerLogsLink"));
  }
}
 
Example 14
Source Project: big-c   Source File: ContainerLaunch.java    License: Apache License 2.0 6 votes vote down vote up
public ContainerLaunch(Context context, Configuration configuration,
    Dispatcher dispatcher, ContainerExecutor exec, Application app,
    Container container, LocalDirsHandlerService dirsHandler,
    ContainerManagerImpl containerManager) {
  this.context = context;
  this.conf = configuration;
  this.app = app;
  this.exec = exec;
  this.container = container;
  this.dispatcher = dispatcher;
  this.dirsHandler = dirsHandler;
  this.containerManager = containerManager;
  this.sleepDelayBeforeSigKill =
      conf.getLong(YarnConfiguration.NM_SLEEP_DELAY_BEFORE_SIGKILL_MS,
          YarnConfiguration.DEFAULT_NM_SLEEP_DELAY_BEFORE_SIGKILL_MS);
  this.maxKillWaitTime =
      conf.getLong(YarnConfiguration.NM_PROCESS_KILL_WAIT_MS,
          YarnConfiguration.DEFAULT_NM_PROCESS_KILL_WAIT_MS);
}
 
Example 15
Source Project: hadoop   Source File: NodeStatusUpdaterImpl.java    License: Apache License 2.0 6 votes vote down vote up
private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
  List<NMContainerStatus> containerStatuses =
      new ArrayList<NMContainerStatus>();
  for (Container container : this.context.getContainers().values()) {
    ContainerId containerId = container.getContainerId();
    ApplicationId applicationId = containerId.getApplicationAttemptId()
        .getApplicationId();
    if (!this.context.getApplications().containsKey(applicationId)) {
      context.getContainers().remove(containerId);
      continue;
    }
    NMContainerStatus status =
        container.getNMContainerStatus();
    containerStatuses.add(status);
    if (status.getContainerState() == ContainerState.COMPLETE) {
      // Adding to finished containers cache. Cache will keep it around at
      // least for #durationToTrackStoppedContainers duration. In the
      // subsequent call to stop container it will get removed from cache.
      addCompletedContainer(containerId);
    }
  }
  LOG.info("Sending out " + containerStatuses.size()
    + " NM container statuses: " + containerStatuses);
  return containerStatuses;
}
 
Example 16
Source Project: big-c   Source File: AllContainersPage.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void render(Block html) {
  TBODY<TABLE<BODY<Hamlet>>> tableBody = html.body()
    .table("#containers")
      .thead()
        .tr()
          .td()._("ContainerId")._()
          .td()._("ContainerState")._()
          .td()._("logs")._()
        ._()
      ._().tbody();
  for (Entry<ContainerId, Container> entry : this.nmContext
      .getContainers().entrySet()) {
    ContainerInfo info = new ContainerInfo(this.nmContext, entry.getValue());
    tableBody
      .tr()
        .td().a(url("container", info.getId()), info.getId())
        ._()
        .td()._(info.getState())._()
        .td()
            .a(url(info.getShortLogLink()), "logs")._()
      ._();
  }
  tableBody._()._()._();
}
 
Example 17
Source Project: big-c   Source File: ContainerLogsUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Finds the log file with the given filename for the given container.
 */
public static File getContainerLogFile(ContainerId containerId,
    String fileName, String remoteUser, Context context) throws YarnException {
  Container container = context.getContainers().get(containerId);
  
  Application application = getApplicationForContainer(containerId, context);
  checkAccess(remoteUser, application, context);
  if (container != null) {
    checkState(container.getContainerState());
  }
  
  try {
    LocalDirsHandlerService dirsHandler = context.getLocalDirsHandler();
    String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(
        application.getAppId().toString(), containerId.toString());
    Path logPath = dirsHandler.getLogPathToRead(
        relativeContainerLogDir + Path.SEPARATOR + fileName);
    URI logPathURI = new File(logPath.toString()).toURI();
    File logFile = new File(logPathURI.getPath());
    return logFile;
  } catch (IOException e) {
    LOG.warn("Failed to find log file", e);
    throw new NotFoundException("Cannot find this log on the local disk.");
  }
}
 
Example 18
Source Project: big-c   Source File: ResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Once a container's resources are localized, kill the corresponding
 * {@link ContainerLocalizer}
 */
private void handleContainerResourcesLocalized(
    ContainerLocalizationEvent event) {
  Container c = event.getContainer();
  String locId = ConverterUtils.toString(c.getContainerId());
  localizerTracker.endContainerLocalization(locId);
}
 
Example 19
Source Project: hadoop   Source File: ContainerInfo.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerInfo(final Context nmContext, final Container container,
     String requestUri, String pathPrefix) {

  this.id = container.getContainerId().toString();
  this.nodeId = nmContext.getNodeId().toString();
  ContainerStatus containerData = container.cloneAndGetContainerStatus();
  this.exitCode = containerData.getExitStatus();
  this.exitStatus =
      (this.exitCode == ContainerExitStatus.INVALID) ?
          "N/A" : String.valueOf(exitCode);
  this.state = container.getContainerState().toString();
  this.diagnostics = containerData.getDiagnostics();
  if (this.diagnostics == null || this.diagnostics.isEmpty()) {
    this.diagnostics = "";
  }

  this.user = container.getUser();
  Resource res = container.getResource();
  if (res != null) {
    this.totalMemoryNeededMB = res.getMemory();
    this.totalVCoresNeeded = res.getVirtualCores();
  }
  this.containerLogsShortLink = ujoin("containerlogs", this.id,
      container.getUser());

  if (requestUri == null) {
    requestUri = "";
  }
  if (pathPrefix == null) {
    pathPrefix = "";
  }
  this.containerLogsLink = join(requestUri, pathPrefix,
      this.containerLogsShortLink);
}
 
Example 20
Source Project: hadoop   Source File: AppInfo.java    License: Apache License 2.0 5 votes vote down vote up
public AppInfo(final Application app) {
  this.id = ConverterUtils.toString(app.getAppId());
  this.state = app.getApplicationState().toString();
  this.user = app.getUser();

  this.containerids = new ArrayList<String>();
  Map<ContainerId, Container> appContainers = app.getContainers();
  for (ContainerId containerId : appContainers.keySet()) {
    String containerIdStr = ConverterUtils.toString(containerId);
    containerids.add(containerIdStr);
  }
}
 
Example 21
Source Project: big-c   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyNodeContainerInfoGeneric(Container cont, String id,
    String state, String user, int exitCode, String diagnostics,
    String nodeId, int totalMemoryNeededMB, int totalVCoresNeeded,
    String logsLink)
    throws JSONException, Exception {
  WebServicesTestUtils.checkStringMatch("id", cont.getContainerId()
      .toString(), id);
  WebServicesTestUtils.checkStringMatch("state", cont.getContainerState()
      .toString(), state);
  WebServicesTestUtils.checkStringMatch("user", cont.getUser().toString(),
      user);
  assertEquals("exitCode wrong", 0, exitCode);
  WebServicesTestUtils
      .checkStringMatch("diagnostics", "testing", diagnostics);

  WebServicesTestUtils.checkStringMatch("nodeId", nmContext.getNodeId()
      .toString(), nodeId);
  assertEquals("totalMemoryNeededMB wrong",
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
    totalMemoryNeededMB);
  assertEquals("totalVCoresNeeded wrong",
    YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
    totalVCoresNeeded);
  String shortLink =
      ujoin("containerlogs", cont.getContainerId().toString(),
          cont.getUser());
  assertTrue("containerLogsLink wrong", logsLink.contains(shortLink));
}
 
Example 22
Source Project: hadoop   Source File: ContainerPage.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void render(Block html) {
  ContainerId containerID;
  try {
    containerID = ConverterUtils.toContainerId($(CONTAINER_ID));
  } catch (IllegalArgumentException e) {
    html.p()._("Invalid containerId " + $(CONTAINER_ID))._();
    return;
  }

  DIV<Hamlet> div = html.div("#content");
  Container container = this.nmContext.getContainers().get(containerID);
  if (container == null) {
    div.h1("Unknown Container. Container might have completed, "
            + "please go back to the previous page and retry.")._();
    return;
  }
  ContainerInfo info = new ContainerInfo(this.nmContext, container);

  info("Container information")
    ._("ContainerID", info.getId())
    ._("ContainerState", info.getState())
    ._("ExitStatus", info.getExitStatus())
    ._("Diagnostics", info.getDiagnostics())
    ._("User", info.getUser())
    ._("TotalMemoryNeeded", info.getMemoryNeeded())
    ._("TotalVCoresNeeded", info.getVCoresNeeded())
    ._("logs", info.getShortLogLink(), "Link to logs");
  html._(InfoBlock.class);
}
 
Example 23
Source Project: hadoop   Source File: ResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Once a container's resources are localized, kill the corresponding
 * {@link ContainerLocalizer}
 */
private void handleContainerResourcesLocalized(
    ContainerLocalizationEvent event) {
  Container c = event.getContainer();
  String locId = ConverterUtils.toString(c.getContainerId());
  localizerTracker.endContainerLocalization(locId);
}
 
Example 24
Source Project: big-c   Source File: AppInfo.java    License: Apache License 2.0 5 votes vote down vote up
public AppInfo(final Application app) {
  this.id = ConverterUtils.toString(app.getAppId());
  this.state = app.getApplicationState().toString();
  this.user = app.getUser();

  this.containerids = new ArrayList<String>();
  Map<ContainerId, Container> appContainers = app.getContainers();
  for (ContainerId containerId : appContainers.keySet()) {
    String containerIdStr = ConverterUtils.toString(containerId);
    containerids.add(containerIdStr);
  }
}
 
Example 25
Source Project: hadoop   Source File: ApplicationImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Map<ContainerId, Container> getContainers() {
  this.readLock.lock();
  try {
    return this.containers;
  } finally {
    this.readLock.unlock();
  }
}
 
Example 26
@Test(expected = IllegalArgumentException.class)
public void testContainerLaunchInvalidImage() throws IOException {
  String appSubmitter = "nobody";
  String appId = "APP_ID";
  String containerId = "CONTAINER_ID";
  String testImage = "testrepo.com/test-image rm -rf $HADOOP_PREFIX/*";

  Container container = mock(Container.class, RETURNS_DEEP_STUBS);
  ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS);
  ContainerLaunchContext context = mock(ContainerLaunchContext.class);
  HashMap<String, String> env = new HashMap<String,String>();

  when(container.getContainerId()).thenReturn(cId);
  when(container.getLaunchContext()).thenReturn(context);
  when(cId.getApplicationAttemptId().getApplicationId().toString()).thenReturn(appId);
  when(cId.toString()).thenReturn(containerId);

  when(context.getEnvironment()).thenReturn(env);
  env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  dockerContainerExecutor.getConf()
    .set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage);
  Path scriptPath = new Path("file:///bin/echo");
  Path tokensPath = new Path("file:///dev/null");

  Path pidFile = new Path(workDir, "pid.txt");

  dockerContainerExecutor.activateContainer(cId, pidFile);
  dockerContainerExecutor.launchContainer(container, scriptPath, tokensPath,
    appSubmitter, appId, workDir, dirsHandler.getLocalDirs(),
    dirsHandler.getLogDirs());
}
 
Example 27
Source Project: big-c   Source File: TestNMWebServicesContainers.java    License: Apache License 2.0 5 votes vote down vote up
public void verifyNodeContainerInfo(JSONObject info, Container cont)
    throws JSONException, Exception {
  assertEquals("incorrect number of elements", 9, info.length());

  verifyNodeContainerInfoGeneric(cont, info.getString("id"),
      info.getString("state"), info.getString("user"),
      info.getInt("exitCode"), info.getString("diagnostics"),
      info.getString("nodeId"), info.getInt("totalMemoryNeededMB"),
      info.getInt("totalVCoresNeeded"),
      info.getString("containerLogsLink"));
}
 
Example 28
Source Project: hadoop   Source File: TrafficControlBandwidthHandlerImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Pre-start hook for 'outbound-bandwidth' resource. A cgroup is created
 * and a net_cls classid is generated and written to a cgroup file. A
 * traffic control shaping rule is created in order to limit outbound
 * bandwidth utilization.
 * @param container Container being launched
 * @return privileged operations for some cgroups/tc operations.
 * @throws ResourceHandlerException
 */
@Override
public List<PrivilegedOperation> preStart(Container container)
    throws ResourceHandlerException {
  String containerIdStr = container.getContainerId().toString();
  int classId = trafficController.getNextClassId();
  String classIdStr = trafficController.getStringForNetClsClassId(classId);

  cGroupsHandler.createCGroup(CGroupsHandler.CGroupController
          .NET_CLS,
      containerIdStr);
  cGroupsHandler.updateCGroupParam(CGroupsHandler.CGroupController.NET_CLS,
      containerIdStr, CGroupsHandler.CGROUP_PARAM_CLASSID, classIdStr);
  containerIdClassIdMap.put(container.getContainerId(), classId);

  //Now create a privileged operation in order to update the tasks file with
  //the pid of the running container process (root of process tree). This can
  //only be done at the time of launching the container, in a privileged
  //executable.
  String tasksFile = cGroupsHandler.getPathForCGroupTasks(
      CGroupsHandler.CGroupController.NET_CLS, containerIdStr);
  String opArg = new StringBuffer(PrivilegedOperation.CGROUP_ARG_PREFIX)
      .append(tasksFile).toString();
  List<PrivilegedOperation> ops = new ArrayList<>();

  ops.add(new PrivilegedOperation(
      PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, opArg));

  //Create a privileged operation to create a tc rule for this container
  //We'll return this to the calling (Linux) Container Executor
  //implementation for batching optimizations so that we don't fork/exec
  //additional times during container launch.
  TrafficController.BatchBuilder builder = trafficController.new
      BatchBuilder(PrivilegedOperation.OperationType.TC_MODIFY_STATE);

  builder.addContainerClass(classId, containerBandwidthMbit, strictMode);
  ops.add(builder.commitBatchToTempFile());

  return ops;
}
 
Example 29
Source Project: hadoop   Source File: ProcessContainerRuntime.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void signalContainer(ContainerRuntimeContext ctx)
    throws ContainerExecutionException {
  Container container = ctx.getContainer();
  PrivilegedOperation signalOp = new PrivilegedOperation(
      PrivilegedOperation.OperationType.SIGNAL_CONTAINER, (String) null);

  signalOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER),
      ctx.getExecutionAttribute(USER),
      Integer.toString(PrivilegedOperation.RunAsUserCommand
          .SIGNAL_CONTAINER.getValue()),
      ctx.getExecutionAttribute(PID),
      Integer.toString(ctx.getExecutionAttribute(SIGNAL).getValue()));

  try {
    PrivilegedOperationExecutor executor = PrivilegedOperationExecutor
        .getInstance(conf);

    executor.executePrivilegedOperation(null,
        signalOp, null, container.getLaunchContext().getEnvironment(),
        false);
  } catch (PrivilegedOperationException e) {
    LOG.warn("Signal container failed. Exception: ", e);

    throw new ContainerExecutionException("Signal container failed", e
        .getExitCode(), e.getOutput(), e.getErrorOutput());
  }
}
 
Example 30
/**
 * Will launch containers within the same JVM as this Container Executor. It
 * will do so by: - extracting Container's class name and program arguments
 * from the launch script (e.g., launch_container.sh) - Creating an isolated
 * ClassLoader for each container - Calling doLaunchContainer(..) method to
 * launch Container
 */
private int doLaunch(Container container, Path containerWorkDir) throws Exception {
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    EnvironmentUtils.putAll(environment);

    Set<URL> additionalClassPathUrls = this.filterAndBuildUserClasspath(container);

    ExecJavaCliParser javaCliParser = this.createExecCommandParser(containerWorkDir.toString());

    UserGroupInformation.setLoginUser(null);
    try {
        // create Isolated Class Loader for each container and set it as context
        // class loader
        URLClassLoader containerCl =
                new URLClassLoader(additionalClassPathUrls.toArray(additionalClassPathUrls.toArray(new URL[]{})), null);
        Thread.currentThread().setContextClassLoader(containerCl);
        String containerLauncher = javaCliParser.getMain();


        Class<?> containerClass = Class.forName(containerLauncher, true, containerCl);
        Method mainMethod = containerClass.getMethod("main", new Class[] { String[].class });
        mainMethod.setAccessible(true);
        String[] arguments = javaCliParser.getMainArguments();

        this.doLaunchContainer(containerClass, mainMethod, arguments);

    }
    catch (Exception e) {
        logger.error("Failed to launch container " + container, e);
        container.handle(new ContainerDiagnosticsUpdateEvent(container.getContainerId(), e.getMessage()));
        return -1;
    }
    finally {
        logger.info("Removing symlinks");
        this.cleanUp();
    }
    return 0;
}