Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 6 votes vote down vote up
private void restartNM(int maxTries) {
  nm.stop();
  nm = new MyNodeManager();
  nm.start();

  int numTries = 0;
  while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(),
    ContainerLocalizer.USERCACHE) > 0
      || numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.FILECACHE) > 0 || numOfLocalDirs(
    nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0)
      && numTries < maxTries) {
    try {
      Thread.sleep(500);
    } catch (InterruptedException ex) {
      // Do nothing
    }
    numTries++;
  }
}
 
Example 2
Source Project: hadoop   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 6 votes vote down vote up
private void checkNumOfLocalDirs() throws IOException {
  Assert
    .assertTrue(
      "After NM reboots, all local files should be deleted",
      numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.USERCACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ContainerLocalizer.FILECACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ResourceLocalizationService.NM_PRIVATE_DIR) == 0);
  
  Assert
  .assertTrue(
    "After NM reboots, usercache_DEL_* directory should be deleted",
    numOfUsercacheDELDirs(nmLocalDir.getAbsolutePath()) == 0);
}
 
Example 3
Source Project: hadoop   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 
Example 4
Source Project: big-c   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 6 votes vote down vote up
private void restartNM(int maxTries) {
  nm.stop();
  nm = new MyNodeManager();
  nm.start();

  int numTries = 0;
  while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(),
    ContainerLocalizer.USERCACHE) > 0
      || numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.FILECACHE) > 0 || numOfLocalDirs(
    nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0)
      && numTries < maxTries) {
    try {
      Thread.sleep(500);
    } catch (InterruptedException ex) {
      // Do nothing
    }
    numTries++;
  }
}
 
Example 5
Source Project: big-c   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 6 votes vote down vote up
private void checkNumOfLocalDirs() throws IOException {
  Assert
    .assertTrue(
      "After NM reboots, all local files should be deleted",
      numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.USERCACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ContainerLocalizer.FILECACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ResourceLocalizationService.NM_PRIVATE_DIR) == 0);
  
  Assert
  .assertTrue(
    "After NM reboots, usercache_DEL_* directory should be deleted",
    numOfUsercacheDELDirs(nmLocalDir.getAbsolutePath()) == 0);
}
 
Example 6
Source Project: big-c   Source File: TestShuffleHandler.java    License: Apache License 2.0 6 votes vote down vote up
private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 
Example 7
Source Project: hadoop   Source File: ContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
protected List<String> getUserLocalDirs(List<String> localDirs) {
  List<String> userLocalDirs = new ArrayList<>(localDirs.size());
  String user = container.getUser();

  for (String localDir : localDirs) {
    String userLocalDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
        + Path.SEPARATOR;

    userLocalDirs.add(userLocalDir);
  }

  return userLocalDirs;
}
 
Example 8
Source Project: hadoop   Source File: ContainerLaunch.java    License: Apache License 2.0 5 votes vote down vote up
protected List<String> getNMFilecacheDirs(List<String> localDirs) {
  List<String> filecacheDirs = new ArrayList<>(localDirs.size());

  for (String localDir : localDirs) {
    String filecacheDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.FILECACHE;

    filecacheDirs.add(filecacheDir);
  }

  return filecacheDirs;
}
 
Example 9
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void startLocalizer(LocalizerStartContext ctx)
  throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 10
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void startLocalizer(LocalizerStartContext ctx)
    throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 11
Source Project: hadoop   Source File: TestLinuxContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void cleanupUserAppCache(String user) throws Exception {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path usercachedir = new Path(dir, ContainerLocalizer.USERCACHE);
    Path userdir = new Path(usercachedir, user);
    Path appcachedir = new Path(userdir, ContainerLocalizer.APPCACHE);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(appcachedir)
        .build());
    FileContext.getLocalFSFileContext().delete(usercachedir, true);
  }
}
 
Example 12
Source Project: hadoop   Source File: TestLinuxContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
private void cleanupUserFileCache(String user) {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path filecache = new Path(dir, ContainerLocalizer.FILECACHE);
    Path filedir = new Path(filecache, user);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(filedir)
        .build());
  }
}
 
Example 13
Source Project: hadoop   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 5 votes vote down vote up
private int numOfUsercacheDELDirs(String localDir) throws IOException {
  int count = 0;
  RemoteIterator<FileStatus> fileStatus = localFS.listStatus(new Path(localDir));
  while (fileStatus.hasNext()) {
    FileStatus status = fileStatus.next();
    if (status.getPath().getName().matches(".*" +
        ContainerLocalizer.USERCACHE + "_DEL_.*")) {
      count++;
    }
  }
  return count;
}
 
Example 14
Source Project: hadoop   Source File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 
Example 15
Source Project: big-c   Source File: DockerContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
                                        InetSocketAddress nmAddr, String user, String appId, String locId,
                                        LocalDirsHandlerService dirsHandler)
  throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  //LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  //LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 16
Source Project: big-c   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
    InetSocketAddress nmAddr, String user, String appId, String locId,
    LocalDirsHandlerService dirsHandler)
    throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example 17
Source Project: big-c   Source File: TestNodeManagerReboot.java    License: Apache License 2.0 5 votes vote down vote up
private int numOfUsercacheDELDirs(String localDir) throws IOException {
  int count = 0;
  RemoteIterator<FileStatus> fileStatus = localFS.listStatus(new Path(localDir));
  while (fileStatus.hasNext()) {
    FileStatus status = fileStatus.next();
    if (status.getPath().getName().matches(".*" +
        ContainerLocalizer.USERCACHE + "_DEL_.*")) {
      count++;
    }
  }
  return count;
}
 
Example 18
Source Project: big-c   Source File: ShuffleHandler.java    License: Apache License 2.0 5 votes vote down vote up
private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 
Example 19
Source Project: hadoop   Source File: WindowsSecureContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void startLocalizer(LocalizerStartContext ctx) throws IOException,
    InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  Path classpathJarPrivateDir = dirsHandler.getLocalPathForWrite(
      ResourceLocalizationService.NM_PRIVATE_DIR);
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(
      ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);

  File cwdApp = new File(appStorageDir.toString());
  if (LOG.isDebugEnabled()) {
    LOG.debug(String.format("cwdApp: %s", cwdApp));
  }

  List<String> command ;

  command = new ArrayList<String>();

  //use same jvm as parent
  File jvm = new File(
      new File(System.getProperty("java.home"), "bin"), "java.exe");
  command.add(jvm.toString());

  Path cwdPath = new Path(cwdApp.getPath());

  // Build a temp classpath jar. See ContainerLaunch.sanitizeEnv().
  // Passing CLASSPATH explicitly is *way* too long for command line.
  String classPath = System.getProperty("java.class.path");
  Map<String, String> env = new HashMap<String, String>(System.getenv());
  String jarCp[] = FileUtil.createJarWithClassPath(classPath,
      classpathJarPrivateDir, cwdPath, env);
  String classPathJar = localizeClasspathJar(
      new Path(jarCp[0]), cwdPath, user).toString();
  command.add("-classpath");
  command.add(classPathJar + jarCp[1]);

  String javaLibPath = System.getProperty("java.library.path");
  if (javaLibPath != null) {
    command.add("-Djava.library.path=" + javaLibPath);
  }
  command.addAll(ContainerLocalizer.getJavaOpts(getConf()));

  ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
      localDirs);

  String cmdLine = StringUtils.join(command, " ");

  String localizerPid = String.format(LOCALIZER_PID_FORMAT, locId);

  WintuilsProcessStubExecutor stubExecutor = new WintuilsProcessStubExecutor(
      cwdApp.getAbsolutePath(),
      localizerPid, user, "nul:", cmdLine);
  try {
    stubExecutor.execute();
    stubExecutor.validateResult();
  } finally {
    stubExecutor.close();
    try
    {
      killContainer(localizerPid, Signal.KILL);
    }
    catch(Throwable e) {
      LOG.warn(String.format(
          "An exception occured during the cleanup of localizer job %s:%n%s",
          localizerPid,
          org.apache.hadoop.util.StringUtils.stringifyException(e)));
    }
  }
}
 
Example 20
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 
Example 21
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}
 
Example 22
Source Project: hadoop   Source File: DockerContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}
 
Example 23
Source Project: hadoop   Source File: LinuxContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public void startLocalizer(LocalizerStartContext ctx)
    throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  verifyUsernamePattern(user);
  String runAsUser = getRunAsUser(user);
  PrivilegedOperation initializeContainerOp = new PrivilegedOperation(
      PrivilegedOperation.OperationType.INITIALIZE_CONTAINER);
  List<String> prefixCommands = new ArrayList<>();

  addSchedPriorityCommand(prefixCommands);
  initializeContainerOp.appendArgs(
      runAsUser,
      user,
      Integer.toString(
          PrivilegedOperation.RunAsUserCommand.INITIALIZE_CONTAINER
              .getValue()),
      appId,
      nmPrivateContainerTokensPath.toUri().getPath().toString(),
      StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
          localDirs),
      StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR,
          logDirs));

    File jvm =                                  // use same jvm as parent
      new File(new File(System.getProperty("java.home"), "bin"), "java");
  initializeContainerOp.appendArgs(jvm.toString());
  initializeContainerOp.appendArgs("-classpath");
  initializeContainerOp.appendArgs(System.getProperty("java.class.path"));
    String javaLibPath = System.getProperty("java.library.path");
    if (javaLibPath != null) {
    initializeContainerOp.appendArgs("-Djava.library.path=" + javaLibPath);
    }

  initializeContainerOp.appendArgs(ContainerLocalizer.getJavaOpts(getConf()));

  List<String> localizerArgs = new ArrayList<>();

  buildMainArgs(localizerArgs, user, appId, locId, nmAddr, localDirs);
  initializeContainerOp.appendArgs(localizerArgs);

    try {
    Configuration conf = super.getConf();
    PrivilegedOperationExecutor privilegedOperationExecutor =
        PrivilegedOperationExecutor.getInstance(conf);

    privilegedOperationExecutor.executePrivilegedOperation(prefixCommands,
        initializeContainerOp, null, null, false);

  } catch (PrivilegedOperationException e) {
    int exitCode = e.getExitCode();
      LOG.warn("Exit code from container " + locId + " startLocalizer is : "
          + exitCode, e);

      throw new IOException("Application " + appId + " initialization failed" +
        " (exitCode=" + exitCode + ") with output: " + e.getOutput(), e);
    }
}
 
Example 24
Source Project: hadoop   Source File: LinuxContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public void buildMainArgs(List<String> command, String user, String appId,
    String locId, InetSocketAddress nmAddr, List<String> localDirs) {
  ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
    localDirs);
}
 
Example 25
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 
Example 26
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}
 
Example 27
Source Project: hadoop   Source File: DefaultContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}
 
Example 28
Source Project: hadoop   Source File: TestContainerManager.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 
Example 29
Source Project: hadoop   Source File: TestLinuxContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws Exception {
  files = FileContext.getLocalFSFileContext();
  Path workSpacePath = new Path(workSpace.getAbsolutePath());
  files.mkdir(workSpacePath, null, true);
  FileUtil.chmod(workSpace.getAbsolutePath(), "777");
  File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
  files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"),
    false);
  File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
  files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"),
    false);
  String exec_path = System.getProperty("container-executor.path");
  if (exec_path != null && !exec_path.isEmpty()) {
    conf = new Configuration(false);
    conf.setClass("fs.AbstractFileSystem.file.impl",
      org.apache.hadoop.fs.local.LocalFs.class,
      org.apache.hadoop.fs.AbstractFileSystem.class);

    appSubmitter = System.getProperty("application.submitter");
    if (appSubmitter == null || appSubmitter.isEmpty()) {
      appSubmitter = "nobody";
    }

    conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter);
    LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
        + "=" + exec_path);
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
    exec = new LinuxContainerExecutor();
    exec.setConf(conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    List<String> localDirs = dirsHandler.getLocalDirs();
    for (String dir : localDirs) {
      Path userDir = new Path(dir, ContainerLocalizer.USERCACHE);
      files.mkdir(userDir, new FsPermission("777"), false);
      // $local/filecache
      Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE);
      files.mkdir(fileDir, new FsPermission("777"), false);
    }
  }

}
 
Example 30
Source Project: hadoop   Source File: TestLinuxContainerExecutor.java    License: Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerLocalizer() throws Exception {

  Assume.assumeTrue(shouldRun());

  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr =
      config.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
        YarnConfiguration.NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
        YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user,
        String appId, String locId, InetSocketAddress nmAddr,
        List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId,
        nmAddr, localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(new LocalizerStartContext.Builder()
      .setNmPrivateContainerTokens(nmPrivateContainerTokensPath)
      .setNmAddr(nmAddr)
      .setUser(appSubmitter)
      .setAppId(appId)
      .setLocId(locId)
      .setDirsHandler(dirsHandler)
      .build());

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(new LocalizerStartContext.Builder()
          .setNmPrivateContainerTokens(nmPrivateContainerTokensPath2)
          .setNmAddr(nmAddr)
          .setUser(appSubmitter)
          .setAppId(appId)
          .setLocId(locId2)
          .setDirsHandler(dirsHandler)
          .build());


  cleanupUserAppCache(appSubmitter);
}