Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#FILECACHE

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#FILECACHE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: ContainerLaunch.java From hadoop with Apache License 2.0

5 votes

protected List<String> getNMFilecacheDirs(List<String> localDirs) {
  List<String> filecacheDirs = new ArrayList<>(localDirs.size());

  for (String localDir : localDirs) {
    String filecacheDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.FILECACHE;

    filecacheDirs.add(filecacheDir);
  }

  return filecacheDirs;
}

Example 2

Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0

5 votes

private void cleanupUserFileCache(String user) {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path filecache = new Path(dir, ContainerLocalizer.FILECACHE);
    Path filedir = new Path(filecache, user);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(filedir)
        .build());
  }
}

Example 3

Source File: DockerContainerExecutor.java From hadoop with Apache License 2.0

4 votes

private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}

Example 4

Source File: DefaultContainerExecutor.java From hadoop with Apache License 2.0

4 votes

private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}

Example 5

Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0

4 votes

@Before
public void setup() throws Exception {
  files = FileContext.getLocalFSFileContext();
  Path workSpacePath = new Path(workSpace.getAbsolutePath());
  files.mkdir(workSpacePath, null, true);
  FileUtil.chmod(workSpace.getAbsolutePath(), "777");
  File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
  files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"),
    false);
  File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
  files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"),
    false);
  String exec_path = System.getProperty("container-executor.path");
  if (exec_path != null && !exec_path.isEmpty()) {
    conf = new Configuration(false);
    conf.setClass("fs.AbstractFileSystem.file.impl",
      org.apache.hadoop.fs.local.LocalFs.class,
      org.apache.hadoop.fs.AbstractFileSystem.class);

    appSubmitter = System.getProperty("application.submitter");
    if (appSubmitter == null || appSubmitter.isEmpty()) {
      appSubmitter = "nobody";
    }

    conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter);
    LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
        + "=" + exec_path);
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
    exec = new LinuxContainerExecutor();
    exec.setConf(conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    List<String> localDirs = dirsHandler.getLocalDirs();
    for (String dir : localDirs) {
      Path userDir = new Path(dir, ContainerLocalizer.USERCACHE);
      files.mkdir(userDir, new FsPermission("777"), false);
      // $local/filecache
      Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE);
      files.mkdir(fileDir, new FsPermission("777"), false);
    }
  }

}

Example 6

Source File: DockerContainerExecutor.java From big-c with Apache License 2.0

4 votes

private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}

Example 7

Source File: DefaultContainerExecutor.java From big-c with Apache License 2.0

4 votes

private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}

Example 8

Source File: TestLinuxContainerExecutor.java From big-c with Apache License 2.0

4 votes

@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}