org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer Java Examples

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestShuffleHandler.java    From big-c with Apache License 2.0 6 votes vote down vote up
private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 
Example #2
Source File: TestNodeManagerReboot.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void checkNumOfLocalDirs() throws IOException {
  Assert
    .assertTrue(
      "After NM reboots, all local files should be deleted",
      numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.USERCACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ContainerLocalizer.FILECACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ResourceLocalizationService.NM_PRIVATE_DIR) == 0);
  
  Assert
  .assertTrue(
    "After NM reboots, usercache_DEL_* directory should be deleted",
    numOfUsercacheDELDirs(nmLocalDir.getAbsolutePath()) == 0);
}
 
Example #3
Source File: TestNodeManagerReboot.java    From big-c with Apache License 2.0 6 votes vote down vote up
private void restartNM(int maxTries) {
  nm.stop();
  nm = new MyNodeManager();
  nm.start();

  int numTries = 0;
  while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(),
    ContainerLocalizer.USERCACHE) > 0
      || numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.FILECACHE) > 0 || numOfLocalDirs(
    nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0)
      && numTries < maxTries) {
    try {
      Thread.sleep(500);
    } catch (InterruptedException ex) {
      // Do nothing
    }
    numTries++;
  }
}
 
Example #4
Source File: TestShuffleHandler.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private static void createShuffleHandlerFiles(File logDir, String user,
    String appId, String appAttemptId, Configuration conf,
    List<File> fileMap) throws IOException {
  String attemptDir =
      StringUtils.join(Path.SEPARATOR,
          Arrays.asList(new String[] { logDir.getAbsolutePath(),
              ContainerLocalizer.USERCACHE, user,
              ContainerLocalizer.APPCACHE, appId, "output", appAttemptId }));
  File appAttemptDir = new File(attemptDir);
  appAttemptDir.mkdirs();
  System.out.println(appAttemptDir.getAbsolutePath());
  File indexFile = new File(appAttemptDir, "file.out.index");
  fileMap.add(indexFile);
  createIndexFile(indexFile, conf);
  File mapOutputFile = new File(appAttemptDir, "file.out");
  fileMap.add(mapOutputFile);
  createMapOutputFile(mapOutputFile, conf);
}
 
Example #5
Source File: TestNodeManagerReboot.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void checkNumOfLocalDirs() throws IOException {
  Assert
    .assertTrue(
      "After NM reboots, all local files should be deleted",
      numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.USERCACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ContainerLocalizer.FILECACHE) == 0
          && numOfLocalDirs(nmLocalDir.getAbsolutePath(),
            ResourceLocalizationService.NM_PRIVATE_DIR) == 0);
  
  Assert
  .assertTrue(
    "After NM reboots, usercache_DEL_* directory should be deleted",
    numOfUsercacheDELDirs(nmLocalDir.getAbsolutePath()) == 0);
}
 
Example #6
Source File: TestNodeManagerReboot.java    From hadoop with Apache License 2.0 6 votes vote down vote up
private void restartNM(int maxTries) {
  nm.stop();
  nm = new MyNodeManager();
  nm.start();

  int numTries = 0;
  while ((numOfLocalDirs(nmLocalDir.getAbsolutePath(),
    ContainerLocalizer.USERCACHE) > 0
      || numOfLocalDirs(nmLocalDir.getAbsolutePath(),
        ContainerLocalizer.FILECACHE) > 0 || numOfLocalDirs(
    nmLocalDir.getAbsolutePath(), ResourceLocalizationService.NM_PRIVATE_DIR) > 0)
      && numTries < maxTries) {
    try {
      Thread.sleep(500);
    } catch (InterruptedException ex) {
      // Do nothing
    }
    numTries++;
  }
}
 
Example #7
Source File: TestNodeManagerReboot.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private int numOfUsercacheDELDirs(String localDir) throws IOException {
  int count = 0;
  RemoteIterator<FileStatus> fileStatus = localFS.listStatus(new Path(localDir));
  while (fileStatus.hasNext()) {
    FileStatus status = fileStatus.next();
    if (status.getPath().getName().matches(".*" +
        ContainerLocalizer.USERCACHE + "_DEL_.*")) {
      count++;
    }
  }
  return count;
}
 
Example #8
Source File: ShuffleHandler.java    From big-c with Apache License 2.0 5 votes vote down vote up
private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 
Example #9
Source File: TestNodeManagerReboot.java    From big-c with Apache License 2.0 5 votes vote down vote up
private int numOfUsercacheDELDirs(String localDir) throws IOException {
  int count = 0;
  RemoteIterator<FileStatus> fileStatus = localFS.listStatus(new Path(localDir));
  while (fileStatus.hasNext()) {
    FileStatus status = fileStatus.next();
    if (status.getPath().getName().matches(".*" +
        ContainerLocalizer.USERCACHE + "_DEL_.*")) {
      count++;
    }
  }
  return count;
}
 
Example #10
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void startLocalizer(Path nmPrivateContainerTokensPath,
    InetSocketAddress nmAddr, String user, String appId, String locId,
    LocalDirsHandlerService dirsHandler)
    throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example #11
Source File: DockerContainerExecutor.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void startLocalizer(Path nmPrivateContainerTokensPath,
                                        InetSocketAddress nmAddr, String user, String appId, String locId,
                                        LocalDirsHandlerService dirsHandler)
  throws IOException, InterruptedException {

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  //LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  //LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example #12
Source File: ShuffleHandler.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 
Example #13
Source File: TestLinuxContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void cleanupUserFileCache(String user) {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path filecache = new Path(dir, ContainerLocalizer.FILECACHE);
    Path filedir = new Path(filecache, user);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(filedir)
        .build());
  }
}
 
Example #14
Source File: TestLinuxContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void cleanupUserAppCache(String user) throws Exception {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path usercachedir = new Path(dir, ContainerLocalizer.USERCACHE);
    Path userdir = new Path(usercachedir, user);
    Path appcachedir = new Path(userdir, ContainerLocalizer.APPCACHE);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(appcachedir)
        .build());
    FileContext.getLocalFSFileContext().delete(usercachedir, true);
  }
}
 
Example #15
Source File: ContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected List<String> getUserLocalDirs(List<String> localDirs) {
  List<String> userLocalDirs = new ArrayList<>(localDirs.size());
  String user = container.getUser();

  for (String localDir : localDirs) {
    String userLocalDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
        + Path.SEPARATOR;

    userLocalDirs.add(userLocalDir);
  }

  return userLocalDirs;
}
 
Example #16
Source File: ContainerLaunch.java    From hadoop with Apache License 2.0 5 votes vote down vote up
protected List<String> getNMFilecacheDirs(List<String> localDirs) {
  List<String> filecacheDirs = new ArrayList<>(localDirs.size());

  for (String localDir : localDirs) {
    String filecacheDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.FILECACHE;

    filecacheDirs.add(filecacheDir);
  }

  return filecacheDirs;
}
 
Example #17
Source File: DockerContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void startLocalizer(LocalizerStartContext ctx)
  throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();

  ContainerLocalizer localizer =
    new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs),
      RecordFactoryProvider.getRecordFactory(getConf()));

  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);
  lfs.setWorkingDirectory(appStorageDir);
  LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory());
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example #18
Source File: DefaultContainerExecutor.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public void startLocalizer(LocalizerStartContext ctx)
    throws IOException, InterruptedException {
  Path nmPrivateContainerTokensPath = ctx.getNmPrivateContainerTokens();
  InetSocketAddress nmAddr = ctx.getNmAddr();
  String user = ctx.getUser();
  String appId = ctx.getAppId();
  String locId = ctx.getLocId();
  LocalDirsHandlerService dirsHandler = ctx.getDirsHandler();

  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  
  createUserLocalDirs(localDirs, user);
  createUserCacheDirs(localDirs, user);
  createAppDirs(localDirs, user, appId);
  createAppLogDirs(appId, logDirs, user);

  // randomly choose the local directory
  Path appStorageDir = getWorkingDir(localDirs, user, appId);

  String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId);
  Path tokenDst = new Path(appStorageDir, tokenFn);
  copyFile(nmPrivateContainerTokensPath, tokenDst, user);
  LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst);


  FileContext localizerFc = FileContext.getFileContext(
      lfs.getDefaultFileSystem(), getConf());
  localizerFc.setUMask(lfs.getUMask());
  localizerFc.setWorkingDirectory(appStorageDir);
  LOG.info("Localizer CWD set to " + appStorageDir + " = " 
      + localizerFc.getWorkingDirectory());
  ContainerLocalizer localizer =
      new ContainerLocalizer(localizerFc, user, appId, locId, 
          getPaths(localDirs), RecordFactoryProvider.getRecordFactory(getConf()));
  // TODO: DO it over RPC for maintaining similarity?
  localizer.runLocalization(nmAddr);
}
 
Example #19
Source File: DockerContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}
 
Example #20
Source File: DockerContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 
Example #21
Source File: DockerContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}
 
Example #22
Source File: TestLinuxContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}
 
Example #23
Source File: TestContainerManager.java    From big-c with Apache License 2.0 4 votes vote down vote up
@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 
Example #24
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}
 
Example #25
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}
 
Example #26
Source File: DefaultContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 
Example #27
Source File: DockerContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up
private Path getFileCacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.FILECACHE);
}
 
Example #28
Source File: LinuxContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
@VisibleForTesting
public void buildMainArgs(List<String> command, String user, String appId,
    String locId, InetSocketAddress nmAddr, List<String> localDirs) {
  ContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
    localDirs);
}
 
Example #29
Source File: TestLinuxContainerExecutor.java    From hadoop with Apache License 2.0 4 votes vote down vote up
@Before
public void setup() throws Exception {
  files = FileContext.getLocalFSFileContext();
  Path workSpacePath = new Path(workSpace.getAbsolutePath());
  files.mkdir(workSpacePath, null, true);
  FileUtil.chmod(workSpace.getAbsolutePath(), "777");
  File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
  files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"),
    false);
  File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
  files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"),
    false);
  String exec_path = System.getProperty("container-executor.path");
  if (exec_path != null && !exec_path.isEmpty()) {
    conf = new Configuration(false);
    conf.setClass("fs.AbstractFileSystem.file.impl",
      org.apache.hadoop.fs.local.LocalFs.class,
      org.apache.hadoop.fs.AbstractFileSystem.class);

    appSubmitter = System.getProperty("application.submitter");
    if (appSubmitter == null || appSubmitter.isEmpty()) {
      appSubmitter = "nobody";
    }

    conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter);
    LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
        + "=" + exec_path);
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
    exec = new LinuxContainerExecutor();
    exec.setConf(conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    List<String> localDirs = dirsHandler.getLocalDirs();
    for (String dir : localDirs) {
      Path userDir = new Path(dir, ContainerLocalizer.USERCACHE);
      files.mkdir(userDir, new FsPermission("777"), false);
      // $local/filecache
      Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE);
      files.mkdir(fileDir, new FsPermission("777"), false);
    }
  }

}
 
Example #30
Source File: DockerContainerExecutor.java    From big-c with Apache License 2.0 4 votes vote down vote up
private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}