Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#USERCACHE

protected List<String> getUserLocalDirs(List<String> localDirs) {
  List<String> userLocalDirs = new ArrayList<>(localDirs.size());
  String user = container.getUser();

  for (String localDir : localDirs) {
    String userLocalDir = localDir + Path.SEPARATOR +
        ContainerLocalizer.USERCACHE + Path.SEPARATOR + user
        + Path.SEPARATOR;

    userLocalDirs.add(userLocalDir);
  }

  return userLocalDirs;
}
 

private void cleanupUserAppCache(String user) throws Exception {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path usercachedir = new Path(dir, ContainerLocalizer.USERCACHE);
    Path userdir = new Path(usercachedir, user);
    Path appcachedir = new Path(userdir, ContainerLocalizer.APPCACHE);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(appcachedir)
        .build());
    FileContext.getLocalFSFileContext().delete(usercachedir, true);
  }
}
 

private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 

private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}
 

private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 

private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 

@Before
public void setup() throws Exception {
  files = FileContext.getLocalFSFileContext();
  Path workSpacePath = new Path(workSpace.getAbsolutePath());
  files.mkdir(workSpacePath, null, true);
  FileUtil.chmod(workSpace.getAbsolutePath(), "777");
  File localDir = new File(workSpace.getAbsoluteFile(), "localDir");
  files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"),
    false);
  File logDir = new File(workSpace.getAbsoluteFile(), "logDir");
  files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"),
    false);
  String exec_path = System.getProperty("container-executor.path");
  if (exec_path != null && !exec_path.isEmpty()) {
    conf = new Configuration(false);
    conf.setClass("fs.AbstractFileSystem.file.impl",
      org.apache.hadoop.fs.local.LocalFs.class,
      org.apache.hadoop.fs.AbstractFileSystem.class);

    appSubmitter = System.getProperty("application.submitter");
    if (appSubmitter == null || appSubmitter.isEmpty()) {
      appSubmitter = "nobody";
    }

    conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter);
    LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH
        + "=" + exec_path);
    conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path);
    exec = new LinuxContainerExecutor();
    exec.setConf(conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath());
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    List<String> localDirs = dirsHandler.getLocalDirs();
    for (String dir : localDirs) {
      Path userDir = new Path(dir, ContainerLocalizer.USERCACHE);
      files.mkdir(userDir, new FsPermission("777"), false);
      // $local/filecache
      Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE);
      files.mkdir(fileDir, new FsPermission("777"), false);
    }
  }

}
 

private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 

private Path getUserCacheDir(Path base, String user) {
  return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
 

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 

@Test
public void testContainerLocalizer() throws Exception {
  if (!shouldRun()) {
    return;
  }
  List<String> localDirs = dirsHandler.getLocalDirs();
  List<String> logDirs = dirsHandler.getLogDirs();
  for (String localDir : localDirs) {
    Path userDir =
        new Path(localDir, ContainerLocalizer.USERCACHE);
    files.mkdir(userDir, new FsPermission("777"), false);
    // $local/filecache
    Path fileDir =
        new Path(localDir, ContainerLocalizer.FILECACHE);
    files.mkdir(fileDir, new FsPermission("777"), false);
  }
  String locId = "container_01_01";
  Path nmPrivateContainerTokensPath =
      dirsHandler.getLocalPathForWrite(
          ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT,
                locId));
  files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE));
  Configuration config = new YarnConfiguration(conf);
  InetSocketAddress nmAddr = config.getSocketAddr(
    YarnConfiguration.NM_BIND_HOST,
    YarnConfiguration.NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS,
    YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT);
  String appId = "application_01_01";
  exec = new LinuxContainerExecutor() {
    @Override
    public void buildMainArgs(List<String> command, String user, String appId,
        String locId, InetSocketAddress nmAddr, List<String> localDirs) {
      MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr,
        localDirs);
    }
  };
  exec.setConf(conf);

  exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter,
    appId, locId, dirsHandler);

  String locId2 = "container_01_02";
  Path nmPrivateContainerTokensPath2 =
      dirsHandler
        .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR
            + Path.SEPARATOR
            + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2));
  files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE));
  exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter,
    appId, locId2, dirsHandler);
}