Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#FILECACHE
The following examples show how to use
org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#FILECACHE .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ContainerLaunch.java From hadoop with Apache License 2.0 | 5 votes |
protected List<String> getNMFilecacheDirs(List<String> localDirs) { List<String> filecacheDirs = new ArrayList<>(localDirs.size()); for (String localDir : localDirs) { String filecacheDir = localDir + Path.SEPARATOR + ContainerLocalizer.FILECACHE; filecacheDirs.add(filecacheDir); } return filecacheDirs; }
Example 2
Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0 | 5 votes |
private void cleanupUserFileCache(String user) { List<String> localDirs = dirsHandler.getLocalDirs(); for (String dir : localDirs) { Path filecache = new Path(dir, ContainerLocalizer.FILECACHE); Path filedir = new Path(filecache, user); exec.deleteAsUser(new DeletionAsUserContext.Builder() .setUser(user) .setSubDir(filedir) .build()); } }
Example 3
Source File: DockerContainerExecutor.java From hadoop with Apache License 2.0 | 4 votes |
private Path getFileCacheDir(Path base, String user) { return new Path(getUserCacheDir(base, user), ContainerLocalizer.FILECACHE); }
Example 4
Source File: DefaultContainerExecutor.java From hadoop with Apache License 2.0 | 4 votes |
private Path getFileCacheDir(Path base, String user) { return new Path(getUserCacheDir(base, user), ContainerLocalizer.FILECACHE); }
Example 5
Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0 | 4 votes |
@Before public void setup() throws Exception { files = FileContext.getLocalFSFileContext(); Path workSpacePath = new Path(workSpace.getAbsolutePath()); files.mkdir(workSpacePath, null, true); FileUtil.chmod(workSpace.getAbsolutePath(), "777"); File localDir = new File(workSpace.getAbsoluteFile(), "localDir"); files.mkdir(new Path(localDir.getAbsolutePath()), new FsPermission("777"), false); File logDir = new File(workSpace.getAbsoluteFile(), "logDir"); files.mkdir(new Path(logDir.getAbsolutePath()), new FsPermission("777"), false); String exec_path = System.getProperty("container-executor.path"); if (exec_path != null && !exec_path.isEmpty()) { conf = new Configuration(false); conf.setClass("fs.AbstractFileSystem.file.impl", org.apache.hadoop.fs.local.LocalFs.class, org.apache.hadoop.fs.AbstractFileSystem.class); appSubmitter = System.getProperty("application.submitter"); if (appSubmitter == null || appSubmitter.isEmpty()) { appSubmitter = "nobody"; } conf.set(YarnConfiguration.NM_NONSECURE_MODE_LOCAL_USER_KEY, appSubmitter); LOG.info("Setting " + YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH + "=" + exec_path); conf.set(YarnConfiguration.NM_LINUX_CONTAINER_EXECUTOR_PATH, exec_path); exec = new LinuxContainerExecutor(); exec.setConf(conf); conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.getAbsolutePath()); dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); List<String> localDirs = dirsHandler.getLocalDirs(); for (String dir : localDirs) { Path userDir = new Path(dir, ContainerLocalizer.USERCACHE); files.mkdir(userDir, new FsPermission("777"), false); // $local/filecache Path fileDir = new Path(dir, ContainerLocalizer.FILECACHE); files.mkdir(fileDir, new FsPermission("777"), false); } } }
Example 6
Source File: DockerContainerExecutor.java From big-c with Apache License 2.0 | 4 votes |
private Path getFileCacheDir(Path base, String user) { return new Path(getUserCacheDir(base, user), ContainerLocalizer.FILECACHE); }
Example 7
Source File: DefaultContainerExecutor.java From big-c with Apache License 2.0 | 4 votes |
private Path getFileCacheDir(Path base, String user) { return new Path(getUserCacheDir(base, user), ContainerLocalizer.FILECACHE); }
Example 8
Source File: TestLinuxContainerExecutor.java From big-c with Apache License 2.0 | 4 votes |
@Test public void testContainerLocalizer() throws Exception { if (!shouldRun()) { return; } List<String> localDirs = dirsHandler.getLocalDirs(); List<String> logDirs = dirsHandler.getLogDirs(); for (String localDir : localDirs) { Path userDir = new Path(localDir, ContainerLocalizer.USERCACHE); files.mkdir(userDir, new FsPermission("777"), false); // $local/filecache Path fileDir = new Path(localDir, ContainerLocalizer.FILECACHE); files.mkdir(fileDir, new FsPermission("777"), false); } String locId = "container_01_01"; Path nmPrivateContainerTokensPath = dirsHandler.getLocalPathForWrite( ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId)); files.create(nmPrivateContainerTokensPath, EnumSet.of(CREATE, OVERWRITE)); Configuration config = new YarnConfiguration(conf); InetSocketAddress nmAddr = config.getSocketAddr( YarnConfiguration.NM_BIND_HOST, YarnConfiguration.NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_ADDRESS, YarnConfiguration.DEFAULT_NM_LOCALIZER_PORT); String appId = "application_01_01"; exec = new LinuxContainerExecutor() { @Override public void buildMainArgs(List<String> command, String user, String appId, String locId, InetSocketAddress nmAddr, List<String> localDirs) { MockContainerLocalizer.buildMainArgs(command, user, appId, locId, nmAddr, localDirs); } }; exec.setConf(conf); exec.startLocalizer(nmPrivateContainerTokensPath, nmAddr, appSubmitter, appId, locId, dirsHandler); String locId2 = "container_01_02"; Path nmPrivateContainerTokensPath2 = dirsHandler .getLocalPathForWrite(ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId2)); files.create(nmPrivateContainerTokensPath2, EnumSet.of(CREATE, OVERWRITE)); exec.startLocalizer(nmPrivateContainerTokensPath2, nmAddr, appSubmitter, appId, locId2, dirsHandler); }