Java Code Examples for org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#APPCACHE

The following examples show how to use org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer#APPCACHE . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestLinuxContainerExecutor.java From hadoop with Apache License 2.0

5 votes

private void cleanupUserAppCache(String user) throws Exception {
  List<String> localDirs = dirsHandler.getLocalDirs();
  for (String dir : localDirs) {
    Path usercachedir = new Path(dir, ContainerLocalizer.USERCACHE);
    Path userdir = new Path(usercachedir, user);
    Path appcachedir = new Path(userdir, ContainerLocalizer.APPCACHE);
    exec.deleteAsUser(new DeletionAsUserContext.Builder()
        .setUser(user)
        .setSubDir(appcachedir)
        .build());
    FileContext.getLocalFSFileContext().delete(usercachedir, true);
  }
}

Example 2

Source File: ShuffleHandler.java From hadoop with Apache License 2.0

5 votes

private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}

Example 3

Source File: ShuffleHandler.java From big-c with Apache License 2.0

5 votes

private String getBaseLocation(String jobId, String user) {
  final JobID jobID = JobID.forName(jobId);
  final ApplicationId appID =
      ApplicationId.newInstance(Long.parseLong(jobID.getJtIdentifier()),
        jobID.getId());
  final String baseStr =
      ContainerLocalizer.USERCACHE + "/" + user + "/"
          + ContainerLocalizer.APPCACHE + "/"
          + ConverterUtils.toString(appID) + "/output" + "/";
  return baseStr;
}

Example 4

Source File: DockerContainerExecutor.java From hadoop with Apache License 2.0

4 votes

private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}

Example 5

Source File: DefaultContainerExecutor.java From hadoop with Apache License 2.0

4 votes

private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}

Example 6

Source File: TestContainerManager.java From hadoop with Apache License 2.0

4 votes

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}

Example 7

Source File: DockerContainerExecutor.java From big-c with Apache License 2.0

4 votes

private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}

Example 8

Source File: DefaultContainerExecutor.java From big-c with Apache License 2.0

4 votes

private Path getAppcacheDir(Path base, String user) {
  return new Path(getUserCacheDir(base, user),
      ContainerLocalizer.APPCACHE);
}

Example 9

Source File: TestContainerManager.java From big-c with Apache License 2.0

4 votes

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}