Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResource#setType()

static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 

/**
 * @param file Path.
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
    throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);

    file = fs.makeQualified(file);

    FileStatus stat = fs.getFileStatus(file);

    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);

    return resource;
}
 

private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 

static LocalResource createTarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".tar");
  archiveFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new FileOutputStream(archiveFile));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar"))
      .getModificationTime());
  return ret;
}
 

public static LocalResource newYarnAppResource(
    FileSystem fs, Path path,
    LocalResourceType type, LocalResourceVisibility vis) throws IOException {
  Path qualified = fs.makeQualified(path);
  FileStatus status = fs.getFileStatus(qualified);
  LocalResource resource = Records.newRecord(LocalResource.class);
  resource.setType(type);
  resource.setVisibility(vis);
  resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified));
  resource.setTimestamp(status.getModificationTime());
  resource.setSize(status.getLen());
  return resource;
}
 

static LocalResource createFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException {
  createFile(files, p, len, r);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  ret.setSize(len);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(p).getModificationTime());
  return ret;
}
 

private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
    throws IOException {
  LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 

static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 

/**
 * Helper function to create a YARN LocalResource
 * @param fs FileSystem object
 * @param p Path of resource to localize
 * @param type LocalResource Type
 * @return a YARN LocalResource for the given Path
 * @throws IOException
 */
static LocalResource createLocalResource(FileSystem fs, Path p,
    LocalResourceType type,
    LocalResourceVisibility visibility) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs.resolvePath(rsrcStat
      .getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(visibility);
  return rsrc;
}
 

private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 

static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 

public void addFile(FileStatus fstatus) {
  Path fpath = fstatus.getPath() ;
  LocalResource libJar = Records.newRecord(LocalResource.class);
  libJar.setResource(ConverterUtils.getYarnUrlFromPath(fpath));
  libJar.setSize(fstatus.getLen());
  libJar.setTimestamp(fstatus.getModificationTime());
  libJar.setType(LocalResourceType.FILE);
  libJar.setVisibility(LocalResourceVisibility.PUBLIC);
  put(fpath.getName(), libJar) ;
}
 

static LocalResource createFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException {
  createFile(files, p, len, r);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  ret.setSize(len);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(p).getModificationTime());
  return ret;
}
 

public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type)
    throws IOException {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  FileStatus fileStatus = fs.getFileStatus(path);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
  localResource.setSize(fileStatus.getLen());
  localResource.setTimestamp(fileStatus.getModificationTime());
  localResource.setType(type);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}
 

/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 

private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 

@Test
public void testContainerSetup() throws Exception {

  containerManager.start();

  // ////// Create the resources for the container
  File dir = new File(tmpDir, "dir");
  dir.mkdirs();
  File file = new File(dir, "file");
  PrintWriter fileWriter = new PrintWriter(file);
  fileWriter.write("Hello World!");
  fileWriter.close();

  // ////// Construct the Container-id
  ContainerId cId = createContainerId(0);

  // ////// Construct the container-spec.
  ContainerLaunchContext containerLaunchContext = 
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(file.getAbsolutePath())));
  LocalResource rsrc_alpha = recordFactory.newRecordInstance(LocalResource.class);    
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(file.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

  BaseContainerManagerTest.waitForContainerState(containerManager, cId,
      ContainerState.COMPLETE);

  // Now ascertain that the resources are localised correctly.
  ApplicationId appId = cId.getApplicationAttemptId().getApplicationId();
  String appIDStr = ConverterUtils.toString(appId);
  String containerIDStr = ConverterUtils.toString(cId);
  File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
  File userDir = new File(userCacheDir, user);
  File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
  File appDir = new File(appCache, appIDStr);
  File containerDir = new File(appDir, containerIDStr);
  File targetFile = new File(containerDir, destinationFile);
  File sysDir =
      new File(localDir,
          ResourceLocalizationService.NM_PRIVATE_DIR);
  File appSysDir = new File(sysDir, appIDStr);
  File containerSysDir = new File(appSysDir, containerIDStr);

  for (File f : new File[] { localDir, sysDir, userCacheDir, appDir,
      appSysDir,
      containerDir, containerSysDir }) {
    Assert.assertTrue(f.getAbsolutePath() + " doesn't exist!!", f.exists());
    Assert.assertTrue(f.getAbsolutePath() + " is not a directory!!",
        f.isDirectory());
  }
  Assert.assertTrue(targetFile.getAbsolutePath() + " doesn't exist!!",
      targetFile.exists());

  // Now verify the contents of the file
  BufferedReader reader = new BufferedReader(new FileReader(targetFile));
  Assert.assertEquals("Hello World!", reader.readLine());
  Assert.assertEquals(null, reader.readLine());
}
 

@Test
public void testLogAggregationForRealContainerLaunch() throws IOException,
    InterruptedException, YarnException {

  this.containerManager.start();


  File scriptFile = new File(tmpDir, "scriptFile.sh");
  PrintWriter fileWriter = new PrintWriter(scriptFile);
  fileWriter.write("\necho Hello World! Stdout! > "
      + new File(localLogDir, "stdout"));
  fileWriter.write("\necho Hello World! Stderr! > "
      + new File(localLogDir, "stderr"));
  fileWriter.write("\necho Hello World! Syslog! > "
      + new File(localLogDir, "syslog"));
  fileWriter.close();

  ContainerLaunchContext containerLaunchContext =
      recordFactory.newRecordInstance(ContainerLaunchContext.class);
  // ////// Construct the Container-id
  ApplicationId appId = ApplicationId.newInstance(0, 0);
  ApplicationAttemptId appAttemptId =
      BuilderUtils.newApplicationAttemptId(appId, 1);
  ContainerId cId = BuilderUtils.newContainerId(appAttemptId, 0);

  URL resource_alpha =
      ConverterUtils.getYarnUrlFromPath(localFS
          .makeQualified(new Path(scriptFile.getAbsolutePath())));
  LocalResource rsrc_alpha =
      recordFactory.newRecordInstance(LocalResource.class);
  rsrc_alpha.setResource(resource_alpha);
  rsrc_alpha.setSize(-1);
  rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
  rsrc_alpha.setType(LocalResourceType.FILE);
  rsrc_alpha.setTimestamp(scriptFile.lastModified());
  String destinationFile = "dest_file";
  Map<String, LocalResource> localResources = 
      new HashMap<String, LocalResource>();
  localResources.put(destinationFile, rsrc_alpha);
  containerLaunchContext.setLocalResources(localResources);
  List<String> commands = new ArrayList<String>();
  commands.add("/bin/bash");
  commands.add(scriptFile.getAbsolutePath());
  containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(containerLaunchContext,
        TestContainerManager.createContainerToken(
          cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user,
          context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  this.containerManager.startContainers(allRequests);
  
  BaseContainerManagerTest.waitForContainerState(this.containerManager,
      cId, ContainerState.COMPLETE);

  this.containerManager.handle(new CMgrCompletedAppsEvent(Arrays
      .asList(appId), CMgrCompletedAppsEvent.Reason.ON_SHUTDOWN));
  this.containerManager.stop();
}
 

private void testContainerLaunchAndExit(int exitCode) throws IOException,
    InterruptedException, YarnException {

 File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile");
 PrintWriter fileWriter = new PrintWriter(scriptFile);
 File processStartFile =
	  new File(tmpDir, "start_file.txt").getAbsoluteFile();

 // ////// Construct the Container-id
 ContainerId cId = createContainerId(0);

 if (Shell.WINDOWS) {
   fileWriter.println("@echo Hello World!> " + processStartFile);
   fileWriter.println("@echo " + cId + ">> " + processStartFile);
   if (exitCode != 0) {
     fileWriter.println("@exit " + exitCode);
   }
 } else {
   fileWriter.write("\numask 0"); // So that start file is readable by the test
   fileWriter.write("\necho Hello World! > " + processStartFile);
   fileWriter.write("\necho $$ >> " + processStartFile); 
   // Have script throw an exit code at the end
   if (exitCode != 0) {
     fileWriter.write("\nexit "+exitCode);
   }
 }
 
 fileWriter.close();

 ContainerLaunchContext containerLaunchContext = 
	  recordFactory.newRecordInstance(ContainerLaunchContext.class);

 URL resource_alpha =
	  ConverterUtils.getYarnUrlFromPath(localFS
			  .makeQualified(new Path(scriptFile.getAbsolutePath())));
 LocalResource rsrc_alpha =
	  recordFactory.newRecordInstance(LocalResource.class);
 rsrc_alpha.setResource(resource_alpha);
 rsrc_alpha.setSize(-1);
 rsrc_alpha.setVisibility(LocalResourceVisibility.APPLICATION);
 rsrc_alpha.setType(LocalResourceType.FILE);
 rsrc_alpha.setTimestamp(scriptFile.lastModified());
 String destinationFile = "dest_file";
 Map<String, LocalResource> localResources = 
	  new HashMap<String, LocalResource>();
 localResources.put(destinationFile, rsrc_alpha);
 containerLaunchContext.setLocalResources(localResources);
 List<String> commands = Arrays.asList(Shell.getRunScriptCommand(scriptFile));
 containerLaunchContext.setCommands(commands);

  StartContainerRequest scRequest =
      StartContainerRequest.newInstance(
        containerLaunchContext,
        createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(),
          user, context.getContainerTokenSecretManager()));
  List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
  list.add(scRequest);
  StartContainersRequest allRequests =
      StartContainersRequest.newInstance(list);
  containerManager.startContainers(allRequests);

 BaseContainerManagerTest.waitForContainerState(containerManager, cId,
	  ContainerState.COMPLETE);

  List<ContainerId> containerIds = new ArrayList<ContainerId>();
  containerIds.add(cId);
  GetContainerStatusesRequest gcsRequest =
      GetContainerStatusesRequest.newInstance(containerIds);
 ContainerStatus containerStatus = 
	  containerManager.getContainerStatuses(gcsRequest).getContainerStatuses().get(0);

 // Verify exit status matches exit state of script
 Assert.assertEquals(exitCode,
	  containerStatus.getExitStatus());	    
}