Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResourceType

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResourceType. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Bats   Source File: StramClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example 2
Source Project: tez   Source File: TestTezLocalCacheManager.java    License: Apache License 2.0 6 votes vote down vote up
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example 3
Source Project: tez   Source File: DagTypeConverters.java    License: Apache License 2.0 6 votes vote down vote up
public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) {
  List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size());
  for (Entry<String, LocalResource> entry : lrs.entrySet()) {
    PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
    String key = entry.getKey();
    LocalResource lr = entry.getValue();
    localResourcesBuilder.setName(key);
    localResourcesBuilder.setUri(
      DagTypeConverters.convertToDAGPlan(lr.getResource()));
    localResourcesBuilder.setSize(lr.getSize());
    localResourcesBuilder.setTimeStamp(lr.getTimestamp());
    localResourcesBuilder.setType(
      DagTypeConverters.convertToDAGPlan(lr.getType()));
    localResourcesBuilder.setVisibility(
      DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
    if (lr.getType() == LocalResourceType.PATTERN) {
      if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
        throw new TezUncheckedException("LocalResource type set to pattern"
          + " but pattern is null or empty");
      }
      localResourcesBuilder.setPattern(lr.getPattern());
    }
    planLrs.add(localResourcesBuilder.build());
  }
  return planLrs;
}
 
Example 4
Source Project: samza   Source File: LocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 
Example 5
Source Project: hadoop   Source File: TestContainerLocalizer.java    License: Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example 6
Source Project: big-c   Source File: MRApps.java    License: Apache License 2.0 6 votes vote down vote up
public static void setupDistributedCache( 
    Configuration conf, 
    Map<String, LocalResource> localResources) 
throws IOException {
  
  // Cache archives
  parseDistributedCacheArtifacts(conf, localResources,  
      LocalResourceType.ARCHIVE, 
      DistributedCache.getCacheArchives(conf), 
      DistributedCache.getArchiveTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
      DistributedCache.getArchiveVisibilities(conf));
  
  // Cache files
  parseDistributedCacheArtifacts(conf, 
      localResources,  
      LocalResourceType.FILE, 
      DistributedCache.getCacheFiles(conf),
      DistributedCache.getFileTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
      DistributedCache.getFileVisibilities(conf));
}
 
Example 7
Source Project: tez   Source File: TestTezClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example 8
Source Project: incubator-gobblin   Source File: GobblinYarnAppLauncher.java    License: Apache License 2.0 6 votes vote down vote up
private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 
Example 9
Source Project: big-c   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example 10
Source Project: big-c   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example 11
Source Project: jstorm   Source File: JstormOnYarn.java    License: Apache License 2.0 6 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}
 
Example 12
Source Project: attic-apex-core   Source File: StramClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example 13
Source Project: zeppelin   Source File: YarnRemoteInterpreterProcess.java    License: Apache License 2.0 6 votes vote down vote up
private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 
Example 14
Source Project: big-c   Source File: TestResourceRetention.java    License: Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example 15
Source Project: Bats   Source File: LaunchContainerRunnable.java    License: Apache License 2.0 5 votes vote down vote up
public static void addFilesToLocalResources(LocalResourceType type, String commaSeparatedFileNames, Map<String, LocalResource> localResources, FileSystem fs) throws IOException
{
  String[] files = StringUtils.splitByWholeSeparator(commaSeparatedFileNames, StramClient.LIB_JARS_SEP);
  for (String file : files) {
    final Path dst = new Path(file);
    addFileToLocalResources(dst.getName(), fs.getFileStatus(dst), type, localResources);
  }
}
 
Example 16
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 17
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 18
Source Project: XLearning   Source File: Utilities.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type)
    throws IOException {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  FileStatus fileStatus = fs.getFileStatus(path);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
  localResource.setSize(fileStatus.getLen());
  localResource.setTimestamp(fileStatus.getModificationTime());
  localResource.setType(type);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}
 
Example 19
Source Project: big-c   Source File: YARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResource createApplicationResource(FileContext fs, Path p, LocalResourceType type)
    throws IOException {
  LocalResource rsrc = recordFactory.newRecordInstance(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 
Example 20
Source Project: hadoop   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setType(LocalResourceType type) {
  maybeInitBuilder();
  if (type == null) {
    builder.clearType();
    return;
  }
  builder.setType(convertToProtoFormat(type));
}
 
Example 21
Source Project: tez   Source File: TestMockDAGAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
  
  // submit the same DAG again to verify it can be done.
  tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 
Example 22
Source Project: hadoop   Source File: TestFSDownload.java    License: Apache License 2.0 5 votes vote down vote up
static LocalResource createFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException {
  createFile(files, p, len, r);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  ret.setSize(len);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example 23
Source Project: incubator-tez   Source File: AMContainerHelpers.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
public static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example 24
Source Project: tez   Source File: YARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 
Example 25
Source Project: hadoop   Source File: TestFSDownload.java    License: Apache License 2.0 5 votes vote down vote up
static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 
Example 26
Source Project: tez   Source File: DagTypeConverters.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResourceType convertFromDAGPlan(PlanLocalResourceType type) {
  switch(type){
  case ARCHIVE : return LocalResourceType.ARCHIVE;
  case FILE : return LocalResourceType.FILE;
  case PATTERN : return LocalResourceType.PATTERN;
  default : throw new IllegalArgumentException("unknown 'type': " + type);
  }
}
 
Example 27
Source Project: hadoop   Source File: LocalResourceRequest.java    License: Apache License 2.0 5 votes vote down vote up
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
    LocalResourceVisibility visibility, String pattern) {
  this.loc = loc;
  this.timestamp = timestamp;
  this.type = type;
  this.visibility = visibility;
  this.pattern = pattern;
}
 
Example 28
Source Project: hadoop   Source File: TestLocalResourcesTrackerImpl.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResourceRequest createLocalResourceRequest(String user, int i,
    long ts, LocalResourceVisibility vis) {
  final LocalResourceRequest req =
      new LocalResourceRequest(new Path("file:///tmp/" + user + "/rsrc" + i),
          ts + i * 2000, LocalResourceType.FILE, vis, null);
  return req;
}
 
Example 29
Source Project: tez   Source File: TestMockDAGAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 5000)
public void testLocalResourceSetup() throws Exception {
  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  
  MockDAGAppMaster mockApp = tezClient.getLocalClient().getMockApp();
  MockContainerLauncher mockLauncher = mockApp.getContainerLauncher();
  mockLauncher.startScheduling(false);
  
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("testLocalResourceSetup").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  DAGClient dagClient = tezClient.submitDAG(dag);
  mockLauncher.waitTillContainersLaunched();
  ContainerData cData = mockLauncher.getContainers().values().iterator().next();
  ContainerLaunchContext launchContext = cData.launchContext;
  Map<String, LocalResource> taskLR = launchContext.getLocalResources();
  // verify tasks are launched with both DAG and task resources.
  Assert.assertTrue(taskLR.containsKey(lrName1));
  Assert.assertTrue(taskLR.containsKey(lrName2));
  
  mockLauncher.startScheduling(true);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 
Example 30
Source Project: reef   Source File: JobResourceUploader.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
 * Arguments:
 * [0] : Local path for file.
 * [1] : Type for file.
 * [2] : Path of job submission directory
 * [3] : File path for output with details of uploaded resource
 */
public static void main(final String[] args) throws InjectionException, IOException {
  Validate.isTrue(args.length == 4, "Job resource uploader requires 4 args");
  final File localFile = new File(args[0]);
  Validate.isTrue(localFile.exists(), "Local file does not exist " + localFile.getAbsolutePath());
  final String fileType = args[1];
  final String jobSubmissionDirectory = args[2];
  final String localOutputPath = args[3];

  LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " +
      jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
      .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
      .build();

  final JobUploader jobUploader = Tang.Factory.getTang()
      .newInjector(configuration)
      .getInstance(JobUploader.class);
  final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
      .uploadAsLocalResource(localFile, LocalResourceType.valueOf(fileType));

  // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
  final URL resource = localResource.getResource();
  final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
      resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
  LOG.log(Level.INFO, "Writing output: " + outputString);
  try (Writer writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream(localOutputPath), "utf-8"))) {
    writer.write(outputString);
  }

  LOG.log(Level.FINER, "Done writing output file");
}