org.apache.hadoop.yarn.api.records.LocalResourceType Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResourceType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestResourceRetention.java    From big-c with Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example #2
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 
Example #3
Source File: DagTypeConverters.java    From tez with Apache License 2.0 6 votes vote down vote up
public static List<PlanLocalResource> convertToDAGPlan(Map<String, LocalResource> lrs) {
  List<PlanLocalResource> planLrs = Lists.newArrayListWithCapacity(lrs.size());
  for (Entry<String, LocalResource> entry : lrs.entrySet()) {
    PlanLocalResource.Builder localResourcesBuilder = PlanLocalResource.newBuilder();
    String key = entry.getKey();
    LocalResource lr = entry.getValue();
    localResourcesBuilder.setName(key);
    localResourcesBuilder.setUri(
      DagTypeConverters.convertToDAGPlan(lr.getResource()));
    localResourcesBuilder.setSize(lr.getSize());
    localResourcesBuilder.setTimeStamp(lr.getTimestamp());
    localResourcesBuilder.setType(
      DagTypeConverters.convertToDAGPlan(lr.getType()));
    localResourcesBuilder.setVisibility(
      DagTypeConverters.convertToDAGPlan(lr.getVisibility()));
    if (lr.getType() == LocalResourceType.PATTERN) {
      if (lr.getPattern() == null || lr.getPattern().isEmpty()) {
        throw new TezUncheckedException("LocalResource type set to pattern"
          + " but pattern is null or empty");
      }
      localResourcesBuilder.setPattern(lr.getPattern());
    }
    planLrs.add(localResourcesBuilder.build());
  }
  return planLrs;
}
 
Example #4
Source File: LocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 
Example #5
Source File: TestTezClient.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example #6
Source File: TestFSDownload.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example #7
Source File: TestContainerLocalizer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example #8
Source File: TestFSDownload.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example #9
Source File: JstormOnYarn.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}
 
Example #10
Source File: MRApps.java    From big-c with Apache License 2.0 6 votes vote down vote up
public static void setupDistributedCache( 
    Configuration conf, 
    Map<String, LocalResource> localResources) 
throws IOException {
  
  // Cache archives
  parseDistributedCacheArtifacts(conf, localResources,  
      LocalResourceType.ARCHIVE, 
      DistributedCache.getCacheArchives(conf), 
      DistributedCache.getArchiveTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_ARCHIVES_SIZES), 
      DistributedCache.getArchiveVisibilities(conf));
  
  // Cache files
  parseDistributedCacheArtifacts(conf, 
      localResources,  
      LocalResourceType.FILE, 
      DistributedCache.getCacheFiles(conf),
      DistributedCache.getFileTimestamps(conf),
      getFileSizes(conf, MRJobConfig.CACHE_FILES_SIZES),
      DistributedCache.getFileVisibilities(conf));
}
 
Example #11
Source File: StramClient.java    From attic-apex-core with Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example #12
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 
Example #13
Source File: TestTezLocalCacheManager.java    From tez with Apache License 2.0 6 votes vote down vote up
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example #14
Source File: StramClient.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example #15
Source File: YarnUtils.java    From twill with Apache License 2.0 5 votes vote down vote up
private static YarnLocalResource setLocalResourceType(YarnLocalResource localResource, LocalFile localFile) {
  if (localFile.isArchive()) {
    if (localFile.getPattern() == null) {
      localResource.setType(LocalResourceType.ARCHIVE);
    } else {
      localResource.setType(LocalResourceType.PATTERN);
      localResource.setPattern(localFile.getPattern());
    }
  } else {
    localResource.setType(LocalResourceType.FILE);
  }
  return localResource;
}
 
Example #16
Source File: LocalResourcePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setType(LocalResourceType type) {
  maybeInitBuilder();
  if (type == null) {
    builder.clearType();
    return;
  }
  builder.setType(convertToProtoFormat(type));
}
 
Example #17
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up
public static PlanLocalResourceType convertToDAGPlan(LocalResourceType type) {
  switch(type){
  case ARCHIVE : return PlanLocalResourceType.ARCHIVE;
  case FILE : return PlanLocalResourceType.FILE;
  case PATTERN : return PlanLocalResourceType.PATTERN;
  default : throw new IllegalArgumentException("unknown 'type': " + type);
  }
}
 
Example #18
Source File: YarnService.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private void addRemoteAppFiles(String hdfsFileList, Map<String, LocalResource> resourceMap) throws IOException {
  for (String hdfsFilePath : SPLITTER.split(hdfsFileList)) {
    Path srcFilePath = new Path(hdfsFilePath);
    YarnHelixUtils.addFileAsLocalResource(
        srcFilePath.getFileSystem(this.yarnConfiguration), srcFilePath, LocalResourceType.FILE, resourceMap);
  }
}
 
Example #19
Source File: JobResourceUploader.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
 * Arguments:
 * [0] : Local path for file.
 * [1] : Type for file.
 * [2] : Path of job submission directory
 * [3] : File path for output with details of uploaded resource
 */
public static void main(final String[] args) throws InjectionException, IOException {
  Validate.isTrue(args.length == 4, "Job resource uploader requires 4 args");
  final File localFile = new File(args[0]);
  Validate.isTrue(localFile.exists(), "Local file does not exist " + localFile.getAbsolutePath());
  final String fileType = args[1];
  final String jobSubmissionDirectory = args[2];
  final String localOutputPath = args[3];

  LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " +
      jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
      .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
      .build();

  final JobUploader jobUploader = Tang.Factory.getTang()
      .newInjector(configuration)
      .getInstance(JobUploader.class);
  final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
      .uploadAsLocalResource(localFile, LocalResourceType.valueOf(fileType));

  // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
  final URL resource = localResource.getResource();
  final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
      resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
  LOG.log(Level.INFO, "Writing output: " + outputString);
  try (Writer writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream(localOutputPath), "utf-8"))) {
    writer.write(outputString);
  }

  LOG.log(Level.FINER, "Done writing output file");
}
 
Example #20
Source File: Util.java    From Scribengin with GNU Affero General Public License v3.0 5 votes vote down vote up
public static LocalResource newYarnAppResource(
    FileSystem fs, Path path,
    LocalResourceType type, LocalResourceVisibility vis) throws IOException {
  Path qualified = fs.makeQualified(path);
  FileStatus status = fs.getFileStatus(qualified);
  LocalResource resource = Records.newRecord(LocalResource.class);
  resource.setType(type);
  resource.setVisibility(vis);
  resource.setResource(ConverterUtils.getYarnUrlFromPath(qualified));
  resource.setTimestamp(status.getModificationTime());
  resource.setSize(status.getLen());
  return resource;
}
 
Example #21
Source File: Utils.java    From stratosphere with Apache License 2.0 5 votes vote down vote up
public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException {
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}
 
Example #22
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 5 votes vote down vote up
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example #23
Source File: TestLocalizedResource.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LocalResource createMockResource() {
  // mock rsrc location
  org.apache.hadoop.yarn.api.records.URL uriA =
    mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uriA.getScheme()).thenReturn("file");
  when(uriA.getHost()).thenReturn(null);
  when(uriA.getFile()).thenReturn("/localA/rsrc");

  LocalResource apiRsrc = mock(LocalResource.class);
  when(apiRsrc.getResource()).thenReturn(uriA);
  when(apiRsrc.getTimestamp()).thenReturn(4344L);
  when(apiRsrc.getType()).thenReturn(LocalResourceType.FILE);
  return apiRsrc;
}
 
Example #24
Source File: DagTypeConverters.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public static PlanLocalResourceType convertToDAGPlan(LocalResourceType type) {
  switch(type){
  case ARCHIVE : return PlanLocalResourceType.ARCHIVE;
  case FILE : return PlanLocalResourceType.FILE;
  case PATTERN : return PlanLocalResourceType.PATTERN;
  default : throw new IllegalArgumentException("unknown 'type': " + type);
  }
}
 
Example #25
Source File: TestMRApps.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflicts() throws Exception {
  Configuration conf = new Configuration();
  conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
  
  URI mockUri = URI.create("mockfs://mock/");
  FileSystem mockFs = ((FilterFileSystem)FileSystem.get(mockUri, conf))
      .getRawFileSystem();
  
  URI archive = new URI("mockfs://mock/tmp/something.zip#something");
  Path archivePath = new Path(archive);
  URI file = new URI("mockfs://mock/tmp/something.txt#something");
  Path filePath = new Path(file);
  
  when(mockFs.resolvePath(archivePath)).thenReturn(archivePath);
  when(mockFs.resolvePath(filePath)).thenReturn(filePath);
  
  DistributedCache.addCacheArchive(archive, conf);
  conf.set(MRJobConfig.CACHE_ARCHIVES_TIMESTAMPS, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_SIZES, "10");
  conf.set(MRJobConfig.CACHE_ARCHIVES_VISIBILITIES, "true");
  DistributedCache.addCacheFile(file, conf);
  conf.set(MRJobConfig.CACHE_FILE_TIMESTAMPS, "11");
  conf.set(MRJobConfig.CACHE_FILES_SIZES, "11");
  conf.set(MRJobConfig.CACHE_FILE_VISIBILITIES, "true");
  Map<String, LocalResource> localResources = 
    new HashMap<String, LocalResource>();
  MRApps.setupDistributedCache(conf, localResources);
  
  assertEquals(1, localResources.size());
  LocalResource lr = localResources.get("something");
  //Archive wins
  assertNotNull(lr);
  assertEquals(10l, lr.getSize());
  assertEquals(10l, lr.getTimestamp());
  assertEquals(LocalResourceType.ARCHIVE, lr.getType());
}
 
Example #26
Source File: TestDAGVerify.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testDAGCreateDataInference() {
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  Map<String, LocalResource> lrs1 = Maps.newHashMap();
  String lrName1 = "LR1";
  lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrs2 = Maps.newHashMap();
  String lrName2 = "LR2";
  lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  Set<String> hosts = Sets.newHashSet();
  hosts.add("h1");
  hosts.add("h2");
  List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
  DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"),
      InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
  v1.addDataSource("i1", ds);

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addTaskLocalFiles(lrs1);
  DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
  Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
  VertexPlan vPlan = dagPlan.getVertex(0);
  PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
  Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
  Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
  Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
  Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}
 
Example #27
Source File: Client.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}
 
Example #28
Source File: Utils.java    From flink with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime,
		LocalResourceVisibility resourceVisibility) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(resourceVisibility);
	return localResource;
}
 
Example #29
Source File: UploaderToJobfolder.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Creates a LocalResource instance for the JAR file referenced by the given Path.
 *
 * @param path
 * @return
 * @throws IOException
 */
LocalResource makeLocalResourceForJarFile(final Path path) throws IOException {
  final LocalResource localResource = Records.newRecord(LocalResource.class);
  final FileStatus status = FileContext.getFileContext(this.fileSystem.getUri()).getFileStatus(path);
  localResource.setType(LocalResourceType.ARCHIVE);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(status.getPath()));
  localResource.setTimestamp(status.getModificationTime());
  localResource.setSize(status.getLen());
  return localResource;
}
 
Example #30
Source File: LaunchContainerRunnable.java    From attic-apex-core with Apache License 2.0 5 votes vote down vote up
public static void addFilesToLocalResources(LocalResourceType type, String commaSeparatedFileNames, Map<String, LocalResource> localResources, FileSystem fs) throws IOException
{
  String[] files = StringUtils.splitByWholeSeparator(commaSeparatedFileNames, StramClient.LIB_JARS_SEP);
  for (String file : files) {
    final Path dst = new Path(file);
    addFileToLocalResources(dst.getName(), fs.getFileStatus(dst), type, localResources);
  }
}