Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResourceVisibility

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResourceVisibility. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: big-c   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example 2
Source Project: samza   Source File: TestLocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 
Example 3
Source Project: big-c   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example 4
Source Project: hadoop   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example 5
Source Project: flink   Source File: YarnApplicationFileUploader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Register all the files in the provided lib directories as Yarn local resources with PUBLIC visibility, which
 * means that they will be cached in the nodes and reused by different applications.
 *
 * @return list of class paths with the file name
 */
List<String> registerProvidedLocalResources() {
	checkNotNull(localResources);

	final ArrayList<String> classPaths = new ArrayList<>();
	providedSharedLibs.forEach(
			(fileName, fileStatus) -> {
				final Path filePath = fileStatus.getPath();
				LOG.debug("Using remote file {} to register local resource", filePath);

				final YarnLocalResourceDescriptor descriptor = YarnLocalResourceDescriptor
						.fromFileStatus(fileName, fileStatus, LocalResourceVisibility.PUBLIC);
				localResources.put(fileName, descriptor.toLocalResource());
				remotePaths.add(filePath);
				envShipResourceList.add(descriptor);

				if (!isFlinkDistJar(filePath.getName()) && !isPlugin(filePath)) {
					classPaths.add(fileName);
				} else if (isFlinkDistJar(filePath.getName())) {
					flinkDist = descriptor;
				}
			});
	return classPaths;
}
 
Example 6
Source Project: samza   Source File: LocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 
Example 7
Source Project: ignite   Source File: IgniteYarnUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * @param file Path.
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
    throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);

    file = fs.makeQualified(file);

    FileStatus stat = fs.getFileStatus(file);

    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);

    return resource;
}
 
Example 8
Source Project: tez   Source File: TestTezClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example 9
Source Project: big-c   Source File: TestContainerLocalizer.java    License: Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example 10
Source Project: jstorm   Source File: JstormOnYarn.java    License: Apache License 2.0 6 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}
 
Example 11
Source Project: big-c   Source File: TestContainer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify correct container request events sent to localizer.
 */
@Test
public void testLocalizationRequest() throws Exception {
  WrappedContainer wc = null;
  try {
    wc = new WrappedContainer(7, 314159265358979L, 4344, "yak");
    assertEquals(ContainerState.NEW, wc.c.getContainerState());
    wc.initContainer();

    // Verify request for public/private resources to localizer
    ResourcesRequestedMatcher matchesReq =
        new ResourcesRequestedMatcher(wc.localResources, EnumSet.of(
            LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE,
            LocalResourceVisibility.APPLICATION));
    verify(wc.localizerBus).handle(argThat(matchesReq));
    assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
  }
  finally {
    if (wc != null) {
      wc.finished();
    }
  } 
}
 
Example 12
Source Project: hadoop   Source File: ContainerImpl.java    License: Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanup() {
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
    new HashMap<LocalResourceVisibility, 
                Collection<LocalResourceRequest>>();
  if (!publicRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
  }
  if (!privateRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
  }
  if (!appRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
  }
  dispatcher.getEventHandler().handle(
      new ContainerLocalizationCleanupEvent(this, rsrc));
}
 
Example 13
Source Project: hadoop   Source File: TestSharedCacheUploader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * If the localPath does not exists, getActualPath should get to one level
 * down
 */
@Test
public void testGetActualPath() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = new Path("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = mock(FileSystem.class);
  // stub it to return a status that indicates a directory
  FileStatus status = mock(FileStatus.class);
  when(status.isDirectory()).thenReturn(true);
  when(localFs.getFileStatus(localPath)).thenReturn(status);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  Path actualPath = spied.getActualPath();
  assertEquals(actualPath.getName(), localPath.getName());
  assertEquals(actualPath.getParent().getName(), localPath.getName());
}
 
Example 14
Source Project: big-c   Source File: ResourceLocalizationService.java    License: Apache License 2.0 6 votes vote down vote up
private Path getPathForLocalization(LocalResource rsrc) throws IOException,
    URISyntaxException {
  String user = context.getUser();
  ApplicationId appId =
      context.getContainerId().getApplicationAttemptId().getApplicationId();
  LocalResourceVisibility vis = rsrc.getVisibility();
  LocalResourcesTracker tracker =
      getLocalResourcesTracker(vis, user, appId);
  String cacheDirectory = null;
  if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only
    cacheDirectory = getUserFileCachePath(user);
  } else {// APPLICATION ONLY
    cacheDirectory = getAppFileCachePath(user, appId.toString());
  }
  Path dirPath =
      dirsHandler.getLocalPathForWrite(cacheDirectory,
        ContainerLocalizer.getEstimatedSize(rsrc), false);
  return tracker.getPathForLocalization(new LocalResourceRequest(rsrc),
      dirPath);
}
 
Example 15
Source Project: hadoop   Source File: TestResourceRetention.java    License: Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService(),null));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example 16
Source Project: hadoop   Source File: TestContainer.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Verify correct container request events sent to localizer.
 */
@Test
public void testLocalizationRequest() throws Exception {
  WrappedContainer wc = null;
  try {
    wc = new WrappedContainer(7, 314159265358979L, 4344, "yak");
    assertEquals(ContainerState.NEW, wc.c.getContainerState());
    wc.initContainer();

    // Verify request for public/private resources to localizer
    ResourcesRequestedMatcher matchesReq =
        new ResourcesRequestedMatcher(wc.localResources, EnumSet.of(
            LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE,
            LocalResourceVisibility.APPLICATION));
    verify(wc.localizerBus).handle(argThat(matchesReq));
    assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
  }
  finally {
    if (wc != null) {
      wc.finished();
    }
  } 
}
 
Example 17
Source Project: big-c   Source File: TestResourceRetention.java    License: Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example 18
Source Project: Flink-CEPplus   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 19
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 *
 * @param remoteRsrcPath	remote location of the resource
 * @param resourceSize		size of the resource
 * @param resourceModificationTime last modification time of the resource
 *
 * @return YARN resource
 */
private static LocalResource registerLocalResource(
		Path remoteRsrcPath,
		long resourceSize,
		long resourceModificationTime) {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(resourceSize);
	localResource.setTimestamp(resourceModificationTime);
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 20
Source Project: big-c   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
    ContainerImpl container, LocalResourceVisibility vis,
    LocalResourceRequest req) {
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> reqs =
      new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
  List<LocalResourceRequest> resourceList =
      new ArrayList<LocalResourceRequest>();
  resourceList.add(req);
  reqs.put(vis, resourceList);
  return new ContainerLocalizationRequestEvent(container, reqs);
}
 
Example 21
Source Project: stratosphere   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
public static void registerLocalResource(FileSystem fs, Path remoteRsrcPath, LocalResource localResource) throws IOException {
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.PUBLIC);
}
 
Example 22
Source Project: AthenaX   Source File: AthenaXYarnClusterDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException {
  FileSystem fs = path.getFileSystem(clusterConf.conf());
  FileStatus stat = fs.getFileStatus(path);
  return LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromPath(path),
          LocalResourceType.FILE,
          visibility,
          stat.getLen(), stat.getModificationTime()
  );
}
 
Example 23
Source Project: XLearning   Source File: Utilities.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResource createApplicationResource(FileSystem fs, Path path, LocalResourceType type)
    throws IOException {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  FileStatus fileStatus = fs.getFileStatus(path);
  localResource.setResource(ConverterUtils.getYarnUrlFromPath(path));
  localResource.setSize(fileStatus.getLen());
  localResource.setTimestamp(fileStatus.getModificationTime());
  localResource.setType(type);
  localResource.setVisibility(LocalResourceVisibility.APPLICATION);
  return localResource;
}
 
Example 24
Source Project: TensorFlowOnYARN   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}
 
Example 25
Source Project: hadoop   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setVisibility(LocalResourceVisibility visibility) {
  maybeInitBuilder();
  if (visibility == null) {
    builder.clearVisibility();
    return;
  }
  builder.setVisibility(convertToProtoFormat(visibility));
}
 
Example 26
public void addFile(FileStatus fstatus) {
  Path fpath = fstatus.getPath() ;
  LocalResource libJar = Records.newRecord(LocalResource.class);
  libJar.setResource(ConverterUtils.getYarnUrlFromPath(fpath));
  libJar.setSize(fstatus.getLen());
  libJar.setTimestamp(fstatus.getModificationTime());
  libJar.setType(LocalResourceType.FILE);
  libJar.setVisibility(LocalResourceVisibility.PUBLIC);
  put(fpath.getName(), libJar) ;
}
 
Example 27
Source Project: big-c   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized LocalResourceVisibility getVisibility() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (!p.hasVisibility()) {
    return null;
  }
  return convertFromProtoFormat(p.getVisibility());
}
 
Example 28
Source Project: hadoop   Source File: TestFSDownload.java    License: Apache License 2.0 5 votes vote down vote up
static LocalResource createTgzFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File gzipFile = new File(p.toUri().getPath() + ".tar.gz");
  gzipFile.createNewFile();
  TarArchiveOutputStream out = new TarArchiveOutputStream(
      new GZIPOutputStream(new FileOutputStream(gzipFile)));
  TarArchiveEntry entry = new TarArchiveEntry(p.getName());
  entry.setSize(bytes.length);
  out.putArchiveEntry(entry);
  out.write(bytes);
  out.closeArchiveEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".tar.gz")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".tar.gz"))
      .getModificationTime());
  return ret;
}
 
Example 29
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 
Example 30
Source Project: big-c   Source File: TestFSDownload.java    License: Apache License 2.0 5 votes vote down vote up
private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}