org.apache.hadoop.yarn.api.records.LocalResourceVisibility Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResourceVisibility. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerLocalizer.java    From big-c with Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example #2
Source File: TestResourceRetention.java    From big-c with Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService()));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example #3
Source File: TestResourceRetention.java    From hadoop with Apache License 2.0 6 votes vote down vote up
LocalResourcesTracker createMockTracker(String user, final long rsrcSize,
    long nRsrcs, long timestamp, long tsstep) {
  Configuration conf = new Configuration();
  ConcurrentMap<LocalResourceRequest,LocalizedResource> trackerResources =
    new ConcurrentHashMap<LocalResourceRequest,LocalizedResource>();
  LocalResourcesTracker ret = spy(new LocalResourcesTrackerImpl(user, null,
    null, trackerResources, false, conf, new NMNullStateStoreService(),null));
  for (int i = 0; i < nRsrcs; ++i) {
    final LocalResourceRequest req = new LocalResourceRequest(
        new Path("file:///" + user + "/rsrc" + i), timestamp + i * tsstep,
        LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, null);
    final long ts = timestamp + i * tsstep;
    final Path p = new Path("file:///local/" + user + "/rsrc" + i);
    LocalizedResource rsrc = new LocalizedResource(req, null) {
      @Override public int getRefCount() { return 0; }
      @Override public long getSize() { return rsrcSize; }
      @Override public Path getLocalPath() { return p; }
      @Override public long getTimestamp() { return ts; }
      @Override
      public ResourceState getState() { return ResourceState.LOCALIZED; }
    };
    trackerResources.put(req, rsrc);
  }
  return ret;
}
 
Example #4
Source File: TestContainer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Verify correct container request events sent to localizer.
 */
@Test
public void testLocalizationRequest() throws Exception {
  WrappedContainer wc = null;
  try {
    wc = new WrappedContainer(7, 314159265358979L, 4344, "yak");
    assertEquals(ContainerState.NEW, wc.c.getContainerState());
    wc.initContainer();

    // Verify request for public/private resources to localizer
    ResourcesRequestedMatcher matchesReq =
        new ResourcesRequestedMatcher(wc.localResources, EnumSet.of(
            LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE,
            LocalResourceVisibility.APPLICATION));
    verify(wc.localizerBus).handle(argThat(matchesReq));
    assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
  }
  finally {
    if (wc != null) {
      wc.finished();
    }
  } 
}
 
Example #5
Source File: TestLocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 
Example #6
Source File: TestFSDownload.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example #7
Source File: ResourceLocalizationService.java    From big-c with Apache License 2.0 6 votes vote down vote up
private Path getPathForLocalization(LocalResource rsrc) throws IOException,
    URISyntaxException {
  String user = context.getUser();
  ApplicationId appId =
      context.getContainerId().getApplicationAttemptId().getApplicationId();
  LocalResourceVisibility vis = rsrc.getVisibility();
  LocalResourcesTracker tracker =
      getLocalResourcesTracker(vis, user, appId);
  String cacheDirectory = null;
  if (vis == LocalResourceVisibility.PRIVATE) {// PRIVATE Only
    cacheDirectory = getUserFileCachePath(user);
  } else {// APPLICATION ONLY
    cacheDirectory = getAppFileCachePath(user, appId.toString());
  }
  Path dirPath =
      dirsHandler.getLocalPathForWrite(cacheDirectory,
        ContainerLocalizer.getEstimatedSize(rsrc), false);
  return tracker.getPathForLocalization(new LocalResourceRequest(rsrc),
      dirPath);
}
 
Example #8
Source File: TestSharedCacheUploader.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * If the localPath does not exists, getActualPath should get to one level
 * down
 */
@Test
public void testGetActualPath() throws Exception {
  Configuration conf = new Configuration();
  conf.setBoolean(YarnConfiguration.SHARED_CACHE_ENABLED, true);
  LocalResource resource = mock(LocalResource.class);
  // give public visibility
  when(resource.getVisibility()).thenReturn(LocalResourceVisibility.PUBLIC);
  Path localPath = new Path("foo.jar");
  String user = "joe";
  SCMUploaderProtocol scmClient = mock(SCMUploaderProtocol.class);
  FileSystem fs = mock(FileSystem.class);
  FileSystem localFs = mock(FileSystem.class);
  // stub it to return a status that indicates a directory
  FileStatus status = mock(FileStatus.class);
  when(status.isDirectory()).thenReturn(true);
  when(localFs.getFileStatus(localPath)).thenReturn(status);
  SharedCacheUploader spied =
      createSpiedUploader(resource, localPath, user, conf, scmClient, fs,
          localFs);

  Path actualPath = spied.getActualPath();
  assertEquals(actualPath.getName(), localPath.getName());
  assertEquals(actualPath.getParent().getName(), localPath.getName());
}
 
Example #9
Source File: TestFSDownload.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example #10
Source File: YarnApplicationFileUploader.java    From flink with Apache License 2.0 6 votes vote down vote up
/**
 * Register all the files in the provided lib directories as Yarn local resources with PUBLIC visibility, which
 * means that they will be cached in the nodes and reused by different applications.
 *
 * @return list of class paths with the file name
 */
List<String> registerProvidedLocalResources() {
	checkNotNull(localResources);

	final ArrayList<String> classPaths = new ArrayList<>();
	providedSharedLibs.forEach(
			(fileName, fileStatus) -> {
				final Path filePath = fileStatus.getPath();
				LOG.debug("Using remote file {} to register local resource", filePath);

				final YarnLocalResourceDescriptor descriptor = YarnLocalResourceDescriptor
						.fromFileStatus(fileName, fileStatus, LocalResourceVisibility.PUBLIC);
				localResources.put(fileName, descriptor.toLocalResource());
				remotePaths.add(filePath);
				envShipResourceList.add(descriptor);

				if (!isFlinkDistJar(filePath.getName()) && !isPlugin(filePath)) {
					classPaths.add(fileName);
				} else if (isFlinkDistJar(filePath.getName())) {
					flinkDist = descriptor;
				}
			});
	return classPaths;
}
 
Example #11
Source File: TestContainer.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Verify correct container request events sent to localizer.
 */
@Test
public void testLocalizationRequest() throws Exception {
  WrappedContainer wc = null;
  try {
    wc = new WrappedContainer(7, 314159265358979L, 4344, "yak");
    assertEquals(ContainerState.NEW, wc.c.getContainerState());
    wc.initContainer();

    // Verify request for public/private resources to localizer
    ResourcesRequestedMatcher matchesReq =
        new ResourcesRequestedMatcher(wc.localResources, EnumSet.of(
            LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE,
            LocalResourceVisibility.APPLICATION));
    verify(wc.localizerBus).handle(argThat(matchesReq));
    assertEquals(ContainerState.LOCALIZING, wc.c.getContainerState());
  }
  finally {
    if (wc != null) {
      wc.finished();
    }
  } 
}
 
Example #12
Source File: LocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 
Example #13
Source File: ContainerImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanup() {
  Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
    new HashMap<LocalResourceVisibility, 
                Collection<LocalResourceRequest>>();
  if (!publicRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
  }
  if (!privateRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
  }
  if (!appRsrcs.isEmpty()) {
    rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
  }
  dispatcher.getEventHandler().handle(
      new ContainerLocalizationCleanupEvent(this, rsrc));
}
 
Example #14
Source File: JstormOnYarn.java    From jstorm with Apache License 2.0 6 votes vote down vote up
private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}
 
Example #15
Source File: IgniteYarnUtils.java    From ignite with Apache License 2.0 6 votes vote down vote up
/**
 * @param file Path.
 * @param fs File system.
 * @param type Local resource type.
 * @throws Exception If failed.
 */
public static LocalResource setupFile(Path file, FileSystem fs, LocalResourceType type)
    throws Exception {
    LocalResource resource = Records.newRecord(LocalResource.class);

    file = fs.makeQualified(file);

    FileStatus stat = fs.getFileStatus(file);

    resource.setResource(ConverterUtils.getYarnUrlFromPath(file));
    resource.setSize(stat.getLen());
    resource.setTimestamp(stat.getModificationTime());
    resource.setType(type);
    resource.setVisibility(LocalResourceVisibility.APPLICATION);

    return resource;
}
 
Example #16
Source File: TestFSDownload.java    From big-c with Apache License 2.0 6 votes vote down vote up
static LocalResource createJarFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException,
    URISyntaxException {
  byte[] bytes = new byte[len];
  r.nextBytes(bytes);

  File archiveFile = new File(p.toUri().getPath() + ".jar");
  archiveFile.createNewFile();
  JarOutputStream out = new JarOutputStream(
      new FileOutputStream(archiveFile));
  out.putNextEntry(new JarEntry(p.getName()));
  out.write(bytes);
  out.closeEntry();
  out.close();

  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(p.toString()
      + ".jar")));
  ret.setSize(len);
  ret.setType(LocalResourceType.ARCHIVE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(new Path(p.toString() + ".jar"))
      .getModificationTime());
  return ret;
}
 
Example #17
Source File: TestTezClient.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example #18
Source File: TestContainer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyCleanupCall(WrappedContainer wc) throws Exception {
  ResourcesReleasedMatcher matchesReq =
      new ResourcesReleasedMatcher(wc.localResources, EnumSet.of(
          LocalResourceVisibility.PUBLIC, LocalResourceVisibility.PRIVATE,
          LocalResourceVisibility.APPLICATION));
  verify(wc.localizerBus).handle(argThat(matchesReq));
}
 
Example #19
Source File: LocalResourceRequest.java    From big-c with Apache License 2.0 5 votes vote down vote up
LocalResourceRequest(Path loc, long timestamp, LocalResourceType type,
    LocalResourceVisibility visibility, String pattern) {
  this.loc = loc;
  this.timestamp = timestamp;
  this.type = type;
  this.visibility = visibility;
  this.pattern = pattern;
}
 
Example #20
Source File: TestLocalResource.java    From hadoop with Apache License 2.0 5 votes vote down vote up
static org.apache.hadoop.yarn.api.records.LocalResource getYarnResource(Path p, long size,
    long timestamp, LocalResourceType type, LocalResourceVisibility state, String pattern)
    throws URISyntaxException {
  org.apache.hadoop.yarn.api.records.LocalResource ret = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(org.apache.hadoop.yarn.api.records.LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromURI(p.toUri()));
  ret.setSize(size);
  ret.setTimestamp(timestamp);
  ret.setType(type);
  ret.setVisibility(state);
  ret.setPattern(pattern);
  return ret;
}
 
Example #21
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 5 votes vote down vote up
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example #22
Source File: TestFSDownload.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LocalResource createFile(FileContext files, Path p, int len,
    Random r, LocalResourceVisibility vis) throws IOException {
  createFile(files, p, len, r);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  ret.setSize(len);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(vis);
  ret.setTimestamp(files.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example #23
Source File: TestDAGVerify.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testDAGCreateDataInference() {
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  Map<String, LocalResource> lrs1 = Maps.newHashMap();
  String lrName1 = "LR1";
  lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrs2 = Maps.newHashMap();
  String lrName2 = "LR2";
  lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  Set<String> hosts = Sets.newHashSet();
  hosts.add("h1");
  hosts.add("h2");
  List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
  DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"),
      InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
  v1.addDataSource("i1", ds);

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addTaskLocalFiles(lrs1);
  DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
  Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
  VertexPlan vPlan = dagPlan.getVertex(0);
  PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
  Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
  Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
  Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
  Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}
 
Example #24
Source File: YarnHelixUtils.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
/**
 * Add a file as a Yarn {@link org.apache.hadoop.yarn.api.records.LocalResource}.
 *
 * @param fs a {@link FileSystem} instance
 * @param destFilePath the destination file path
 * @param resourceType the {@link org.apache.hadoop.yarn.api.records.LocalResourceType} of the file
 * @param resourceMap a {@link Map} of file names to their corresponding
 *                    {@link org.apache.hadoop.yarn.api.records.LocalResource}s
 * @throws IOException if there's something wrong adding the file as a
 *                     {@link org.apache.hadoop.yarn.api.records.LocalResource}
 */
public static void addFileAsLocalResource(FileSystem fs, Path destFilePath, LocalResourceType resourceType,
    Map<String, LocalResource> resourceMap) throws IOException {
  LocalResource fileResource = Records.newRecord(LocalResource.class);
  FileStatus fileStatus = fs.getFileStatus(destFilePath);
  fileResource.setResource(ConverterUtils.getYarnUrlFromPath(destFilePath));
  fileResource.setSize(fileStatus.getLen());
  fileResource.setTimestamp(fileStatus.getModificationTime());
  fileResource.setType(resourceType);
  fileResource.setVisibility(LocalResourceVisibility.APPLICATION);
  resourceMap.put(destFilePath.getName(), fileResource);
}
 
Example #25
Source File: TestLocalizedResource.java    From big-c with Apache License 2.0 5 votes vote down vote up
public LocalizerEventMatcher(ContainerId idRef, Credentials creds,
    LocalResourceVisibility vis, LocalizerEventType type) {
  this.vis = vis;
  this.type = type;
  this.creds = creds;
  this.idRef = idRef;
}
 
Example #26
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
 
Example #27
Source File: LocalResourcePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setVisibility(LocalResourceVisibility visibility) {
  maybeInitBuilder();
  if (visibility == null) {
    builder.clearVisibility();
    return;
  }
  builder.setVisibility(convertToProtoFormat(visibility));
}
 
Example #28
Source File: TestContainer.java    From big-c with Apache License 2.0 5 votes vote down vote up
ResourcesReleasedMatcher(Map<String, LocalResource> allResources,
    EnumSet<LocalResourceVisibility> vis) throws URISyntaxException {
  for (Entry<String, LocalResource> e : allResources.entrySet()) {
    if (vis.contains(e.getValue().getVisibility())) {
      resources.add(new LocalResourceRequest(e.getValue()));
    }
  }
}
 
Example #29
Source File: LocalizerResourceRequestEvent.java    From big-c with Apache License 2.0 5 votes vote down vote up
public LocalizerResourceRequestEvent(LocalizedResource resource,
    LocalResourceVisibility vis, LocalizerContext context, String pattern) {
  super(LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION,
      ConverterUtils.toString(context.getContainerId()));
  this.vis = vis;
  this.context = context;
  this.resource = resource;
  this.pattern = pattern;
}
 
Example #30
Source File: TestContainer.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}