Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResource

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResource. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: TestPBRecordImpl.java    License: Apache License 2.0 6 votes vote down vote up
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 
Example 2
Source Project: Bats   Source File: StramClient.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example 3
/**
 * Uploads and registers a single resource and adds it to <tt>localResources</tt>.
 *
 * @param key
 * 		the key to add the resource under
 * @param fs
 * 		the remote file system to upload to
 * @param appId
 * 		application ID
 * @param localSrcPath
 * 		local path to the file
 * @param localResources
 * 		map of resources
 *
 * @return the remote path to the uploaded resource
 */
private static Path setupSingleLocalResource(
		String key,
		FileSystem fs,
		ApplicationId appId,
		Path localSrcPath,
		Map<String, LocalResource> localResources,
		Path targetHomeDir,
		String relativeTargetPath) throws IOException, URISyntaxException {

	Tuple2<Path, LocalResource> resource = Utils.setupLocalResource(
		fs,
		appId.toString(),
		localSrcPath,
		targetHomeDir,
		relativeTargetPath);

	localResources.put(key, resource.f1);

	return resource.f0;
}
 
Example 4
Source Project: hadoop   Source File: TestFSDownload.java    License: Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example 5
Source Project: reef   Source File: GlobalJarUploader.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates the JAR file with the global files on the driver and then uploads it to the job folder on
 * (H)DFS.
 *
 * @return the map to be used as the "global" resources when submitting Evaluators.
 * @throws IOException if the creation of the JAR or the upload fails
 */
@Override
public synchronized Map<String, LocalResource> call() throws IOException {
  final Map<String, LocalResource> globalResources = new HashMap<>(1);
  if (!this.isUploaded){
    this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
    this.isUploaded = true;
  }

  final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);

  if (this.globalJarResource != null
      && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
    LOG.log(Level.WARNING,
            "The global JAR LocalResource timestamp has been changed from "
            + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
  }

  this.globalJarResource = updatedGlobalJarResource;

  // For now, always rewrite the information due to REEF-348
  globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);

  return globalResources;
}
 
Example 6
Source Project: samza   Source File: TestLocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 
Example 7
Source Project: hadoop   Source File: TestContainerLocalizer.java    License: Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example 8
Source Project: samza   Source File: LocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 
Example 9
Source Project: incubator-tez   Source File: ContainerContext.java    License: Apache License 2.0 6 votes vote down vote up
private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
    Map<String, LocalResource> reqLRs) {
  Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
  for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
    LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
    if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cannot match container: Attempting to use same target resource name: "
            + srcLREntry.getKey()
            + ", but with different source resources. Already localized: "
            + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
      }
      return false;
    }
  }
  for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
    LocalResource lr = additionalLREntry.getValue();
    if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
      return false;
    }
  }
  return true;
}
 
Example 10
Source Project: incubator-gobblin   Source File: GobblinYarnAppLauncher.java    License: Apache License 2.0 6 votes vote down vote up
private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 
Example 11
Source Project: zeppelin   Source File: YarnRemoteInterpreterProcess.java    License: Apache License 2.0 6 votes vote down vote up
private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 
Example 12
Source Project: tez   Source File: TestContainerReuse.java    License: Apache License 2.0 5 votes vote down vote up
private AMSchedulerEventTALaunchRequest createLaunchRequestEvent(TezTaskAttemptID taID,
    TaskAttempt ta, Resource capability, String[] hosts, String[] racks, Priority priority,
    String jvmOpts, Map<String, String> environment) {
  return createLaunchRequestEvent(taID, ta, capability, hosts, racks, priority,
      new ContainerContext(new HashMap<String, LocalResource>(), new Credentials(), environment,
          jvmOpts));
}
 
Example 13
Source Project: big-c   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource getMockedResource(Random r, 
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = getPath("/local/PRIVATE/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return rsrc;
}
 
Example 14
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	LocalResource localResource = Records.newRecord(LocalResource.class);
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
	localResource.setSize(jarStat.getLen());
	localResource.setTimestamp(jarStat.getModificationTime());
	localResource.setType(LocalResourceType.FILE);
	localResource.setVisibility(LocalResourceVisibility.APPLICATION);
	return localResource;
}
 
Example 15
Source Project: sylph   Source File: YarnJobDescriptor.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath)
        throws IOException
{
    LocalResource localResource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    return localResource;
}
 
Example 16
Source Project: tez   Source File: TezCommonUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static void addAdditionalLocalResources(Map<String, LocalResource> additionalLrs,
    Map<String, LocalResource> originalLRs, String logContext) {
  // TODO TEZ-1798. Handle contents of Tez archives for duplicate LocalResource checks
  if (additionalLrs != null && !additionalLrs.isEmpty()) {
    StringBuilder sb = new StringBuilder();
    for (Map.Entry<String, LocalResource> lrEntry : additionalLrs.entrySet()) {
      LocalResource originalLr = originalLRs.get(lrEntry.getKey());
      if (originalLr != null) {
        LocalResource additionalLr = lrEntry.getValue();
        if (originalLr.getSize() != additionalLr.getSize()) {
          throw new TezUncheckedException(
              "Duplicate Resources found with different size for [" + logContext + "]: " + lrEntry.getKey() +
                  " : " + "[" + additionalLr.getResource() + "=" + additionalLr.getSize() +
                  "],[" + originalLr.getResource() + "=" + originalLr.getSize());
        } else {
          if (originalLr.getResource().equals(additionalLr.getResource())) {
            sb.append("[").append(lrEntry.getKey()).append(" : Duplicate]");
          } else {
            sb.append("[").append(lrEntry.getKey()).append(" : DuplicateDifferentPath]");
          }
        }
      }
      // The LR either does not exist, or is an 'equivalent' dupe.
      // Prefer the tez specified LR instead of the equivalent user specified LR for container reuse matching
      originalLRs.put(lrEntry.getKey(), lrEntry.getValue());
    }
    String logString = sb.toString();
    if (!logString.isEmpty()) {
      LOG.warn("Found Resources Duplication in " + logContext + " after including resources from " +
          TezConfiguration.TEZ_LIB_URIS + " and " + TezConfiguration.TEZ_AUX_URIS + ": " +
          logString);
    }
  }
}
 
Example 17
Source Project: reef   Source File: EvaluatorSetupHelper.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Sets up the LocalResources for a new Evaluator.
 *
 * @param resourceLaunchEvent
 * @return
 * @throws IOException
 */
Map<String, LocalResource> getResources(
    final ResourceLaunchEvent resourceLaunchEvent)
    throws IOException {

  final Map<String, LocalResource> result = new HashMap<>();
  result.putAll(getGlobalResources());

  final File localStagingFolder = this.tempFileCreator.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix());

  // Write the configuration
  final File configurationFile = new File(localStagingFolder, this.fileNames.getEvaluatorConfigurationName());
  this.configurationSerializer.toFile(makeEvaluatorConfiguration(resourceLaunchEvent), configurationFile);

  // Copy files to the staging folder
  JobJarMaker.copy(resourceLaunchEvent.getFileSet(), localStagingFolder);

  // Make a JAR file out of it
  final File localFile = tempFileCreator.createTempFile(
      this.fileNames.getEvaluatorFolderPrefix(), this.fileNames.getJarFileSuffix());
  new JARFileMaker(localFile).addChildren(localStagingFolder).close();

  // Upload the JAR to the job folder
  final Path pathToEvaluatorJar = this.uploader.uploadToJobFolder(localFile);
  result.put(this.fileNames.getLocalFolderPath(), this.uploader.makeLocalResourceForJarFile(pathToEvaluatorJar));

  if (this.deleteTempFiles) {
    LOG.log(Level.FINE, "Marking [{0}] for deletion at the exit of this JVM and deleting [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
    localFile.deleteOnExit();
    if (!localStagingFolder.delete()) {
      LOG.log(Level.WARNING, "Failed to delete [{0}]", localStagingFolder.getAbsolutePath());
    }
  } else {
    LOG.log(Level.FINE, "The evaluator staging folder will be kept at [{0}], the JAR at [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
  }
  return result;
}
 
Example 18
Source Project: big-c   Source File: LocalResourceStatusPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setResource(LocalResource resource) {
  maybeInitBuilder();
  if (resource == null)
    builder.clearResource();
  this.resource = resource;
}
 
Example 19
Source Project: TensorFlowOnYARN   Source File: LaunchContainerThread.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void run() {
  try {
    Map<String, String> env = Utils.setJavaEnv(appMaster.getConfiguration());
    String current = ApplicationConstants.Environment.LD_LIBRARY_PATH.$$();
    env.put("LD_LIBRARY_PATH", current + ":" + "`pwd`");

    Map<String, Path> files = new HashMap<>();
    files.put(Constants.TF_JAR_NAME, new Path(tfJar));
    files.put(Constants.TF_LIB_NAME, new Path(tfLib));

    FileSystem fs = FileSystem.get(appMaster.getConfiguration());
    Map<String, LocalResource> localResources =
        Utils.makeLocalResources(fs, files);

    String command = makeContainerCommand(
        containerMemory, clusterSpec.toBase64EncodedJsonString(),
        taskInfo.jobName, taskInfo.taskIndex);

    LOG.info("Launching a new container."
        + ", containerId=" + container.getId()
        + ", containerNode=" + container.getNodeId().getHost()
        + ":" + container.getNodeId().getPort()
        + ", containerNodeURI=" + container.getNodeHttpAddress()
        + ", containerResourceMemory="
        + container.getResource().getMemorySize()
        + ", containerResourceVirtualCores="
        + container.getResource().getVirtualCores()
        + ", command: " + command);
    ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
        localResources, env, Lists.newArrayList(command), null, null, null, null);
    appMaster.addContainer(container);
    appMaster.getNMClientAsync().startContainerAsync(container, ctx);
  } catch (Exception e) {
    throw new RuntimeException(e);
  }
}
 
Example 20
Source Project: incubator-tez   Source File: ContainerContext.java    License: Apache License 2.0 5 votes vote down vote up
public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts,
    @Nullable Vertex vertex) {
  Preconditions.checkNotNull(localResources,
      "localResources should not be null");
  Preconditions.checkNotNull(credentials, "credentials should not be null");
  Preconditions.checkNotNull(environment, "environment should not be null");
  Preconditions.checkNotNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = vertex;
}
 
Example 21
Source Project: hadoop   Source File: ContainerLaunchContextPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
private void initLocalResources() {
  if (this.localResources != null) {
    return;
  }
  ContainerLaunchContextProtoOrBuilder p = viaProto ? proto : builder;
  List<StringLocalResourceMapProto> list = p.getLocalResourcesList();
  this.localResources = new HashMap<String, LocalResource>();

  for (StringLocalResourceMapProto c : list) {
    this.localResources.put(c.getKey(), convertFromProtoFormat(c.getValue()));
  }
}
 
Example 22
public void addFile(FileStatus fstatus) {
  Path fpath = fstatus.getPath() ;
  LocalResource libJar = Records.newRecord(LocalResource.class);
  libJar.setResource(ConverterUtils.getYarnUrlFromPath(fpath));
  libJar.setSize(fstatus.getLen());
  libJar.setTimestamp(fstatus.getModificationTime());
  libJar.setType(LocalResourceType.FILE);
  libJar.setVisibility(LocalResourceVisibility.PUBLIC);
  put(fpath.getName(), libJar) ;
}
 
Example 23
Source Project: flink   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Creates a YARN resource for the remote object at the given location.
 * @param fs remote filesystem
 * @param remoteRsrcPath resource path to be registered
 * @return YARN resource
 */
private static LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath) throws IOException {
	FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
	return registerLocalResource(
		remoteRsrcPath,
		jarStat.getLen(),
		jarStat.getModificationTime(),
		LocalResourceVisibility.APPLICATION);
}
 
Example 24
Source Project: big-c   Source File: ContainerLaunchContextPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setLocalResources(
    final Map<String, LocalResource> localResources) {
  if (localResources == null)
    return;
  initLocalResources();
  this.localResources.clear();
  this.localResources.putAll(localResources);
}
 
Example 25
Source Project: tez   Source File: DagTypeConverters.java    License: Apache License 2.0 5 votes vote down vote up
public static PlanLocalResourcesProto convertFromLocalResources(
  Map<String, LocalResource> localResources) {
  PlanLocalResourcesProto.Builder builder =
    PlanLocalResourcesProto.newBuilder();
  for (Map.Entry<String, LocalResource> entry : localResources.entrySet()) {
    PlanLocalResource plr = convertLocalResourceToPlanLocalResource(
        entry.getKey(), entry.getValue());
    builder.addLocalResources(plr);
  }
  return builder.build();
}
 
Example 26
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 
Example 27
Source Project: incubator-gobblin   Source File: YarnService.java    License: Apache License 2.0 5 votes vote down vote up
private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
  if (!this.fs.exists(destDir)) {
    LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
    return;
  }

  FileStatus[] statuses = this.fs.listStatus(destDir);
  if (statuses != null) {
    for (FileStatus status : statuses) {
      YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
    }
  }
}
 
Example 28
Source Project: tez   Source File: TestTezClientUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 *
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  StringBuilder tezLibUris = new StringBuilder();

  // Create 2 files
  Path topDir = new Path(TEST_ROOT_DIR, "validatemultipletarballs");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);

  Path tarFile1 = new Path(topDir, "f1.tar.gz");
  Path tarFile2 = new Path(topDir, "f2.tar.gz");

  Assert.assertTrue(localFs.createNewFile(tarFile1));
  Assert.assertTrue(localFs.createNewFile(tarFile2));
  tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(",");
  tezLibUris.append(localFs.makeQualified(tarFile2).toString()).append("#tar2").append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(2, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("tar1"));
  Assert.assertTrue(resourceNames.contains("tar2"));
  Assert.assertFalse(resourceNames.contains("f1.tar.gz"));
  Assert.assertFalse(resourceNames.contains("f2.tar.gz"));


  Assert.assertTrue(localFs.delete(tarFile1, true));
  Assert.assertTrue(localFs.delete(tarFile2, true));
  Assert.assertTrue(localFs.delete(topDir, true));
}
 
Example 29
Source Project: big-c   Source File: TestContainer.java    License: Apache License 2.0 5 votes vote down vote up
private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 
Example 30
Source Project: tez   Source File: YARNRunner.java    License: Apache License 2.0 5 votes vote down vote up
private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}