org.apache.hadoop.yarn.api.records.LocalResource Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResource. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestContainerLocalizer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example #2
Source File: GobblinYarnAppLauncher.java    From incubator-gobblin with Apache License 2.0 6 votes vote down vote up
private void addAppLocalFiles(String localFilePathList, Optional<Map<String, LocalResource>> resourceMap,
    Path destDir, FileSystem localFs) throws IOException {

  for (String localFilePath : SPLITTER.split(localFilePathList)) {
    Path srcFilePath = new Path(localFilePath);
    Path destFilePath = new Path(destDir, srcFilePath.getName());
    if (localFs.exists(srcFilePath)) {
      this.fs.copyFromLocalFile(srcFilePath, destFilePath);
      if (resourceMap.isPresent()) {
        YarnHelixUtils.addFileAsLocalResource(this.fs, destFilePath, LocalResourceType.FILE, resourceMap.get());
      }
    } else {
      LOGGER.warn(String.format("The request file %s doesn't exist", srcFilePath));
    }
  }
}
 
Example #3
Source File: TestPBRecordImpl.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static LocalizerHeartbeatResponse createLocalizerHeartbeatResponse() 
    throws URISyntaxException {
  LocalizerHeartbeatResponse ret =
    recordFactory.newRecordInstance(LocalizerHeartbeatResponse.class);
  assertTrue(ret instanceof LocalizerHeartbeatResponsePBImpl);
  ret.setLocalizerAction(LocalizerAction.LIVE);
  LocalResource rsrc = createResource();
  ArrayList<ResourceLocalizationSpec> rsrcs =
    new ArrayList<ResourceLocalizationSpec>();
  ResourceLocalizationSpec resource =
    recordFactory.newRecordInstance(ResourceLocalizationSpec.class);
  resource.setResource(rsrc);
  resource.setDestinationDirectory(ConverterUtils
    .getYarnUrlFromPath(new Path("/tmp" + System.currentTimeMillis())));
  rsrcs.add(resource);
  ret.setResourceSpecs(rsrcs);
  System.out.println(resource);
  return ret;
}
 
Example #4
Source File: YarnRemoteInterpreterProcess.java    From zeppelin with Apache License 2.0 6 votes vote down vote up
private void addResource(
        FileSystem fs,
        Path destPath,
        Map<String, LocalResource> localResources,
        LocalResourceType resourceType,
        String link) throws IOException {

  FileStatus destStatus = fs.getFileStatus(destPath);
  LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
  amJarRsrc.setType(resourceType);
  amJarRsrc.setVisibility(LocalResourceVisibility.PUBLIC);
  amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(destPath));
  amJarRsrc.setTimestamp(destStatus.getModificationTime());
  amJarRsrc.setSize(destStatus.getLen());
  localResources.put(link, amJarRsrc);
}
 
Example #5
Source File: ContainerContext.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
private static boolean localResourcesCompatible(Map<String, LocalResource> srcLRs,
    Map<String, LocalResource> reqLRs) {
  Map<String, LocalResource> reqLRsCopy = new HashMap<String, LocalResource>(reqLRs);
  for (Entry<String, LocalResource> srcLREntry : srcLRs.entrySet()) {
    LocalResource requestedLocalResource = reqLRsCopy.remove(srcLREntry.getKey());
    if (requestedLocalResource != null && !srcLREntry.getValue().equals(requestedLocalResource)) {
      if (LOG.isDebugEnabled()) {
        LOG.debug("Cannot match container: Attempting to use same target resource name: "
            + srcLREntry.getKey()
            + ", but with different source resources. Already localized: "
            + srcLREntry.getValue() + ", requested: " + requestedLocalResource);
      }
      return false;
    }
  }
  for (Entry<String, LocalResource> additionalLREntry : reqLRsCopy.entrySet()) {
    LocalResource lr = additionalLREntry.getValue();
    if (EnumSet.of(LocalResourceType.ARCHIVE, LocalResourceType.PATTERN).contains(lr.getType())) {
      return false;
    }
  }
  return true;
}
 
Example #6
Source File: GlobalJarUploader.java    From reef with Apache License 2.0 6 votes vote down vote up
/**
 * Creates the JAR file with the global files on the driver and then uploads it to the job folder on
 * (H)DFS.
 *
 * @return the map to be used as the "global" resources when submitting Evaluators.
 * @throws IOException if the creation of the JAR or the upload fails
 */
@Override
public synchronized Map<String, LocalResource> call() throws IOException {
  final Map<String, LocalResource> globalResources = new HashMap<>(1);
  if (!this.isUploaded){
    this.pathToGlobalJar = this.uploader.uploadToJobFolder(makeGlobalJar());
    this.isUploaded = true;
  }

  final LocalResource updatedGlobalJarResource = this.uploader.makeLocalResourceForJarFile(this.pathToGlobalJar);

  if (this.globalJarResource != null
      && this.globalJarResource.getTimestamp() != updatedGlobalJarResource.getTimestamp()) {
    LOG.log(Level.WARNING,
            "The global JAR LocalResource timestamp has been changed from "
            + this.globalJarResource.getTimestamp() + " to " + updatedGlobalJarResource.getTimestamp());
  }

  this.globalJarResource = updatedGlobalJarResource;

  // For now, always rewrite the information due to REEF-348
  globalResources.put(this.fileNames.getGlobalFolderPath(), updatedGlobalJarResource);

  return globalResources;
}
 
Example #7
Source File: TestFSDownload.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static LocalResource createJar(FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  LOG.info("Create jar file " + p);
  File jarFile = new File((files.makeQualified(p)).toUri());
  FileOutputStream stream = new FileOutputStream(jarFile);
  LOG.info("Create jar out stream ");
  JarOutputStream out = new JarOutputStream(stream, new Manifest());
  LOG.info("Done writing jar stream ");
  out.close();
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(p));
  FileStatus status = files.getFileStatus(p);
  ret.setSize(status.getLen());
  ret.setTimestamp(status.getModificationTime());
  ret.setType(LocalResourceType.PATTERN);
  ret.setVisibility(vis);
  ret.setPattern("classes/.*");
  return ret;
}
 
Example #8
Source File: LocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
private Map<String, LocalResource> buildResourceMapping() {
  ImmutableMap.Builder<String, LocalResource>  localResourceMapBuilder = ImmutableMap.builder();

  List<String> resourceNames = resourceConfig.getResourceNames();
  for (String resourceName : resourceNames) {
    String resourceLocalName = resourceConfig.getResourceLocalName(resourceName);
    LocalResourceType resourceType = resourceConfig.getResourceLocalType(resourceName);
    LocalResourceVisibility resourceVisibility = resourceConfig.getResourceLocalVisibility(resourceName);
    Path resourcePath = resourceConfig.getResourcePath(resourceName);

    LocalResource localResource = createLocalResource(resourcePath, resourceType, resourceVisibility);

    localResourceMapBuilder.put(resourceLocalName, localResource);
    log.info("preparing local resource: {}", resourceLocalName);
  }

  return localResourceMapBuilder.build();
}
 
Example #9
Source File: TestLocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
@Test
public void testResourceMapWithDefaultValues() {

  Map<String, String> configMap = new HashMap<>();

  configMap.put("yarn.resources.myResource1.path", "http://host1.com/readme");

  Config conf = new MapConfig(configMap);

  YarnConfiguration yarnConfiguration = new YarnConfiguration();
  yarnConfiguration.set("fs.http.impl", HttpFileSystem.class.getName());

  LocalizerResourceMapper mapper = new LocalizerResourceMapper(new LocalizerResourceConfig(conf), yarnConfiguration);
  Map<String, LocalResource> resourceMap = mapper.getResourceMap();

  assertNull("Resource does not exist with a name readme", resourceMap.get("readme"));
  assertNotNull("Resource exists with a name myResource1", resourceMap.get("myResource1"));
  assertEquals("host1.com", resourceMap.get("myResource1").getResource().getHost());
  assertEquals(LocalResourceType.FILE, resourceMap.get("myResource1").getType());
  assertEquals(LocalResourceVisibility.APPLICATION, resourceMap.get("myResource1").getVisibility());
}
 
Example #10
Source File: AbstractYarnClusterDescriptor.java    From Flink-CEPplus with Apache License 2.0 6 votes vote down vote up
/**
 * Uploads and registers a single resource and adds it to <tt>localResources</tt>.
 *
 * @param key
 * 		the key to add the resource under
 * @param fs
 * 		the remote file system to upload to
 * @param appId
 * 		application ID
 * @param localSrcPath
 * 		local path to the file
 * @param localResources
 * 		map of resources
 *
 * @return the remote path to the uploaded resource
 */
private static Path setupSingleLocalResource(
		String key,
		FileSystem fs,
		ApplicationId appId,
		Path localSrcPath,
		Map<String, LocalResource> localResources,
		Path targetHomeDir,
		String relativeTargetPath) throws IOException, URISyntaxException {

	Tuple2<Path, LocalResource> resource = Utils.setupLocalResource(
		fs,
		appId.toString(),
		localSrcPath,
		targetHomeDir,
		relativeTargetPath);

	localResources.put(key, resource.f1);

	return resource.f0;
}
 
Example #11
Source File: StramClient.java    From Bats with Apache License 2.0 6 votes vote down vote up
/**
 * Process SSLConfig object to set up SSL resources
 *
 * @param sslConfig  SSLConfig object derived from SSL_CONFIG attribute
 * @param fs    HDFS file system object
 * @param appPath    application path for the current application
 * @param localResources  Local resources to modify
 * @throws IOException
 */
private void setupSSLResources(SSLConfig sslConfig, FileSystem fs, Path appPath, Map<String, LocalResource> localResources) throws IOException
{
  if (sslConfig != null) {
    String nodeLocalConfig = sslConfig.getConfigPath();

    if (StringUtils.isNotEmpty(nodeLocalConfig)) {
      // all others should be empty
      if (StringUtils.isNotEmpty(sslConfig.getKeyStorePath()) || StringUtils.isNotEmpty(sslConfig.getKeyStorePassword())
          || StringUtils.isNotEmpty(sslConfig.getKeyStoreKeyPassword())) {
        throw new IllegalArgumentException("Cannot specify both nodeLocalConfigPath and other parameters in " + sslConfig);
      }
      // pass thru: Stram will implement reading the node local SSL config file
    } else {
      // need to package and copy the keyStore file
      String keystorePath = sslConfig.getKeyStorePath();
      String[] sslFileArray = {keystorePath};
      String sslFileNames = copyFromLocal(fs, appPath, sslFileArray);
      LaunchContainerRunnable.addFilesToLocalResources(LocalResourceType.FILE, sslFileNames, localResources, fs);
    }
  }
}
 
Example #12
Source File: EvaluatorSetupHelper.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * Sets up the LocalResources for a new Evaluator.
 *
 * @param resourceLaunchEvent
 * @return
 * @throws IOException
 */
Map<String, LocalResource> getResources(
    final ResourceLaunchEvent resourceLaunchEvent)
    throws IOException {

  final Map<String, LocalResource> result = new HashMap<>();
  result.putAll(getGlobalResources());

  final File localStagingFolder = this.tempFileCreator.createTempDirectory(this.fileNames.getEvaluatorFolderPrefix());

  // Write the configuration
  final File configurationFile = new File(localStagingFolder, this.fileNames.getEvaluatorConfigurationName());
  this.configurationSerializer.toFile(makeEvaluatorConfiguration(resourceLaunchEvent), configurationFile);

  // Copy files to the staging folder
  JobJarMaker.copy(resourceLaunchEvent.getFileSet(), localStagingFolder);

  // Make a JAR file out of it
  final File localFile = tempFileCreator.createTempFile(
      this.fileNames.getEvaluatorFolderPrefix(), this.fileNames.getJarFileSuffix());
  new JARFileMaker(localFile).addChildren(localStagingFolder).close();

  // Upload the JAR to the job folder
  final Path pathToEvaluatorJar = this.uploader.uploadToJobFolder(localFile);
  result.put(this.fileNames.getLocalFolderPath(), this.uploader.makeLocalResourceForJarFile(pathToEvaluatorJar));

  if (this.deleteTempFiles) {
    LOG.log(Level.FINE, "Marking [{0}] for deletion at the exit of this JVM and deleting [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
    localFile.deleteOnExit();
    if (!localStagingFolder.delete()) {
      LOG.log(Level.WARNING, "Failed to delete [{0}]", localStagingFolder.getAbsolutePath());
    }
  } else {
    LOG.log(Level.FINE, "The evaluator staging folder will be kept at [{0}], the JAR at [{1}]",
        new Object[]{localFile.getAbsolutePath(), localStagingFolder.getAbsolutePath()});
  }
  return result;
}
 
Example #13
Source File: TaskAppmaster.java    From spring-cloud-deployer-yarn with Apache License 2.0 5 votes vote down vote up
@Override
public ContainerLaunchContext preLaunch(Container container, ContainerLaunchContext context) {
	Map<String, LocalResource> resources = new HashMap<String, LocalResource>();
	resources.putAll(context.getLocalResources());
	resources.putAll(artifactResourceLocalizer.getResources());
	context.setLocalResources(resources);
	return context;
}
 
Example #14
Source File: AMContainerTask.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public AMContainerTask(boolean shouldDie, TaskSpec tezTask,
    Map<String, LocalResource> additionalResources, Credentials credentials, boolean credentialsChanged) {
  this.shouldDie = shouldDie;
  this.tezTask = tezTask;
  this.additionalResources = additionalResources;
  this.credentials = credentials;
  this.credentialsChanged = credentialsChanged;
}
 
Example #15
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 
Example #16
Source File: YarnService.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
private void addContainerLocalResources(Path destDir, Map<String, LocalResource> resourceMap) throws IOException {
  if (!this.fs.exists(destDir)) {
    LOGGER.warn(String.format("Path %s does not exist so no container LocalResource to add", destDir));
    return;
  }

  FileStatus[] statuses = this.fs.listStatus(destDir);
  if (statuses != null) {
    for (FileStatus status : statuses) {
      YarnHelixUtils.addFileAsLocalResource(this.fs, status.getPath(), LocalResourceType.FILE, resourceMap);
    }
  }
}
 
Example #17
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 *
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesMultipleTarballs() throws Exception {
  FileSystem localFs = FileSystem.getLocal(new Configuration());
  StringBuilder tezLibUris = new StringBuilder();

  // Create 2 files
  Path topDir = new Path(TEST_ROOT_DIR, "validatemultipletarballs");
  if (localFs.exists(topDir)) {
    localFs.delete(topDir, true);
  }
  localFs.mkdirs(topDir);

  Path tarFile1 = new Path(topDir, "f1.tar.gz");
  Path tarFile2 = new Path(topDir, "f2.tar.gz");

  Assert.assertTrue(localFs.createNewFile(tarFile1));
  Assert.assertTrue(localFs.createNewFile(tarFile2));
  tezLibUris.append(localFs.makeQualified(tarFile1).toString()).append("#tar1").append(",");
  tezLibUris.append(localFs.makeQualified(tarFile2).toString()).append("#tar2").append(",");

  TezConfiguration conf = new TezConfiguration();
  conf.set(TezConfiguration.TEZ_LIB_URIS, tezLibUris.toString());
  Credentials credentials = new Credentials();
  Map<String, LocalResource> localizedMap = new HashMap<String, LocalResource>();
  TezClientUtils.setupTezJarsLocalResources(conf, credentials, localizedMap);
  Set<String> resourceNames = localizedMap.keySet();
  Assert.assertEquals(2, resourceNames.size());
  Assert.assertTrue(resourceNames.contains("tar1"));
  Assert.assertTrue(resourceNames.contains("tar2"));
  Assert.assertFalse(resourceNames.contains("f1.tar.gz"));
  Assert.assertFalse(resourceNames.contains("f2.tar.gz"));


  Assert.assertTrue(localFs.delete(tarFile1, true));
  Assert.assertTrue(localFs.delete(tarFile2, true));
  Assert.assertTrue(localFs.delete(topDir, true));
}
 
Example #18
Source File: TestContainer.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 
Example #19
Source File: YarnJobDescriptor.java    From sylph with Apache License 2.0 5 votes vote down vote up
private LocalResource registerLocalResource(FileSystem fs, Path remoteRsrcPath)
        throws IOException
{
    LocalResource localResource = Records.newRecord(LocalResource.class);
    FileStatus jarStat = fs.getFileStatus(remoteRsrcPath);
    localResource.setResource(ConverterUtils.getYarnUrlFromURI(remoteRsrcPath.toUri()));
    localResource.setSize(jarStat.getLen());
    localResource.setTimestamp(jarStat.getModificationTime());
    localResource.setType(LocalResourceType.FILE);
    localResource.setVisibility(LocalResourceVisibility.APPLICATION);
    return localResource;
}
 
Example #20
Source File: YARNRunner.java    From tez with Apache License 2.0 5 votes vote down vote up
private LocalResource createApplicationResource(FileContext fs, Path p,
    LocalResourceType type) throws IOException {
  LocalResource rsrc = Records.newRecord(LocalResource.class);
  FileStatus rsrcStat = fs.getFileStatus(p);
  rsrc.setResource(ConverterUtils.getYarnUrlFromPath(fs
      .getDefaultFileSystem().resolvePath(rsrcStat.getPath())));
  rsrc.setSize(rsrcStat.getLen());
  rsrc.setTimestamp(rsrcStat.getModificationTime());
  rsrc.setType(type);
  rsrc.setVisibility(LocalResourceVisibility.APPLICATION);
  return rsrc;
}
 
Example #21
Source File: TestDAGPlan.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testEdgeManagerSerde() {
  DAG dag = DAG.create("testDag");
  ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
  ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
  Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
  Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
  v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());
  v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
      .addTaskLocalFiles(new HashMap<String, LocalResource>());

  InputDescriptor inputDescriptor = InputDescriptor.create("input")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
  OutputDescriptor outputDescriptor = OutputDescriptor.create("output")
      .setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
  Edge edge = Edge.create(v1, v2, EdgeProperty.create(
      EdgeManagerPluginDescriptor.create("emClass").setUserPayload(
          UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))),
      DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));

  dag.addVertex(v1).addVertex(v2).addEdge(edge);

  DAGPlan dagProto = dag.createDag(new TezConfiguration(), null, null, null, true);

  EdgeProperty edgeProperty = DagTypeConverters.createEdgePropertyMapFromDAGPlan(dagProto
      .getEdgeList().get(0));

  EdgeManagerPluginDescriptor emDesc = edgeProperty.getEdgeManagerDescriptor();
  Assert.assertNotNull(emDesc);
  Assert.assertEquals("emClass", emDesc.getClassName());
  Assert.assertTrue(
      Arrays.equals("emPayload".getBytes(), emDesc.getUserPayload().deepCopyAsArray()));
}
 
Example #22
Source File: TezResourceManager.java    From spork with Apache License 2.0 5 votes vote down vote up
public Map<String, LocalResource> addTezResources(Set<URI> resources) throws Exception {
    Set<String> resourceNames = new HashSet<String>();
    for (URI uri : resources) {
        addTezResource(uri);
        resourceNames.add(new Path(uri.getPath()).getName());
    }
    return getTezResources(resourceNames);
}
 
Example #23
Source File: TestSharedCacheUploader.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private SharedCacheUploader createSpiedUploader(LocalResource resource, Path localPath,
    String user, Configuration conf, SCMUploaderProtocol scmClient,
    FileSystem fs, FileSystem localFs)
        throws IOException {
  SharedCacheUploader uploader = new SharedCacheUploader(resource, localPath, user, conf, scmClient,
      fs, localFs);
  return spy(uploader);
}
 
Example #24
Source File: TestAMContainer.java    From tez with Apache License 2.0 5 votes vote down vote up
public void assignTaskAttempt(TezTaskAttemptID taID,
    Map<String, LocalResource> additionalResources, Credentials credentials) {
  reset(eventHandler);
  doReturn(taID).when(taskSpec).getTaskAttemptID();
  amContainer.handle(new AMContainerEventAssignTA(containerID, taID, taskSpec,
      additionalResources, credentials, taskPriority));
}
 
Example #25
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up
public static Map<String, LocalResource> convertFromPlanLocalResources(
  PlanLocalResourcesProto proto) {
  Map<String, LocalResource> localResources =
    new HashMap<String, LocalResource>(proto.getLocalResourcesCount());
  for (PlanLocalResource plr : proto.getLocalResourcesList()) {
    String name = plr.getName();
    LocalResource lr = convertPlanLocalResourceToLocalResource(plr);
    localResources.put(name, lr);
  }
  return localResources;
}
 
Example #26
Source File: TestTezClientUtils.java    From tez with Apache License 2.0 5 votes vote down vote up
/**
 * 
 */
@Test (timeout=5000)
public void validateSetTezJarLocalResourcesNotDefined() throws Exception {

  TezConfiguration conf = new TezConfiguration(false);
  Credentials credentials = new Credentials();
  try {
    Map<String,LocalResource> resources = new HashMap<String, LocalResource>();
    TezClientUtils.setupTezJarsLocalResources(conf, credentials, resources);
    Assert.fail("Expected TezUncheckedException");
  } catch (TezUncheckedException e) {
    Assert.assertTrue(e.getMessage().contains("Invalid configuration of tez jars"));
  }
}
 
Example #27
Source File: AMConfiguration.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
AMConfiguration(TezConfiguration tezConf, Map<String, LocalResource> localResources,
    Credentials credentials) {
  this.localResources = Maps.newHashMap();
  this.tezConf = tezConf;
  if (localResources != null) {
    addLocalResources(localResources);
  }
  if (credentials != null) {
    setCredentials(credentials);
  }

}
 
Example #28
Source File: TestPBRecordImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
static LocalResource createResource() {
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  assertTrue(ret instanceof LocalResourcePBImpl);
  ret.setResource(ConverterUtils.getYarnUrlFromPath(new Path(
    "hdfs://y.ak:8020/foo/bar")));
  ret.setSize(4344L);
  ret.setTimestamp(3141592653589793L);
  ret.setVisibility(LocalResourceVisibility.PUBLIC);
  return ret;
}
 
Example #29
Source File: ContainerContext.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts) {
  Preconditions.checkNotNull(localResources,
      "localResources should not be null");
  Preconditions.checkNotNull(credentials, "credentials should not be null");
  Preconditions.checkNotNull(environment, "environment should not be null");
  Preconditions.checkNotNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = null;
}
 
Example #30
Source File: ContainerContext.java    From tez with Apache License 2.0 5 votes vote down vote up
public ContainerContext(Map<String, LocalResource> localResources,
    Credentials credentials, Map<String, String> environment, String javaOpts) {
  Objects.requireNonNull(localResources,
      "localResources should not be null");
  Objects.requireNonNull(credentials, "credentials should not be null");
  Objects.requireNonNull(environment, "environment should not be null");
  Objects.requireNonNull(javaOpts, "javaOpts should not be null");
  this.localResources = localResources;
  this.credentials = credentials;
  this.environment = environment;
  this.javaOpts = javaOpts;
  this.vertex = null;
}