org.apache.hadoop.yarn.api.records.URL Java Examples

The following examples show how to use org.apache.hadoop.yarn.api.records.URL. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TezConverterUtils.java    From incubator-tez with Apache License 2.0 6 votes vote down vote up
/**
 * return a {@link URI} from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 
Example #2
Source File: TestTezClient.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example #3
Source File: ConverterUtils.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 
Example #4
Source File: TezConverterUtils.java    From tez with Apache License 2.0 6 votes vote down vote up
/**
 * return a {@link URI} from a given url
 *
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 
Example #5
Source File: LocalizerResourceMapper.java    From samza with Apache License 2.0 6 votes vote down vote up
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 
Example #6
Source File: ConverterUtils.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 
Example #7
Source File: TestContainerLocalizer.java    From hadoop with Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example #8
Source File: TestContainerLocalizer.java    From big-c with Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example #9
Source File: TestDAG.java    From tez with Apache License 2.0 6 votes vote down vote up
@Test
public void testRecreateDAG() {
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor1"), 1,
      Resource.newInstance(1, 1));
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("dummyProcessor2"), 1,
      Resource.newInstance(1, 1));
  DAG dag = DAG.create("dag1").addVertex(v1).addVertex(v2).addTaskLocalFiles(lrDAG);

  TezConfiguration tezConf = new TezConfiguration();
  DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false);
  for (int i = 0; i < 3; ++i) {
      DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
      Assert.assertEquals(dagPlan, firstPlan);
  }
}
 
Example #10
Source File: TestTezLocalCacheManager.java    From tez with Apache License 2.0 6 votes vote down vote up
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example #11
Source File: TestDagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testYarnPathTranslation() {
  // Without port
  String p1String = "hdfs://mycluster/file";
  Path p1Path = new Path(p1String);
  // Users would translate this via this mechanic.
  URL lr1Url = ConverterUtils.getYarnUrlFromPath(p1Path);
  // Serialize to dag plan.
  String p1StringSerialized = DagTypeConverters.convertToDAGPlan(lr1Url);
  // Deserialize
  URL lr1UrlDeserialized = DagTypeConverters.convertToYarnURL(p1StringSerialized);
  Assert.assertEquals("mycluster", lr1UrlDeserialized.getHost());
  Assert.assertEquals("/file", lr1UrlDeserialized.getFile());
  Assert.assertEquals("hdfs", lr1UrlDeserialized.getScheme());


  // With port
  String p2String = "hdfs://mycluster:2311/file";
  Path p2Path = new Path(p2String);
  // Users would translate this via this mechanic.
  URL lr2Url = ConverterUtils.getYarnUrlFromPath(p2Path);
  // Serialize to dag plan.
  String p2StringSerialized = DagTypeConverters.convertToDAGPlan(lr2Url);
  // Deserialize
  URL lr2UrlDeserialized = DagTypeConverters.convertToYarnURL(p2StringSerialized);
  Assert.assertEquals("mycluster", lr2UrlDeserialized.getHost());
  Assert.assertEquals("/file", lr2UrlDeserialized.getFile());
  Assert.assertEquals("hdfs", lr2UrlDeserialized.getScheme());
  Assert.assertEquals(2311, lr2UrlDeserialized.getPort());
}
 
Example #12
Source File: AMContainerHelpers.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
public static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example #13
Source File: TestMRRJobsDAGApi.java    From incubator-tez with Apache License 2.0 5 votes vote down vote up
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example #14
Source File: JobResourceUploader.java    From reef with Apache License 2.0 5 votes vote down vote up
/**
 * This class is invoked from Org.Apache.REEF.Client.Yarn.LegacyJobResourceUploader in .NET code.
 * Arguments:
 * [0] : Local path for file.
 * [1] : Type for file.
 * [2] : Path of job submission directory
 * [3] : File path for output with details of uploaded resource
 */
public static void main(final String[] args) throws InjectionException, IOException {
  Validate.isTrue(args.length == 4, "Job resource uploader requires 4 args");
  final File localFile = new File(args[0]);
  Validate.isTrue(localFile.exists(), "Local file does not exist " + localFile.getAbsolutePath());
  final String fileType = args[1];
  final String jobSubmissionDirectory = args[2];
  final String localOutputPath = args[3];

  LOG.log(Level.INFO, "Received args: LocalPath " + localFile.getAbsolutePath() + " Submission directory " +
      jobSubmissionDirectory + " LocalOutputPath " + localOutputPath);
  final Configuration configuration = Tang.Factory.getTang().newConfigurationBuilder()
      .bindImplementation(RuntimeClasspathProvider.class, YarnClasspathProvider.class)
      .bindConstructor(org.apache.hadoop.yarn.conf.YarnConfiguration.class, YarnConfigurationConstructor.class)
      .build();

  final JobUploader jobUploader = Tang.Factory.getTang()
      .newInjector(configuration)
      .getInstance(JobUploader.class);
  final LocalResource localResource = jobUploader.createJobFolder(jobSubmissionDirectory)
      .uploadAsLocalResource(localFile, LocalResourceType.valueOf(fileType));

  // Output: <UploadedPath>;<LastModificationUnixTimestamp>;<ResourceSize>
  final URL resource = localResource.getResource();
  final String outputString = String.format("%s://%s:%d%s;%d;%d", resource.getScheme(), resource.getHost(),
      resource.getPort(), resource.getFile(), localResource.getTimestamp(), localResource.getSize());
  LOG.log(Level.INFO, "Writing output: " + outputString);
  try (Writer writer = new BufferedWriter(new OutputStreamWriter(
      new FileOutputStream(localOutputPath), "utf-8"))) {
    writer.write(outputString);
  }

  LOG.log(Level.FINER, "Done writing output file");
}
 
Example #15
Source File: LocalResourceStatusPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public URL getLocalPath() {
  LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
  if (this.localPath != null) {
    return this.localPath;
  }
  if (!p.hasLocalPath()) {
    return null;
  }
  this.localPath = convertFromProtoFormat(p.getLocalPath());
  return this.localPath;
}
 
Example #16
Source File: TestContainer.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 
Example #17
Source File: LocalResourcePBImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 
Example #18
Source File: LocalResourceStatusPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public void setLocalPath(URL localPath) {
  maybeInitBuilder();
  if (localPath == null)
    builder.clearLocalPath();
  this.localPath = localPath;
}
 
Example #19
Source File: DagTypeConverters.java    From tez with Apache License 2.0 5 votes vote down vote up
public static String convertToDAGPlan(URL resource) {
  Path p;
  try {
    p = ConverterUtils.getPathFromYarnURL(resource);
  } catch (URISyntaxException e) {
    throw new TezUncheckedException("Unable to translate resource: " + resource + " to Path");
  }
  String urlString = p.toString();
  return urlString;
}
 
Example #20
Source File: ResourceLocalizationSpecPBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public URL getDestinationDirectory() {
  ResourceLocalizationSpecProtoOrBuilder p = viaProto ? proto : builder;
  if (destinationDirectory != null) {
    return destinationDirectory;
  }
  if (!p.hasDestinationDirectory()) {
    return null;
  }
  destinationDirectory = new URLPBImpl(p.getDestinationDirectory());
  return destinationDirectory;
}
 
Example #21
Source File: TestMRRJobsDAGApi.java    From tez with Apache License 2.0 5 votes vote down vote up
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}
 
Example #22
Source File: TestResourceLocalizationService.java    From big-c with Apache License 2.0 5 votes vote down vote up
private static LocalResource getMockedResource(Random r, 
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = getPath("/local/PRIVATE/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return rsrc;
}
 
Example #23
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static URL newURL(String scheme, String host, int port, String file) {
  URL url = recordFactory.newRecordInstance(URL.class);
  url.setScheme(scheme);
  url.setHost(host);
  url.setPort(port);
  url.setFile(file);
  return url;
}
 
Example #24
Source File: BuilderUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 
Example #25
Source File: TestConverterUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
  Path expectedPath = new Path("hdfs://foo.com");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}
 
Example #26
Source File: ConverterUtils.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
 
Example #27
Source File: LocalResourcePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setResource(URL resource) {
  maybeInitBuilder();
  if (resource == null) 
    builder.clearResource();
  this.url = resource;
}
 
Example #28
Source File: LocalResourcePBImpl.java    From big-c with Apache License 2.0 5 votes vote down vote up
@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 
Example #29
Source File: TestMockDAGAppMaster.java    From tez with Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
  
  // submit the same DAG again to verify it can be done.
  tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 
Example #30
Source File: TaskAttemptImpl.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}