Java Code Examples for org.apache.hadoop.yarn.api.records.URL

The following examples show how to use org.apache.hadoop.yarn.api.records.URL. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hadoop   Source File: ConverterUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 
Example 2
Source Project: tez   Source File: TestTezClient.java    License: Apache License 2.0 6 votes vote down vote up
@Test(timeout = 5000)
public void testClientResubmit() throws Exception {
  TezClientForTest client = configureAndCreateTezClient(null, true, null);
  client.start();
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex vertex1 = Vertex.create("Vertex1", ProcessorDescriptor.create("P1"), 1,
      Resource.newInstance(1, 1));
  vertex1.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  Vertex vertex2 = Vertex.create("Vertex2", ProcessorDescriptor.create("P2"), 1,
      Resource.newInstance(1, 1));
  vertex2.setTaskLaunchCmdOpts("-XX:+UseParallelGC -XX:+UseG1GC");
  DAG dag = DAG.create("DAG").addVertex(vertex1).addVertex(vertex2).addTaskLocalFiles(lrDAG);
  for (int i = 0; i < 3; ++i) {
    try {
      client.submitDAG(dag);
      Assert.fail("Expected TezUncheckedException here.");
    } catch(TezUncheckedException ex) {
      Assert.assertTrue(ex.getMessage().contains("Invalid/conflicting GC options found"));
    }
  }
  client.stop();
}
 
Example 3
Source Project: tez   Source File: TezConverterUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * return a {@link URI} from a given url
 *
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 
Example 4
Source Project: hadoop   Source File: TestContainerLocalizer.java    License: Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example 5
Source Project: big-c   Source File: ConverterUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * return a hadoop path from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
public static Path getPathFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();
  
  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }
  
  return new Path(
      (new URI(scheme, authority, url.getFile(), null, null)).normalize());
}
 
Example 6
Source Project: big-c   Source File: TestContainerLocalizer.java    License: Apache License 2.0 6 votes vote down vote up
static ResourceLocalizationSpec getMockRsrc(Random r,
    LocalResourceVisibility vis, Path p) {
  ResourceLocalizationSpec resourceLocalizationSpec =
    mock(ResourceLocalizationSpec.class);

  LocalResource rsrc = mock(LocalResource.class);
  String name = Long.toHexString(r.nextLong());
  URL uri = mock(org.apache.hadoop.yarn.api.records.URL.class);
  when(uri.getScheme()).thenReturn("file");
  when(uri.getHost()).thenReturn(null);
  when(uri.getFile()).thenReturn("/local/" + vis + "/" + name);

  when(rsrc.getResource()).thenReturn(uri);
  when(rsrc.getSize()).thenReturn(r.nextInt(1024) + 1024L);
  when(rsrc.getTimestamp()).thenReturn(r.nextInt(1024) + 2048L);
  when(rsrc.getType()).thenReturn(LocalResourceType.FILE);
  when(rsrc.getVisibility()).thenReturn(vis);

  when(resourceLocalizationSpec.getResource()).thenReturn(rsrc);
  when(resourceLocalizationSpec.getDestinationDirectory()).
    thenReturn(ConverterUtils.getYarnUrlFromPath(p));
  return resourceLocalizationSpec;
}
 
Example 7
Source Project: tez   Source File: TestTezLocalCacheManager.java    License: Apache License 2.0 6 votes vote down vote up
private static LocalResource createFile(String content) throws IOException {
  FileContext fs = FileContext.getLocalFSFileContext();

  java.nio.file.Path tempFile = Files.createTempFile("test-cache-manager", ".txt");
  File temp = tempFile.toFile();
  temp.deleteOnExit();
  Path p = new Path("file:///" + tempFile.toAbsolutePath().toString());

  Files.write(tempFile, content.getBytes());

  RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
  LocalResource ret = recordFactory.newRecordInstance(LocalResource.class);
  URL yarnUrlFromPath = ConverterUtils.getYarnUrlFromPath(p);
  ret.setResource(yarnUrlFromPath);
  ret.setSize(content.getBytes().length);
  ret.setType(LocalResourceType.FILE);
  ret.setVisibility(LocalResourceVisibility.PRIVATE);
  ret.setTimestamp(fs.getFileStatus(p).getModificationTime());
  return ret;
}
 
Example 8
Source Project: tez   Source File: TestDAG.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testRecreateDAG() {
  Map<String, LocalResource> lrDAG = Collections.singletonMap("LR1",
      LocalResource.newInstance(
          URL.newInstance("file", "localhost", 0, "/test1"),
          LocalResourceType.FILE,
          LocalResourceVisibility.PUBLIC, 1, 1));
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create("dummyProcessor1"), 1,
      Resource.newInstance(1, 1));
  Vertex v2 = Vertex.create("v2", ProcessorDescriptor.create("dummyProcessor2"), 1,
      Resource.newInstance(1, 1));
  DAG dag = DAG.create("dag1").addVertex(v1).addVertex(v2).addTaskLocalFiles(lrDAG);

  TezConfiguration tezConf = new TezConfiguration();
  DAGPlan firstPlan = dag.createDag(tezConf, null, null, null, false);
  for (int i = 0; i < 3; ++i) {
      DAGPlan dagPlan = dag.createDag(tezConf, null, null, null, false);
      Assert.assertEquals(dagPlan, firstPlan);
  }
}
 
Example 9
Source Project: samza   Source File: LocalizerResourceMapper.java    License: Apache License 2.0 6 votes vote down vote up
private LocalResource createLocalResource(Path resourcePath, LocalResourceType resourceType, LocalResourceVisibility resourceVisibility) {
  LocalResource localResource = Records.newRecord(LocalResource.class);
  URL resourceUrl = ConverterUtils.getYarnUrlFromPath(resourcePath);
  try {
    FileStatus resourceFileStatus = resourcePath.getFileSystem(yarnConfiguration).getFileStatus(resourcePath);

    if (null == resourceFileStatus) {
      throw new LocalizerResourceException("Check getFileStatus implementation. getFileStatus gets unexpected null for resourcePath " + resourcePath);
    }

    localResource.setResource(resourceUrl);
    log.info("setLocalizerResource for {}", resourceUrl);
    localResource.setSize(resourceFileStatus.getLen());
    localResource.setTimestamp(resourceFileStatus.getModificationTime());
    localResource.setType(resourceType);
    localResource.setVisibility(resourceVisibility);
    return localResource;
  } catch (IOException ioe) {
    log.error("IO Exception when accessing the resource file status from the filesystem: " + resourcePath, ioe);
    throw new LocalizerResourceException("IO Exception when accessing the resource file status from the filesystem: " + resourcePath);
  }

}
 
Example 10
Source Project: incubator-tez   Source File: TezConverterUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * return a {@link URI} from a given url
 * 
 * @param url
 *          url to convert
 * @return path from {@link URL}
 * @throws URISyntaxException
 */
@Private
public static URI getURIFromYarnURL(URL url) throws URISyntaxException {
  String scheme = url.getScheme() == null ? "" : url.getScheme();

  String authority = "";
  if (url.getHost() != null) {
    authority = url.getHost();
    if (url.getUserInfo() != null) {
      authority = url.getUserInfo() + "@" + authority;
    }
    if (url.getPort() > 0) {
      authority += ":" + url.getPort();
    }
  }

  return new URI(scheme, authority, url.getFile(), null, null).normalize();
}
 
Example 11
Source Project: TensorFlowOnYARN   Source File: Utils.java    License: Apache License 2.0 5 votes vote down vote up
private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}
 
Example 12
Source Project: hadoop   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 
Example 13
Source Project: hadoop   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setResource(URL resource) {
  maybeInitBuilder();
  if (resource == null) 
    builder.clearResource();
  this.url = resource;
}
 
Example 14
Source Project: hadoop   Source File: TestConverterUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertUrlWithUserinfo() throws URISyntaxException {
  Path expectedPath = new Path("foo://username:[email protected]:8042");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}
 
Example 15
Source Project: tez   Source File: TestDAGVerify.java    License: Apache License 2.0 5 votes vote down vote up
@Test(timeout = 5000)
public void testDAGCreateDataInference() {
  Vertex v1 = Vertex.create("v1", ProcessorDescriptor.create(dummyProcessorClassName));
  Map<String, LocalResource> lrs1 = Maps.newHashMap();
  String lrName1 = "LR1";
  lrs1.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrs2 = Maps.newHashMap();
  String lrName2 = "LR2";
  lrs2.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  Set<String> hosts = Sets.newHashSet();
  hosts.add("h1");
  hosts.add("h2");
  List<TaskLocationHint> taskLocationHints = Lists.newLinkedList();
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  taskLocationHints.add(TaskLocationHint.createTaskLocationHint(hosts, null));
  VertexLocationHint vLoc = VertexLocationHint.create(taskLocationHints);
  DataSourceDescriptor ds = DataSourceDescriptor.create(InputDescriptor.create("I.class"),
      InputInitializerDescriptor.create(dummyInputInitClassName), dummyTaskCount, null, vLoc, lrs2);
  v1.addDataSource("i1", ds);

  DAG dag = DAG.create("testDag");
  dag.addVertex(v1);
  dag.addTaskLocalFiles(lrs1);
  DAGPlan dagPlan = dag.createDag(new TezConfiguration(), null, null, null, true);
  Assert.assertEquals(lrName1, dagPlan.getLocalResource(0).getName());
  VertexPlan vPlan = dagPlan.getVertex(0);
  PlanTaskConfiguration taskPlan = vPlan.getTaskConfig();
  Assert.assertEquals(dummyTaskCount, taskPlan.getNumTasks());
  Assert.assertEquals(TezConfiguration.TEZ_TASK_RESOURCE_MEMORY_MB_DEFAULT, taskPlan.getMemoryMb());
  Assert.assertEquals(lrName2, taskPlan.getLocalResource(0).getName());
  Assert.assertEquals(dummyTaskCount, vPlan.getTaskLocationHintCount());
}
 
Example 16
Source Project: hadoop   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static URL newURL(String scheme, String host, int port, String file) {
  URL url = recordFactory.newRecordInstance(URL.class);
  url.setScheme(scheme);
  url.setHost(host);
  url.setPort(port);
  url.setFile(file);
  return url;
}
 
Example 17
Source Project: hadoop   Source File: LocalResourceStatusPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public URL getLocalPath() {
  LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
  if (this.localPath != null) {
    return this.localPath;
  }
  if (!p.hasLocalPath()) {
    return null;
  }
  this.localPath = convertFromProtoFormat(p.getLocalPath());
  return this.localPath;
}
 
Example 18
Source Project: hadoop   Source File: LocalResourceStatusPBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setLocalPath(URL localPath) {
  maybeInitBuilder();
  if (localPath == null)
    builder.clearLocalPath();
  this.localPath = localPath;
}
 
Example 19
Source Project: hadoop   Source File: NodeManagerBuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static ResourceLocalizationSpec newResourceLocalizationSpec(
    LocalResource rsrc, Path path) {
  URL local = ConverterUtils.getYarnUrlFromPath(path);
  ResourceLocalizationSpec resourceLocalizationSpec =
      Records.newRecord(ResourceLocalizationSpec.class);
  resourceLocalizationSpec.setDestinationDirectory(local);
  resourceLocalizationSpec.setResource(rsrc);
  return resourceLocalizationSpec;
}
 
Example 20
Source Project: hadoop   Source File: MockLocalResourceStatus.java    License: Apache License 2.0 5 votes vote down vote up
MockLocalResourceStatus(LocalResource rsrc, ResourceStatusType tag,
    URL localPath, SerializedException ex) {
  this.rsrc = rsrc;
  this.tag = tag;
  this.localPath = localPath;
  this.ex = ex;
}
 
Example 21
Source Project: hadoop   Source File: TestResourceLocalizationService.java    License: Apache License 2.0 5 votes vote down vote up
private static LocalResource getMockedResource(Random r, 
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = getPath("/local/PRIVATE/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return rsrc;
}
 
Example 22
Source Project: hadoop   Source File: TestContainer.java    License: Apache License 2.0 5 votes vote down vote up
private static Entry<String, LocalResource> getMockRsrc(Random r,
    LocalResourceVisibility vis) {
  String name = Long.toHexString(r.nextLong());
  URL url = BuilderUtils.newURL("file", null, 0, "/local" + vis + "/" + name);
  LocalResource rsrc =
      BuilderUtils.newLocalResource(url, LocalResourceType.FILE, vis,
          r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
  return new SimpleEntry<String, LocalResource>(name, rsrc);
}
 
Example 23
Source Project: hadoop   Source File: TaskAttemptImpl.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}
 
Example 24
Source Project: tez   Source File: TestMockDAGAppMaster.java    License: Apache License 2.0 5 votes vote down vote up
@Test (timeout = 10000)
public void testMultipleSubmissions() throws Exception {
  Map<String, LocalResource> lrDAG = Maps.newHashMap();
  String lrName1 = "LR1";
  lrDAG.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
  Map<String, LocalResource> lrVertex = Maps.newHashMap();
  String lrName2 = "LR2";
  lrVertex.put(lrName2, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test1"),
      LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));

  DAG dag = DAG.create("test").addTaskLocalFiles(lrDAG);
  Vertex vA = Vertex.create("A", ProcessorDescriptor.create("Proc.class"), 5).addTaskLocalFiles(lrVertex);
  dag.addVertex(vA);

  TezConfiguration tezconf = new TezConfiguration(defaultConf);
  
  MockTezClient tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  DAGClient dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
  
  // submit the same DAG again to verify it can be done.
  tezClient = new MockTezClient("testMockAM", tezconf, true, null, null, null, null);
  tezClient.start();
  dagClient = tezClient.submitDAG(dag);
  dagClient.waitForCompletion();
  Assert.assertEquals(DAGStatus.State.SUCCEEDED, dagClient.getDAGStatus(null).getState());
  tezClient.stop();
}
 
Example 25
Source Project: big-c   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized URL getResource() {
  LocalResourceProtoOrBuilder p = viaProto ? proto : builder;
  if (this.url != null) {
    return this.url;
  }
  if (!p.hasResource()) {
    return null;
  }
  this.url = convertFromProtoFormat(p.getResource());
  return this.url;
}
 
Example 26
Source Project: big-c   Source File: LocalResourcePBImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setResource(URL resource) {
  maybeInitBuilder();
  if (resource == null) 
    builder.clearResource();
  this.url = resource;
}
 
Example 27
Source Project: big-c   Source File: ConverterUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static URL getYarnUrlFromURI(URI uri) {
  URL url = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(URL.class);
  if (uri.getHost() != null) {
    url.setHost(uri.getHost());
  }
  if (uri.getUserInfo() != null) {
    url.setUserInfo(uri.getUserInfo());
  }
  url.setPort(uri.getPort());
  url.setScheme(uri.getScheme());
  url.setFile(uri.getPath());
  return url;
}
 
Example 28
Source Project: big-c   Source File: TestConverterUtils.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testConvertUrlWithNoPort() throws URISyntaxException {
  Path expectedPath = new Path("hdfs://foo.com");
  URL url = ConverterUtils.getYarnUrlFromPath(expectedPath);
  Path actualPath = ConverterUtils.getPathFromYarnURL(url);
  assertEquals(expectedPath, actualPath);
}
 
Example 29
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static LocalResource newLocalResource(URL url, LocalResourceType type,
    LocalResourceVisibility visibility, long size, long timestamp,
    boolean shouldBeUploadedToSharedCache) {
  LocalResource resource =
    recordFactory.newRecordInstance(LocalResource.class);
  resource.setResource(url);
  resource.setType(type);
  resource.setVisibility(visibility);
  resource.setSize(size);
  resource.setTimestamp(timestamp);
  resource.setShouldBeUploadedToSharedCache(shouldBeUploadedToSharedCache);
  return resource;
}
 
Example 30
Source Project: big-c   Source File: BuilderUtils.java    License: Apache License 2.0 5 votes vote down vote up
public static URL newURL(String scheme, String host, int port, String file) {
  URL url = recordFactory.newRecordInstance(URL.class);
  url.setScheme(scheme);
  url.setHost(host);
  url.setPort(port);
  url.setFile(file);
  return url;
}