Java Code Examples for org.apache.hadoop.yarn.api.records.LocalResource#newInstance()

The following examples show how to use org.apache.hadoop.yarn.api.records.LocalResource#newInstance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TezResourceManager.java From spork with Apache License 2.0

6 votes

public Map<String, LocalResource> getTezResources(Set<String> resourceNames) throws Exception {
    Map<String, LocalResource> tezResources = new HashMap<String, LocalResource>();
    for (String resourceName : resourceNames) {
        // The resource name will be symlinked to the resource path in the
        // container's working directory.
        Path resourcePath = resources.get(resourceName);
        FileStatus fstat = remoteFs.getFileStatus(resourcePath);

        LocalResource tezResource = LocalResource.newInstance(
                ConverterUtils.getYarnUrlFromPath(fstat.getPath()),
                LocalResourceType.FILE,
                LocalResourceVisibility.APPLICATION,
                fstat.getLen(),
                fstat.getModificationTime());

        tezResources.put(resourceName, tezResource);
    }
    return tezResources;
}

Example 2

Source File: JstormOnYarn.java From jstorm with Apache License 2.0

6 votes

private void addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
    String suffix = jstormClientContext.appName + JOYConstants.BACKLASH + appId + JOYConstants.BACKLASH + fileDstPath;
    Path dst =
            new Path(fs.getHomeDirectory(), suffix);
    if (fileSrcPath == null) {
        FSDataOutputStream ostream = null;
        try {
            ostream = FileSystem
                    .create(fs, dst, new FsPermission(JOYConstants.FS_PERMISSION));
            ostream.writeUTF(resources);
        } finally {
            IOUtils.closeQuietly(ostream);
        }
    } else {
        fs.copyFromLocalFile(new Path(fileSrcPath), dst);
    }
    FileStatus scFileStatus = fs.getFileStatus(dst);
    LocalResource scRsrc =
            LocalResource.newInstance(
                    ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                    LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                    scFileStatus.getLen(), scFileStatus.getModificationTime());
    localResources.put(fileDstPath, scRsrc);
}

Example 3

Source File: AthenaXYarnClusterDescriptor.java From AthenaX with Apache License 2.0

5 votes

private LocalResource toLocalResource(Path path, LocalResourceVisibility visibility) throws IOException {
  FileSystem fs = path.getFileSystem(clusterConf.conf());
  FileStatus stat = fs.getFileStatus(path);
  return LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromPath(path),
          LocalResourceType.FILE,
          visibility,
          stat.getLen(), stat.getModificationTime()
  );
}

Example 4

Source File: Utils.java From TensorFlowOnYARN with Apache License 2.0

5 votes

private static void addToLocalResources(FileSystem fs, String key, Path dst,
    Map<String, LocalResource> localResources) throws IOException {
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource resource =
      LocalResource.newInstance(
          URL.fromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(key, resource);
}

Example 5

Source File: DagTypeConverters.java From tez with Apache License 2.0

5 votes

public static LocalResource convertPlanLocalResourceToLocalResource(
    PlanLocalResource plr) {
  return LocalResource.newInstance(
      ConverterUtils.getYarnUrlFromPath(new Path(plr.getUri())),
      DagTypeConverters.convertFromDAGPlan(plr.getType()),
      DagTypeConverters.convertFromDAGPlan(plr.getVisibility()),
      plr.getSize(), plr.getTimeStamp(),
      plr.hasPattern() ? plr.getPattern() : null);
}

Example 6

Source File: TaskAttemptImpl.java From hadoop with Apache License 2.0

5 votes

/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}

Example 7

Source File: Client.java From big-c with Apache License 2.0

5 votes

private void addToLocalResources(FileSystem fs, String fileSrcPath,
    String fileDstPath, String appId, Map<String, LocalResource> localResources,
    String resources) throws IOException {
  String suffix =
      appName + "/" + appId + "/" + fileDstPath;
  Path dst =
      new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
          .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
      LocalResource.newInstance(
          ConverterUtils.getYarnUrlFromURI(dst.toUri()),
          LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
          scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
}

Example 8

Source File: TaskAttemptImpl.java From big-c with Apache License 2.0

5 votes

/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
    resourceSize, resourceModificationTime);
}

Example 9

Source File: Client.java From metron with Apache License 2.0

5 votes

private Path addToLocalResources(FileSystem fs, String fileSrcPath,
                                 String fileDstPath, String appId, Map<String, LocalResource> localResources,
                                 String resources) throws IOException {
  String suffix =
          appName + "/" + appId + "/" + fileDstPath;
  Path dst =
          new Path(fs.getHomeDirectory(), suffix);
  if (fileSrcPath == null) {
    FSDataOutputStream ostream = null;
    try {
      ostream = FileSystem
              .create(fs, dst, new FsPermission((short) 0710));
      ostream.writeUTF(resources);
    } finally {
      IOUtils.closeQuietly(ostream);
    }
  } else {
    fs.copyFromLocalFile(new Path(fileSrcPath), dst);
  }
  fs.setPermission(dst, new FsPermission((short)0755));
  FileStatus scFileStatus = fs.getFileStatus(dst);
  LocalResource scRsrc =
          LocalResource.newInstance(
                  ConverterUtils.getYarnUrlFromURI(dst.toUri()),
                  LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
                  scFileStatus.getLen(), scFileStatus.getModificationTime());
  localResources.put(fileDstPath, scRsrc);
  return dst;
}

Example 10

Source File: TestMRRJobsDAGApi.java From incubator-tez with Apache License 2.0

5 votes

private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}

Example 11

Source File: AMContainerHelpers.java From tez with Apache License 2.0

5 votes

/**
 * Create a {@link LocalResource} record with all the given parameters.
 */
public static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}

Example 12

Source File: DagTypeConverters.java From incubator-tez with Apache License 2.0

5 votes

public static LocalResource convertPlanLocalResourceToLocalResource(
    PlanLocalResource plr) {
  return LocalResource.newInstance(
    ConverterUtils.getYarnUrlFromPath(new Path(plr.getUri())),
    DagTypeConverters.convertFromDAGPlan(plr.getType()),
    DagTypeConverters.convertFromDAGPlan(plr.getVisibility()),
    plr.getSize(), plr.getTimeStamp(),
    plr.hasPattern() ? plr.getPattern() : null);
}

Example 13

Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0

5 votes

private static LocalResource createLocalResource(FileSystem fc, Path file,
    LocalResourceType type, LocalResourceVisibility visibility)
    throws IOException {
  FileStatus fstat = fc.getFileStatus(file);
  URL resourceURL = ConverterUtils.getYarnUrlFromPath(fc.resolvePath(fstat
      .getPath()));
  long resourceSize = fstat.getLen();
  long resourceModificationTime = fstat.getModificationTime();

  return LocalResource.newInstance(resourceURL, type, visibility,
      resourceSize, resourceModificationTime);
}

Example 14

Source File: TestAMContainer.java From tez with Apache License 2.0

4 votes

private LocalResource createLocalResource(String name) {
  LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name),
      LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000);
  return lr;
}

Example 15

Source File: LaunchContainerRunnable.java From Bats with Apache License 2.0

4 votes

public static void addFileToLocalResources(final String name, final FileStatus fileStatus, final LocalResourceType type, final Map<String, LocalResource> localResources)
{
  final LocalResource localResource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fileStatus.getPath()),
      type, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime());
  localResources.put(name, localResource);
}

Example 16

Source File: TestDAGClientAMProtocolBlockingPBServerImpl.java From tez with Apache License 2.0

4 votes

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testSubmitDagInSessionWithLargeDagPlan() throws Exception {
  int maxIPCMsgSize = 1024;
  String dagPlanName = "dagplan-name";
  File requestFile = tmpFolder.newFile("request-file");
  TezConfiguration conf = new TezConfiguration();
  conf.setInt(CommonConfigurationKeys.IPC_MAXIMUM_DATA_LENGTH, maxIPCMsgSize);

  // Check with 70 MB (64 MB is CodedInputStream's default limit in earlier versions of protobuf)
  byte[] randomBytes = new byte[70 << 20];
  (new Random()).nextBytes(randomBytes);
  UserPayload payload = UserPayload.create(ByteBuffer.wrap(randomBytes));
  Vertex vertex = Vertex.create("V", ProcessorDescriptor.create("P").setUserPayload(payload), 1);
  DAGPlan dagPlan = DAG.create(dagPlanName).addVertex(vertex).createDag(conf, null, null, null, false);

  String lrName = "localResource";
  String scheme = "file";
  String host = "localhost";
  int port = 80;
  String path = "/test";
  URL lrURL = URL.newInstance(scheme, host, port, path);
  LocalResource localResource = LocalResource.newInstance(lrURL, LocalResourceType.FILE,
      LocalResourceVisibility.PUBLIC, 1, 1);
  Map<String, LocalResource> localResources = new HashMap<>();
  localResources.put(lrName, localResource);

  SubmitDAGRequestProto.Builder requestBuilder = SubmitDAGRequestProto.newBuilder().setDAGPlan(dagPlan)
      .setAdditionalAmResources(DagTypeConverters.convertFromLocalResources(localResources));
  try (FileOutputStream fileOutputStream = new FileOutputStream(requestFile)) {
    requestBuilder.build().writeTo(fileOutputStream);
  }

  DAGClientHandler dagClientHandler = mock(DAGClientHandler.class);
  ACLManager aclManager = mock(ACLManager.class);
  DAGClientAMProtocolBlockingPBServerImpl serverImpl = spy(new DAGClientAMProtocolBlockingPBServerImpl(
      dagClientHandler, FileSystem.get(conf)));
  when(dagClientHandler.getACLManager()).thenReturn(aclManager);
  when(dagClientHandler.submitDAG((DAGPlan)any(), (Map<String, LocalResource>)any())).thenReturn("dag-id");
  when(aclManager.checkAMModifyAccess((UserGroupInformation) any())).thenReturn(true);

  requestBuilder.clear().setSerializedRequestPath(requestFile.getAbsolutePath());
  serverImpl.submitDAG(null, requestBuilder.build());

  ArgumentCaptor<DAGPlan> dagPlanCaptor = ArgumentCaptor.forClass(DAGPlan.class);
  verify(dagClientHandler).submitDAG(dagPlanCaptor.capture(), localResourcesCaptor.capture());
  dagPlan = dagPlanCaptor.getValue();
  localResources = localResourcesCaptor.getValue();

  assertEquals(dagPlan.getName(), dagPlanName);
  assertEquals(dagPlan.getVertexCount(), 1);
  assertTrue(dagPlan.getSerializedSize() > maxIPCMsgSize);
  assertArrayEquals(randomBytes, dagPlan.getVertex(0).getProcessorDescriptor().getTezUserPayload().getUserPayload().
      toByteArray());
  assertEquals(localResources.size(), 1);
  assertTrue(localResources.containsKey(lrName));
  localResource = localResources.get(lrName);
  assertEquals(localResource.getType(), LocalResourceType.FILE);
  assertEquals(localResource.getVisibility(), LocalResourceVisibility.PUBLIC);
  lrURL = localResource.getResource();
  assertEquals(lrURL.getScheme(), scheme);
  assertEquals(lrURL.getHost(), host);
  assertEquals(lrURL.getPort(), port);
  assertEquals(lrURL.getFile(), path);
}

Example 17

Source File: TestMRRJobsDAGApi.java From tez with Apache License 2.0

4 votes

private LocalResource createLrObjFromPath(Path filePath) {
  return LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(filePath),
      LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, 0, 0);
}

Example 18

Source File: TestAMContainer.java From incubator-tez with Apache License 2.0

4 votes

private LocalResource createLocalResource(String name) {
  LocalResource lr = LocalResource.newInstance(URL.newInstance(null, "localhost", 2321, name),
      LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 1, 1000000);
  return lr;
}

Example 19

Source File: TestMRRJobsDAGApi.java From incubator-tez with Apache License 2.0

4 votes

private LocalResource createLrObjFromPath(Path filePath) {
  return LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(filePath),
      LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, 0, 0);
}

Example 20

Source File: LaunchContainerRunnable.java From attic-apex-core with Apache License 2.0

4 votes

public static void addFileToLocalResources(final String name, final FileStatus fileStatus, final LocalResourceType type, final Map<String, LocalResource> localResources)
{
  final LocalResource localResource = LocalResource.newInstance(ConverterUtils.getYarnUrlFromPath(fileStatus.getPath()),
      type, LocalResourceVisibility.APPLICATION, fileStatus.getLen(), fileStatus.getModificationTime());
  localResources.put(name, localResource);
}