Java Code Examples for org.apache.hadoop.fs.FileStatus#isSymlink()

The following examples show how to use org.apache.hadoop.fs.FileStatus#isSymlink() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestFileStatus.java    From hadoop with Apache License 2.0 6 votes vote down vote up
/**
 * Validates the toString method for FileStatus.
 * @param fileStatus FileStatus to be validated
 */
private void validateToString(FileStatus fileStatus) throws IOException {
  StringBuilder expected = new StringBuilder();
  expected.append("FileStatus{");
  expected.append("path=").append(fileStatus.getPath()).append("; ");
  expected.append("isDirectory=").append(fileStatus.isDirectory()).append("; ");
  if(!fileStatus.isDirectory()) {
    expected.append("length=").append(fileStatus.getLen()).append("; ");
    expected.append("replication=").append(fileStatus.getReplication()).append("; ");
    expected.append("blocksize=").append(fileStatus.getBlockSize()).append("; ");
  }
  expected.append("modification_time=").append(fileStatus.getModificationTime()).append("; ");
  expected.append("access_time=").append(fileStatus.getAccessTime()).append("; ");
  expected.append("owner=").append(fileStatus.getOwner()).append("; ");
  expected.append("group=").append(fileStatus.getGroup()).append("; ");
  expected.append("permission=").append(fileStatus.getPermission()).append("; ");
  if(fileStatus.isSymlink()) {
    expected.append("isSymlink=").append(true).append("; ");
    expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
  } else {
    expected.append("isSymlink=").append(false).append("}");
  }
  
  assertEquals(expected.toString(), fileStatus.toString());
}
 
Example 2
Source File: FileStatusDTO.java    From hudi with Apache License 2.0 6 votes vote down vote up
public static FileStatusDTO fromFileStatus(FileStatus fileStatus) {
  if (null == fileStatus) {
    return null;
  }

  FileStatusDTO dto = new FileStatusDTO();
  try {
    dto.path = FilePathDTO.fromPath(fileStatus.getPath());
    dto.length = fileStatus.getLen();
    dto.isdir = fileStatus.isDirectory();
    dto.blockReplication = fileStatus.getReplication();
    dto.blocksize = fileStatus.getBlockSize();
    dto.modificationTime = fileStatus.getModificationTime();
    dto.accessTime = fileStatus.getModificationTime();
    dto.symlink = fileStatus.isSymlink() ? FilePathDTO.fromPath(fileStatus.getSymlink()) : null;
    safeReadAndSetMetadata(dto, fileStatus);
  } catch (IOException ioe) {
    throw new HoodieException(ioe);
  }
  return dto;
}
 
Example 3
Source File: TestFileStatus.java    From big-c with Apache License 2.0 6 votes vote down vote up
/**
 * Validates the toString method for FileStatus.
 * @param fileStatus FileStatus to be validated
 */
private void validateToString(FileStatus fileStatus) throws IOException {
  StringBuilder expected = new StringBuilder();
  expected.append("FileStatus{");
  expected.append("path=").append(fileStatus.getPath()).append("; ");
  expected.append("isDirectory=").append(fileStatus.isDirectory()).append("; ");
  if(!fileStatus.isDirectory()) {
    expected.append("length=").append(fileStatus.getLen()).append("; ");
    expected.append("replication=").append(fileStatus.getReplication()).append("; ");
    expected.append("blocksize=").append(fileStatus.getBlockSize()).append("; ");
  }
  expected.append("modification_time=").append(fileStatus.getModificationTime()).append("; ");
  expected.append("access_time=").append(fileStatus.getAccessTime()).append("; ");
  expected.append("owner=").append(fileStatus.getOwner()).append("; ");
  expected.append("group=").append(fileStatus.getGroup()).append("; ");
  expected.append("permission=").append(fileStatus.getPermission()).append("; ");
  if(fileStatus.isSymlink()) {
    expected.append("isSymlink=").append(true).append("; ");
    expected.append("symlink=").append(fileStatus.getSymlink()).append("}");
  } else {
    expected.append("isSymlink=").append(false).append("}");
  }
  
  assertEquals(expected.toString(), fileStatus.toString());
}
 
Example 4
Source File: TestFSDownload.java    From hadoop with Apache License 2.0 5 votes vote down vote up
private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 
Example 5
Source File: BaseExpression.java    From examples with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link FileStatus} from the {@link PathData} item.
 * If the current options require links to be followed then the
 * returned file status is that of the linked file.
 * @param item PathData
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item) throws IOException {
  FileStatus fileStatus = item.stat;
  if(getOptions().isFollowLink() && fileStatus.isSymlink()) {
    Path linkedFile = fileStatus.getSymlink();
    fileStatus = getFileSystem(item).getFileStatus(linkedFile);
  }
  return fileStatus;
}
 
Example 6
Source File: InstrumentedFileSystemUtils.java    From incubator-gobblin with Apache License 2.0 5 votes vote down vote up
/**
 * Replace the scheme of the input {@link FileStatus} if it matches the string to replace.
 */
public static FileStatus replaceScheme(FileStatus st, String replace, String replacement) {
  if (replace != null && replace.equals(replacement)) {
    return st;
  }
  try {
    return new FileStatus(st.getLen(), st.isDir(), st.getReplication(), st.getBlockSize(), st.getModificationTime(),
        st.getAccessTime(), st.getPermission(), st.getOwner(), st.getGroup(), st.isSymlink() ? st.getSymlink() : null,
        replaceScheme(st.getPath(), replace, replacement));
  } catch (IOException ioe) {
    throw new RuntimeException(ioe);
  }
}
 
Example 7
Source File: FileStatusEntity.java    From eagle with Apache License 2.0 5 votes vote down vote up
public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if(status.isSymlink()) {
        this.symlink = status.getSymlink();
    }
}
 
Example 8
Source File: FileStatusEntity.java    From Eagle with Apache License 2.0 5 votes vote down vote up
public FileStatusEntity(FileStatus status) throws IOException {
    //this.path = status.getPath();
    this.length = status.getLen();
    this.isdir = status.isDirectory();
    this.block_replication = status.getReplication();
    this.blocksize = status.getBlockSize();
    this.modification_time = status.getModificationTime();
    this.access_time = status.getAccessTime();
    this.permission = status.getPermission();
    this.owner = status.getOwner();
    this.group = status.getGroup();
    if(status.isSymlink()) {
        this.symlink = status.getSymlink();
    }
}
 
Example 9
Source File: BaseExpression.java    From big-c with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link FileStatus} from the {@link PathData} item. If the
 * current options require links to be followed then the returned file status
 * is that of the linked file.
 *
 * @param item
 *          PathData
 * @param depth
 *          current depth in the process directories
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item, int depth)
    throws IOException {
  FileStatus fileStatus = item.stat;
  if (fileStatus.isSymlink()) {
    if (options.isFollowLink() || (options.isFollowArgLink() &&
        (depth == 0))) {
      Path linkedFile = item.fs.resolvePath(fileStatus.getSymlink());
      fileStatus = getFileSystem(item).getFileStatus(linkedFile);
    }
  }
  return fileStatus;
}
 
Example 10
Source File: HttpFSFileSystem.java    From big-c with Apache License 2.0 5 votes vote down vote up
public static FILE_TYPE getType(FileStatus fileStatus) {
  if (fileStatus.isFile()) {
    return FILE;
  }
  if (fileStatus.isDirectory()) {
    return DIRECTORY;
  }
  if (fileStatus.isSymlink()) {
    return SYMLINK;
  }
  throw new IllegalArgumentException("Could not determine filetype for: " +
                                     fileStatus.getPath());
}
 
Example 11
Source File: TestFSDownload.java    From big-c with Apache License 2.0 5 votes vote down vote up
private void verifyPermsRecursively(FileSystem fs,
    FileContext files, Path p,
    LocalResourceVisibility vis) throws IOException {
  FileStatus status = files.getFileStatus(p);
  if (status.isDirectory()) {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_DIR_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_DIR_PERMS.toShort());
    }
    if (!status.isSymlink()) {
      FileStatus[] statuses = fs.listStatus(p);
      for (FileStatus stat : statuses) {
        verifyPermsRecursively(fs, files, stat.getPath(), vis);
      }
    }
  }
  else {
    if (vis == LocalResourceVisibility.PUBLIC) {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PUBLIC_FILE_PERMS.toShort());
    }
    else {
      Assert.assertTrue(status.getPermission().toShort() ==
        FSDownload.PRIVATE_FILE_PERMS.toShort());
    }
  }      
}
 
Example 12
Source File: RemoteNodeFileSystem.java    From dremio-oss with Apache License 2.0 5 votes vote down vote up
/**
 * Converts a Hadoop {@link FileStatus} instance into a protobuf
 * {@link DFSProtos.FileStatus}
 *
 * @param status
 *          the Hadoop status instance to convert
 * @return a protobuf status instance
 * @throws IOException
 */
static DFS.FileStatus toProtoFileStatus(FileStatus status) throws IOException {
  DFS.FileStatus.Builder builder = DFS.FileStatus.newBuilder();

  builder
    .setLength(status.getLen())
    .setIsDirectory(status.isDirectory())
    .setBlockReplication(status.getReplication())
    .setBlockSize(status.getBlockSize())
    .setModificationTime(status.getModificationTime())
    .setAccessTime(status.getAccessTime());

  // Handling potential null values
  if (status.getPath() != null) {
    builder = builder.setPath(status.getPath().toUri().getPath());
  }
  if (status.getPermission() != null) {
    builder = builder.setPermission(status.getPermission().toExtendedShort());
  }
  if (status.getOwner() != null) {
    builder = builder.setOwner(status.getOwner());
  }
  if (status.getGroup() != null) {
    builder = builder.setGroup(status.getGroup());
  }
  if (status.isSymlink()) {
    builder = builder.setSymlink(status.getSymlink().toString());
  }

  return builder.build();
}
 
Example 13
Source File: BaseExpression.java    From hadoop with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the {@link FileStatus} from the {@link PathData} item. If the
 * current options require links to be followed then the returned file status
 * is that of the linked file.
 *
 * @param item
 *          PathData
 * @param depth
 *          current depth in the process directories
 * @return FileStatus
 */
protected FileStatus getFileStatus(PathData item, int depth)
    throws IOException {
  FileStatus fileStatus = item.stat;
  if (fileStatus.isSymlink()) {
    if (options.isFollowLink() || (options.isFollowArgLink() &&
        (depth == 0))) {
      Path linkedFile = item.fs.resolvePath(fileStatus.getSymlink());
      fileStatus = getFileSystem(item).getFileStatus(linkedFile);
    }
  }
  return fileStatus;
}
 
Example 14
Source File: HttpFSFileSystem.java    From hadoop with Apache License 2.0 5 votes vote down vote up
public static FILE_TYPE getType(FileStatus fileStatus) {
  if (fileStatus.isFile()) {
    return FILE;
  }
  if (fileStatus.isDirectory()) {
    return DIRECTORY;
  }
  if (fileStatus.isSymlink()) {
    return SYMLINK;
  }
  throw new IllegalArgumentException("Could not determine filetype for: " +
                                     fileStatus.getPath());
}
 
Example 15
Source File: BasicFormatMatcher.java    From Bats with Apache License 2.0 5 votes vote down vote up
public boolean matches(DrillFileSystem fs, FileStatus status) throws IOException{
  if (ranges.isEmpty() || status.isDirectory()) {
    return false;
  }
  // walk all the way down in the symlinks until a hard entry is reached
  FileStatus current = status;
  while (current.isSymlink()) {
    current = fs.getFileStatus(status.getSymlink());
  }
  // if hard entry is not a file nor can it be a symlink then it is not readable simply deny matching.
  if (!current.isFile()) {
    return false;
  }

  final Range<Long> fileRange = Range.closedOpen( 0L, status.getLen());

  try (FSDataInputStream is = fs.open(status.getPath())) {
    for(RangeMagics rMagic : ranges) {
      Range<Long> r = rMagic.range;
      if (!fileRange.encloses(r)) {
        continue;
      }
      int len = (int) (r.upperEndpoint() - r.lowerEndpoint());
      byte[] bytes = new byte[len];
      is.readFully(r.lowerEndpoint(), bytes);
      for (byte[] magic : rMagic.magics) {
        if (Arrays.equals(magic, bytes)) {
          return true;
        }
      }
    }
  }
  return false;
}
 
Example 16
Source File: DistributedFileSystem.java    From big-c with Apache License 2.0 4 votes vote down vote up
/**
 * Move blocks from srcs to trg and delete srcs afterwards.
 * The file block sizes must be the same.
 * 
 * @param trg existing file to append to
 * @param psrcs list of files (same block size, same replication)
 * @throws IOException
 */
@Override
public void concat(Path trg, Path [] psrcs) throws IOException {
  statistics.incrementWriteOps(1);
  // Make target absolute
  Path absF = fixRelativePart(trg);
  // Make all srcs absolute
  Path[] srcs = new Path[psrcs.length];
  for (int i=0; i<psrcs.length; i++) {
    srcs[i] = fixRelativePart(psrcs[i]);
  }
  // Try the concat without resolving any links
  String[] srcsStr = new String[psrcs.length];
  try {
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(trg), srcsStr);
  } catch (UnresolvedLinkException e) {
    // Exception could be from trg or any src.
    // Fully resolve trg and srcs. Fail if any of them are a symlink.
    FileStatus stat = getFileLinkStatus(absF);
    if (stat.isSymlink()) {
      throw new IOException("Cannot concat with a symlink target: "
          + trg + " -> " + stat.getPath());
    }
    absF = fixRelativePart(stat.getPath());
    for (int i=0; i<psrcs.length; i++) {
      stat = getFileLinkStatus(srcs[i]);
      if (stat.isSymlink()) {
        throw new IOException("Cannot concat with a symlink src: "
            + psrcs[i] + " -> " + stat.getPath());
      }
      srcs[i] = fixRelativePart(stat.getPath());
    }
    // Try concat again. Can still race with another symlink.
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(absF), srcsStr);
  }
}
 
Example 17
Source File: DistributedFileSystem.java    From hadoop with Apache License 2.0 4 votes vote down vote up
/**
 * Move blocks from srcs to trg and delete srcs afterwards.
 * The file block sizes must be the same.
 * 
 * @param trg existing file to append to
 * @param psrcs list of files (same block size, same replication)
 * @throws IOException
 */
@Override
public void concat(Path trg, Path [] psrcs) throws IOException {
  statistics.incrementWriteOps(1);
  // Make target absolute
  Path absF = fixRelativePart(trg);
  // Make all srcs absolute
  Path[] srcs = new Path[psrcs.length];
  for (int i=0; i<psrcs.length; i++) {
    srcs[i] = fixRelativePart(psrcs[i]);
  }
  // Try the concat without resolving any links
  String[] srcsStr = new String[psrcs.length];
  try {
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(trg), srcsStr);
  } catch (UnresolvedLinkException e) {
    // Exception could be from trg or any src.
    // Fully resolve trg and srcs. Fail if any of them are a symlink.
    FileStatus stat = getFileLinkStatus(absF);
    if (stat.isSymlink()) {
      throw new IOException("Cannot concat with a symlink target: "
          + trg + " -> " + stat.getPath());
    }
    absF = fixRelativePart(stat.getPath());
    for (int i=0; i<psrcs.length; i++) {
      stat = getFileLinkStatus(srcs[i]);
      if (stat.isSymlink()) {
        throw new IOException("Cannot concat with a symlink src: "
            + psrcs[i] + " -> " + stat.getPath());
      }
      srcs[i] = fixRelativePart(stat.getPath());
    }
    // Try concat again. Can still race with another symlink.
    for (int i=0; i<psrcs.length; i++) {
      srcsStr[i] = getPathName(srcs[i]);
    }
    dfs.concat(getPathName(absF), srcsStr);
  }
}
 
Example 18
Source File: Type.java    From examples with Apache License 2.0 4 votes vote down vote up
public boolean matches(FileStatus stat) {
  return stat.isSymlink();
}