Java Code Examples for org.apache.hadoop.fs.LocatedFileStatus#isDir()

The following examples show how to use org.apache.hadoop.fs.LocatedFileStatus#isDir() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: DatanodeBenThread.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
public int getNumberOfFiles() throws IOException {
  DistributedFileSystem dfs = (DistributedFileSystem)fs;
  RemoteIterator<LocatedFileStatus> iter = dfs.listLocatedStatus(outputPath);
  int fn = 0;
  while (iter.hasNext()) {
    LocatedFileStatus lfs = iter.next();
    if (lfs.isDir()) 
      continue;
    if (lfs.getBlockLocations().length != 1) 
      continue;
    String curHost = rtc.cur_datanode;
    for (String host: lfs.getBlockLocations()[0].getHosts()) {
      if (curHost.equals(host)){
        fn++;
        break;
      }
    }
  }
  LOG.info(" Found " + fn + " files in " + dfs.getUri());
  return fn;
}
Example 2
Source File: RaidNode.java    From RDFS with Apache License 2.0 6 votes vote down vote up 
public static List<LocatedFileStatus> listDirectoryRaidLocatedFileStatus(
		Configuration conf, FileSystem srcFs, Path p) throws IOException {
	long minFileSize = conf.getLong(MINIMUM_RAIDABLE_FILESIZE_KEY,
			MINIMUM_RAIDABLE_FILESIZE);
	List<LocatedFileStatus> lfs = new ArrayList<LocatedFileStatus>();
	RemoteIterator<LocatedFileStatus> iter = srcFs.listLocatedStatus(p);
	while (iter.hasNext()) {
		LocatedFileStatus stat = iter.next();
		if (stat.isDir()) {
			return null;
		}
		// We don't raid too small files
		if (stat.getLen() < minFileSize) {
			continue;
		}
		lfs.add(stat);
	}
	if (lfs.size() == 0)
		return null;
	return lfs;
}
Example 3
Source File: FileInputFormat.java    From RDFS with Apache License 2.0 5 votes vote down vote up 
/**
 * Add files in the input path recursively into the results.
 * @param result
 *          The List to store all files together with their block locations
 * @param fs
 *          The FileSystem.
 * @param path
 *          The input path.
 * @param inputFilter
 *          The input filter that can be used to filter files/dirs. 
 * @throws IOException
 */
protected void addLocatedInputPathRecursively(List<LocatedFileStatus> result,
    FileSystem fs, Path path, PathFilter inputFilter) 
    throws IOException {
  for(RemoteIterator<LocatedFileStatus> itor = 
    fs.listLocatedStatus(path, inputFilter); itor.hasNext();) {
    LocatedFileStatus stat = itor.next();
    if (stat.isDir()) {
      addLocatedInputPathRecursively(result, fs, stat.getPath(), inputFilter);
    } else {
      result.add(stat);
    }
  }          
}