Java Code Examples for org.apache.hadoop.fs.LocatedFileStatus#getPath()

void checkPath() {
    try {
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(folderPath, false);
        if (files == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            Path path = fileStatus.getPath();
            String name = path.getName();

            if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
                dataFilePath = path;
            } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
                metaFilePath = path;
            } else {
                logger.warn("Contains invalid file {} in path {}", path, folderPath);
            }
        }
        if (dataFilePath == null || metaFilePath == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
    } catch (IOException e) {
        throw new RuntimeException("io error", e);
    }
}
 

private void verifyLocatedFileStatus(
    JobConf conf, List<LocatedFileStatus> stats)
    throws IOException {
  if (!conf.getBoolean("mapred.fileinputformat.verifysplits", true)) {
    return;
  }
  for (LocatedFileStatus stat: stats) {
    long fileLen = stat.getLen();
    long blockLenTotal = 0;
    for (BlockLocation loc: stat.getBlockLocations()) {
      blockLenTotal += loc.getLength();
    }
    if (blockLenTotal != fileLen) {
      throw new IOException("Error while getting located status, " +
        stat.getPath() + " has length " + fileLen + " but blocks total is " +
        blockLenTotal);
    }
  }
}
 

public static void literalAllSegments(FileSystem fileSystem, Path dir, Consumer<LocatedFileStatus> consumer) throws IOException {
    RemoteIterator<LocatedFileStatus> files = fileSystem.listFiles(dir, true);
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        if (!fileStatus.isFile()) {
            continue;
        }
        if (fileStatus.getLen() == 0) {
            continue;
        }

        Path path = fileStatus.getPath();
        if (checkSegmentByPath(path)) {
            consumer.accept(fileStatus);
        }
    }
}
 

void checkPath() {
    try {
        RemoteIterator<LocatedFileStatus> files = fs.listFiles(folderPath, false);
        if (files == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
        while (files.hasNext()) {
            LocatedFileStatus fileStatus = files.next();
            Path path = fileStatus.getPath();
            String name = path.getName();

            if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
                dataFilePath = path;
            } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
                metaFilePath = path;
            } else {
                logger.warn("Contains invalid file {} in path {}", path, folderPath);
            }
        }
        if (dataFilePath == null || metaFilePath == null) {
            throw new IllegalArgumentException("Invalid path " + folderPath);
        }
    } catch (IOException e) {
        throw new RuntimeException("io error", e);
    }
}
 

private static SortedSet<byte[]> readFileToSearch(final Configuration conf,
    final FileSystem fs, final LocatedFileStatus keyFileStatus) throws IOException,
    InterruptedException {
  SortedSet<byte []> result = new TreeSet<>(Bytes.BYTES_COMPARATOR);
  // Return entries that are flagged Counts.UNDEFINED in the value. Return the row. This is
  // what is missing.
  TaskAttemptContext context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
  try (SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader rr =
      new SequenceFileAsBinaryInputFormat.SequenceFileAsBinaryRecordReader()) {
    InputSplit is =
      new FileSplit(keyFileStatus.getPath(), 0, keyFileStatus.getLen(), new String [] {});
    rr.initialize(is, context);
    while (rr.nextKeyValue()) {
      rr.getCurrentKey();
      BytesWritable bw = rr.getCurrentValue();
      if (Verify.VerifyReducer.whichType(bw.getBytes()) == Verify.Counts.UNDEFINED) {
        byte[] key = new byte[rr.getCurrentKey().getLength()];
        System.arraycopy(rr.getCurrentKey().getBytes(), 0, key, 0, rr.getCurrentKey()
            .getLength());
        result.add(key);
      }
    }
  }
  return result;
}
 

/**
 * Appends the compressed files found within the directory to the zip stream as the named file
 */
private void appendPreCompressedFile(ModalZipOutputStream out, Path dir, String filename, String headerRow)
  throws IOException {
  RemoteIterator<LocatedFileStatus> files = sourceFs.listFiles(dir, false);
  List<InputStream> parts = Lists.newArrayList();

  // Add the header first, which must also be compressed
  ByteArrayOutputStream header = new ByteArrayOutputStream();
  D2Utils.compress(new ByteArrayInputStream(headerRow.getBytes()), header);
  parts.add(new ByteArrayInputStream(header.toByteArray()));

  // Locate the streams to the compressed content on HDFS
  while (files.hasNext()) {
    LocatedFileStatus fs = files.next();
    Path path = fs.getPath();
    if (path.toString().endsWith(D2Utils.FILE_EXTENSION)) {
      LOG.info("Deflated content to merge: {} ", path);
      parts.add(sourceFs.open(path));
    }
  }

  // create the Zip entry, and write the compressed bytes
  org.gbif.hadoop.compress.d2.zip.ZipEntry ze = new org.gbif.hadoop.compress.d2.zip.ZipEntry(filename);
  out.putNextEntry(ze, ModalZipOutputStream.MODE.PRE_DEFLATED);
  try (D2CombineInputStream in = new D2CombineInputStream(parts)) {
    ByteStreams.copy(in, out);
    in.close(); // important so counts are accurate
    ze.setSize(in.getUncompressedLength()); // important to set the sizes and CRC
    ze.setCompressedSize(in.getCompressedLength());
    ze.setCrc(in.getCrc32());
  } finally {
    out.closeEntry();
  }
}
 

/** 
 * Generate the list of files and make them into FileSplits.
 */ 
public List<InputSplit> getSplits(JobContext job
                                  ) throws IOException {
  long minSize = Math.max(getFormatMinSplitSize(), getMinSplitSize(job));
  long maxSize = getMaxSplitSize(job);

  // generate splits
  List<InputSplit> splits = new ArrayList<InputSplit>();
  for (LocatedFileStatus file: listLocatedStatus(job)) {
    Path path = file.getPath();
    long length = file.getLen();
    BlockLocation[] blkLocations = file.getBlockLocations();

    if ((length != 0) && isSplitable(job, path)) { 
      long blockSize = file.getBlockSize();
      long splitSize = computeSplitSize(blockSize, minSize, maxSize);

      long bytesRemaining = length;
      while (((double) bytesRemaining)/splitSize > SPLIT_SLOP) {
        int blkIndex = getBlockIndex(blkLocations, length-bytesRemaining);
        splits.add(new FileSplit(path, length-bytesRemaining, splitSize, 
                                 blkLocations[blkIndex].getHosts()));
        bytesRemaining -= splitSize;
      }
      
      if (bytesRemaining != 0) {
        splits.add(new FileSplit(path, length-bytesRemaining, bytesRemaining, 
                   blkLocations[blkLocations.length-1].getHosts()));
      }
    } else if (length != 0) {
      splits.add(new FileSplit(path, 0, length, blkLocations[0].getHosts()));
    } else { 
      //Create empty hosts array for zero length files
      splits.add(new FileSplit(path, 0, length, new String[0]));
    }
  }
  LOG.debug("Total # of splits: " + splits.size());
  return splits;
}
 

@Test
public void testProcessEvents() throws FileNotFoundException, IOException {
	int searchEventsCount = 101;
	List<Event> searchEvents = generateSearchAnalyticsDataService
			.getSearchEvents(searchEventsCount);

	flumeHDFSSinkService.processEvents(searchEvents);

	// list all files and check data.
	Path dirPath = new Path(hadoopClusterService.getHDFSUri()
			+ "/searchevents");
	// FileStatus[] dirStat = fs.listStatus(dirPath);
	// Path fList[] = FileUtil.stat2Paths(dirStat);

	DistributedFileSystem fs = hadoopClusterService.getFileSystem();
	RemoteIterator<LocatedFileStatus> files = fs.listFiles(dirPath, true);
	while (files.hasNext()) {
		LocatedFileStatus locatedFileStatus = files.next();
		System.out.println("Check:" + locatedFileStatus.getPath());
		if (locatedFileStatus.isFile()) {
			Path path = locatedFileStatus.getPath();
			if (path.getName().startsWith("searchevents")) {
				FSDataInputStream input = fs.open(path);
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(input));
				String body = null;
				while ((body = reader.readLine()) != null) {
					System.out.println("body is:" + body);
				}
				reader.close();
				input.close();
			}
		}
	}
}
 

protected int printAndCountHdfsFileDirData(String path, String filePrefix,
		boolean print, boolean count) throws IOException {
	int recordsCount = 0;
	DistributedFileSystem fs = hadoopClusterService.getFileSystem();
	RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path(path),
			true);
	while (files.hasNext()) {
		LocatedFileStatus locatedFileStatus = files.next();
		System.out.println("Check:" + locatedFileStatus.getPath());
		if (locatedFileStatus.isFile()) {
			Path filePath = locatedFileStatus.getPath();
			if (filePath.getName().startsWith(filePrefix)) {
				FSDataInputStream input = fs.open(filePath);
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(input));
				String body = null;
				while ((body = reader.readLine()) != null) {
					if (print) {
						System.out.println("file is: " + filePath.getName() + "body is:" + body);
					}
					if (count) {
						recordsCount++;
					}
				}
				reader.close();
				input.close();
			}
		}
	}
	return recordsCount;
}
 

private void FlumehdfsSinkAndTestData(List<Event> searchEvents)
		throws EventDeliveryException, IOException, FileNotFoundException {

	flumeHDFSSinkService.processEvents(searchEvents);

	// list all files and check data.
	Path dirPath = new Path(hadoopClusterService.getHDFSUri()
			+ "/searchevents");
	// FileStatus[] dirStat = fs.listStatus(dirPath);
	// Path fList[] = FileUtil.stat2Paths(dirStat);

	DistributedFileSystem fs = hadoopClusterService.getFileSystem();
	RemoteIterator<LocatedFileStatus> files = fs.listFiles(dirPath, true);
	while (files.hasNext()) {
		LocatedFileStatus locatedFileStatus = files.next();
		System.out.println("Check:" + locatedFileStatus.getPath());
		if (locatedFileStatus.isFile()) {
			Path path = locatedFileStatus.getPath();
			if (path.getName().startsWith("searchevents")) {
				FSDataInputStream input = fs.open(path);
				BufferedReader reader = new BufferedReader(
						new InputStreamReader(input));
				String body = null;
				while ((body = reader.readLine()) != null) {
					System.out.println("body is:" + body);
				}
				reader.close();
				input.close();
			}
		}
	}
}
 

private boolean isValidFragmentPath(FileSystem fs, Path path) throws IOException {
    RemoteIterator<LocatedFileStatus> files = fs.listFiles(path, false);
    if (files == null) {
        logger.warn("Invalid fragment path:{}, empty folder", path);
        return false;
    }
    boolean hasDataFile = false;
    boolean hasMetaFile = false;
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        Path childPath = fileStatus.getPath();
        String name = childPath.getName();

        if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
            hasDataFile = true;
        } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
            hasMetaFile = true;
        } else {
            logger.warn("Contains invalid file {} in path {}", childPath, path);
        }
    }
    if (hasDataFile && hasMetaFile) {
        return true;
    } else {
        logger.warn("Invalid fragment path:{}, data file exists:{}, meta file exists:{}", path, hasDataFile,
                hasMetaFile);
        return false;
    }
}
 

protected Path getTestFile(FileSystem fs) throws IOException {
    createTestFiles(1, 1);
    Path file = null;
    for (RemoteIterator<LocatedFileStatus> it = fs.listFiles(new Path(this.fmc.getBaseHDFSDir()), true); it.hasNext();) {
        LocatedFileStatus status = it.next();
        if (status.isFile()) {
            file = status.getPath();
            break;
        }
    }
    return file;
}
 

private boolean isValidFragmentPath(FileSystem fs, Path path) throws IOException {
    RemoteIterator<LocatedFileStatus> files = fs.listFiles(path, false);
    if (files == null) {
        logger.warn("Invalid fragment path:{}, empty folder", path);
        return false;
    }
    boolean hasDataFile = false;
    boolean hasMetaFile = false;
    while (files.hasNext()) {
        LocatedFileStatus fileStatus = files.next();
        Path childPath = fileStatus.getPath();
        String name = childPath.getName();

        if (name.endsWith(Constants.DATA_FILE_SUFFIX)) {
            hasDataFile = true;
        } else if (name.endsWith(Constants.META_FILE_SUFFIX)) {
            hasMetaFile = true;
        } else {
            logger.warn("Contains invalid file {} in path {}", childPath, path);
        }
    }
    if (hasDataFile && hasMetaFile) {
        return true;
    } else {
        logger.warn("Invalid fragment path:{}, data file exists:{}, meta file exists:{}", path, hasDataFile,
                hasMetaFile);
        return false;
    }
}
 

public static void main(String[] args) throws IOException, InterruptedException, SQLException {
  if(args.length < 3 || args.length > 4) {
    System.err.println("Usage: DumpHDFSData namenode_url homedir tablename");
    System.err.println("  Dumps the RAW data for the table tablename into a CSV format for debugging purposes");
    System.exit(1);
  }
  
  
  String namenodeURL = args[0];
  String homeDir = args[1];
  String table = args[2];
  
  
  Configuration conf = new Configuration();
  conf.set("fs.default.name", namenodeURL);
  FileSystem fs = FileSystem.get(conf);
  
  PrintStream out = new PrintStream(table + ".csv");
  try {
    String fullTable = RowInputFormat.getFullyQualifiedTableName(table);
    String folder = HdfsRegionManager.getRegionFolder(Misc.getRegionPath(fullTable));
    RemoteIterator<LocatedFileStatus> fileItr = fs.listFiles(new Path(homeDir + "/" + folder), true);
    
    conf.set(RowInputFormat.HOME_DIR, homeDir);
    conf.set(RowInputFormat.INPUT_TABLE, table);
    
    boolean wroteHeader = false;
    TaskAttemptContextImpl context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    
    while(fileItr.hasNext()) {
      LocatedFileStatus file = fileItr.next();
      Path path = file.getPath();
      if(!path.getName().endsWith("hop")) {
        continue;
      }
      CombineFileSplit split = new CombineFileSplit(new Path[] { path } , new long[] { file.getLen()});
      RowRecordReader reader = new RowRecordReader();
      reader.initialize(split, context);
      while(reader.nextKeyValue()) {
        Row row = reader.getCurrentValue();
        ResultSet rs = row.getRowAsResultSet();
        Type op = row.getEventType();
        long ts = row.getTimestamp();

        int numColumns = rs.getMetaData().getColumnCount();
        if(!wroteHeader) {
          out.print("timestamp,operation,path");
          for(int i =1; i <= numColumns; i++) {
            out.print(",");
            out.print(rs.getMetaData().getColumnName(i));
          }
          out.println();
          wroteHeader = true;
        }

        out.print(ts);
        out.print(",");
        out.print(op);
        out.print(",");
        out.print(path);
        for(int i =1; i <= numColumns; i++) {
          out.print(",");
          String s= rs.getString(i);
          if(s != null) {
            s = s.replaceAll("([,\n])", "\\\\1");
          } else {
            s = "NULL";
          }
          out.print(s);
        }
        out.println();
      }
    }
  
  } finally {
    out.close();
  }
}
 

public static void main(String[] args) throws IOException, InterruptedException, SQLException {
  if(args.length < 3 || args.length > 4) {
    System.err.println("Usage: DumpHDFSData namenode_url homedir tablename");
    System.err.println("  Dumps the RAW data for the table tablename into a CSV format for debugging purposes");
    System.exit(1);
  }
  
  
  String namenodeURL = args[0];
  String homeDir = args[1];
  String table = args[2];
  
  
  Configuration conf = new Configuration();
  conf.set("fs.default.name", namenodeURL);
  FileSystem fs = FileSystem.get(conf);
  
  PrintStream out = new PrintStream(table + ".csv");
  try {
    String fullTable = RowInputFormat.getFullyQualifiedTableName(table);
    String folder = HdfsRegionManager.getRegionFolder(Misc.getRegionPath(fullTable));
    RemoteIterator<LocatedFileStatus> fileItr = fs.listFiles(new Path(homeDir + "/" + folder), true);
    
    conf.set(RowInputFormat.HOME_DIR, homeDir);
    conf.set(RowInputFormat.INPUT_TABLE, table);
    
    boolean wroteHeader = false;
    TaskAttemptContextImpl context = new TaskAttemptContextImpl(conf, new TaskAttemptID());
    
    while(fileItr.hasNext()) {
      LocatedFileStatus file = fileItr.next();
      Path path = file.getPath();
      if(!path.getName().endsWith("hop")) {
        continue;
      }
      CombineFileSplit split = new CombineFileSplit(new Path[] { path } , new long[] { file.getLen()});
      RowRecordReader reader = new RowRecordReader();
      reader.initialize(split, context);
      while(reader.nextKeyValue()) {
        Row row = reader.getCurrentValue();
        ResultSet rs = row.getRowAsResultSet();
        Type op = row.getEventType();
        long ts = row.getTimestamp();

        int numColumns = rs.getMetaData().getColumnCount();
        if(!wroteHeader) {
          out.print("timestamp,operation,path");
          for(int i =1; i <= numColumns; i++) {
            out.print(",");
            out.print(rs.getMetaData().getColumnName(i));
          }
          out.println();
          wroteHeader = true;
        }

        out.print(ts);
        out.print(",");
        out.print(op);
        out.print(",");
        out.print(path);
        for(int i =1; i <= numColumns; i++) {
          out.print(",");
          String s= rs.getString(i);
          if(s != null) {
            s = s.replaceAll("([,\n])", "\\\\1");
          } else {
            s = "NULL";
          }
          out.print(s);
        }
        out.println();
      }
    }
  
  } finally {
    out.close();
  }
}
 

public static HiveReaderProto.FileSystemPartitionUpdateKey getFSBasedUpdateKey(String partitionDir, JobConf job,
                                                                               boolean isRecursive, boolean directoriesOnly,
                                                                               int partitionId) {
  final List<HiveReaderProto.FileSystemCachedEntity> cachedEntities = new ArrayList<>();
  final Path rootLocation = new Path(partitionDir);
  try {
    // TODO: DX-16001 - make async configurable for Hive.
    final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(rootLocation, job);

    if (fs.exists(rootLocation)) {
      final FileStatus rootStatus = fs.getFileStatus(rootLocation);
      if (rootStatus.isDirectory()) {
        cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
          .setPath(EMPTY_STRING)
          .setLastModificationTime(rootStatus.getModificationTime())
          .setIsDir(true)
          .build());

        final RemoteIterator<LocatedFileStatus> statuses = isRecursive ? fs.listFiles(rootLocation, true) : fs.listFiles(rootLocation, false);
        while (statuses.hasNext()) {
          LocatedFileStatus fileStatus = statuses.next();
          final Path filePath = fileStatus.getPath();
          if (fileStatus.isDirectory()) {
            cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
              .setPath(PathUtils.relativePath(filePath, rootLocation))
              .setLastModificationTime(fileStatus.getModificationTime())
              .setIsDir(true)
              .build());
          } else if (fileStatus.isFile() && !directoriesOnly) {
            cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
              .setPath(PathUtils.relativePath(filePath, rootLocation))
              .setLastModificationTime(fileStatus.getModificationTime())
              .setIsDir(false)
              .build());
          }
        }
      } else {
        cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
          .setPath(EMPTY_STRING)
          .setLastModificationTime(rootStatus.getModificationTime())
          .setIsDir(false)
          .build());
      }
      return HiveReaderProto.FileSystemPartitionUpdateKey.newBuilder()
        .setPartitionId(partitionId)
        .setPartitionRootDir(fs.makeQualified(rootLocation).toString())
        .addAllCachedEntities(cachedEntities)
        .build();
    }
    return null;
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 

public static HiveReaderProto.FileSystemPartitionUpdateKey getFSBasedUpdateKey(String partitionDir, JobConf job,
                                                                               boolean isRecursive, boolean directoriesOnly,
                                                                               int partitionId) {
  final List<HiveReaderProto.FileSystemCachedEntity> cachedEntities = new ArrayList<>();
  final Path rootLocation = new Path(partitionDir);
  try {
    // TODO: DX-16001 - make async configurable for Hive.
    final HadoopFileSystemWrapper fs = new HadoopFileSystemWrapper(rootLocation, job);

    if (fs.exists(rootLocation)) {
      final FileStatus rootStatus = fs.getFileStatus(rootLocation);
      if (rootStatus.isDirectory()) {
        cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
          .setPath(EMPTY_STRING)
          .setLastModificationTime(rootStatus.getModificationTime())
          .setIsDir(true)
          .build());

        final RemoteIterator<LocatedFileStatus> statuses = isRecursive ? fs.listFiles(rootLocation, true) : fs.listFiles(rootLocation, false);
        while (statuses.hasNext()) {
          LocatedFileStatus fileStatus = statuses.next();
          final Path filePath = fileStatus.getPath();
          if (fileStatus.isDirectory()) {
            cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
              .setPath(PathUtils.relativePath(filePath, rootLocation))
              .setLastModificationTime(fileStatus.getModificationTime())
              .setIsDir(true)
              .build());
          } else if (fileStatus.isFile() && !directoriesOnly) {
            cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
              .setPath(PathUtils.relativePath(filePath, rootLocation))
              .setLastModificationTime(fileStatus.getModificationTime())
              .setIsDir(false)
              .build());
          }
        }
      } else {
        cachedEntities.add(HiveReaderProto.FileSystemCachedEntity.newBuilder()
          .setPath(EMPTY_STRING)
          .setLastModificationTime(rootStatus.getModificationTime())
          .setIsDir(false)
          .build());
      }
      return HiveReaderProto.FileSystemPartitionUpdateKey.newBuilder()
        .setPartitionId(partitionId)
        .setPartitionRootDir(fs.makeQualified(rootLocation).toString())
        .addAllCachedEntities(cachedEntities)
        .build();
    }
    return null;
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}