Java Code Examples for org.apache.hadoop.hbase.util.FSUtils#isSameHdfs()

/**
 * Bulk load: Add a specified store file to the specified family.
 * If the source file is on the same different file-system is moved from the
 * source location to the destination location, otherwise is copied over.
 *
 * @param familyName Family that will gain the file
 * @param srcPath {@link Path} to the file to import
 * @param seqNum Bulk Load sequence number
 * @return The destination {@link Path} of the bulk loaded file
 * @throws IOException
 */
Pair<Path, Path> bulkLoadStoreFile(final String familyName, Path srcPath, long seqNum)
    throws IOException {
  // Copy the file if it's on another filesystem
  FileSystem srcFs = srcPath.getFileSystem(conf);
  srcPath = srcFs.resolvePath(srcPath);
  FileSystem realSrcFs = srcPath.getFileSystem(conf);
  FileSystem desFs = fs instanceof HFileSystem ? ((HFileSystem)fs).getBackingFs() : fs;

  // We can't compare FileSystem instances as equals() includes UGI instance
  // as part of the comparison and won't work when doing SecureBulkLoad
  // TODO deal with viewFS
  if (!FSUtils.isSameHdfs(conf, realSrcFs, desFs)) {
    LOG.info("Bulk-load file " + srcPath + " is on different filesystem than " +
        "the destination store. Copying file over to destination filesystem.");
    Path tmpPath = createTempName();
    FileUtil.copy(realSrcFs, srcPath, fs, tmpPath, false, conf);
    LOG.info("Copied " + srcPath + " to temporary path on destination filesystem: " + tmpPath);
    srcPath = tmpPath;
  }

  return new Pair<>(srcPath, preCommitStoreFile(familyName, srcPath, seqNum, true));
}
 

@Override
public String prepareBulkLoad(final byte[] family, final String srcPath, boolean copyFile)
    throws IOException {
  Path p = new Path(srcPath);
  Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));

  // In case of Replication for bulk load files, hfiles are already copied in staging directory
  if (p.equals(stageP)) {
    LOG.debug(p.getName()
        + " is already available in staging directory. Skipping copy or rename.");
    return stageP.toString();
  }

  if (srcFs == null) {
    srcFs = FileSystem.newInstance(p.toUri(), conf);
  }

  if(!isFile(p)) {
    throw new IOException("Path does not reference a file: " + p);
  }

  // Check to see if the source and target filesystems are the same
  if (!FSUtils.isSameHdfs(conf, srcFs, fs)) {
    LOG.debug("Bulk-load file " + srcPath + " is on different filesystem than " +
        "the destination filesystem. Copying file over to destination staging dir.");
    FileUtil.copy(srcFs, p, fs, stageP, false, conf);
  } else if (copyFile) {
    LOG.debug("Bulk-load file " + srcPath + " is copied to destination staging dir.");
    FileUtil.copy(srcFs, p, fs, stageP, false, conf);
  } else {
    LOG.debug("Moving " + p + " to " + stageP);
    FileStatus origFileStatus = fs.getFileStatus(p);
    origPermissions.put(srcPath, origFileStatus.getPermission());
    if(!fs.rename(p, stageP)) {
      throw new IOException("Failed to move HFile: " + p + " to " + stageP);
    }
  }
  fs.setPermission(stageP, PERM_ALL_ACCESS);
  return stageP.toString();
}
 

@Override
public void failedBulkLoad(final byte[] family, final String srcPath) throws IOException {
  try {
    Path p = new Path(srcPath);
    if (srcFs == null) {
      srcFs = FileSystem.newInstance(p.toUri(), conf);
    }
    if (!FSUtils.isSameHdfs(conf, srcFs, fs)) {
      // files are copied so no need to move them back
      return;
    }
    Path stageP = new Path(stagingDir, new Path(Bytes.toString(family), p.getName()));

    // In case of Replication for bulk load files, hfiles are not renamed by end point during
    // prepare stage, so no need of rename here again
    if (p.equals(stageP)) {
      LOG.debug(p.getName() + " is already available in source directory. Skipping rename.");
      return;
    }

    LOG.debug("Moving " + stageP + " back to " + p);
    if (!fs.rename(stageP, p)) {
      throw new IOException("Failed to move HFile: " + stageP + " to " + p);
    }

    // restore original permission
    if (origPermissions.containsKey(srcPath)) {
      fs.setPermission(p, origPermissions.get(srcPath));
    } else {
      LOG.warn("Can't find previous permission for path=" + srcPath);
    }
  } finally {
    closeSrcFs();
  }
}