Java Code Examples for org.apache.hadoop.hbase.regionserver.HRegionFileSystem#getStoreFiles()

/**
 * Create reference file(s) to parent region hfiles in the <code>mergeDir</code>
 * @param regionFs merge parent region file system
 * @param mergeDir the temp directory in which we are accumulating references.
 */
private void mergeStoreFiles(final MasterProcedureEnv env, final HRegionFileSystem regionFs,
    final Path mergeDir) throws IOException {
  final TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
  for (ColumnFamilyDescriptor hcd : htd.getColumnFamilies()) {
    String family = hcd.getNameAsString();
    final Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(family);
    if (storeFiles != null && storeFiles.size() > 0) {
      for (StoreFileInfo storeFileInfo : storeFiles) {
        // Create reference file(s) to parent region file here in mergedDir.
        // As this procedure is running on master, use CacheConfig.DISABLED means
        // don't cache any block.
        regionFs.mergeStoreFile(mergedRegion, family, new HStoreFile(
            storeFileInfo, hcd.getBloomFilterType(), CacheConfig.DISABLED), mergeDir);
      }
    }
  }
}
 

boolean shouldCFBeCompacted(HRegionFileSystem fileSystem, String family, long ts)
    throws IOException {

  // do we have any store files?
  Collection<StoreFileInfo> storeFiles = fileSystem.getStoreFiles(family);
  if (storeFiles == null) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName(), " has no store files");
    return false;
  }
  // check for reference files
  if (fileSystem.hasReferences(family) && familyHasReferenceFile(fileSystem, family, ts)) {
    LOG.info("Including store: " + family + " with: " + storeFiles.size()
        + " files for compaction for region: " + fileSystem.getRegionInfo().getEncodedName());
    return true;
  }
  // check store file timestamps
  boolean includeStore = this.shouldIncludeStore(fileSystem, family, storeFiles, ts);
  if (!includeStore) {
    LOG.info("Excluding store: " + family + " for compaction for region:  " + fileSystem
        .getRegionInfo().getEncodedName() + " already compacted");
  }
  return includeStore;
}
 

@VisibleForTesting
protected void addRegion(final Path tableDir, final RegionInfo regionInfo, RegionVisitor visitor)
    throws IOException {
  boolean isMobRegion = MobUtils.isMobRegionInfo(regionInfo);
  try {
    Path baseDir = tableDir;
    // Open the RegionFS
    if (isMobRegion) {
      baseDir = CommonFSUtils.getTableDir(MobUtils.getMobHome(conf), regionInfo.getTable());
    }
    HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, rootFs,
      baseDir, regionInfo, true);
    monitor.rethrowException();

    // 1. dump region meta info into the snapshot directory
    LOG.debug("Storing region-info for snapshot.");
    Object regionData = visitor.regionOpen(regionInfo);
    monitor.rethrowException();

    // 2. iterate through all the stores in the region
    LOG.debug("Creating references for hfiles");

    // This ensures that we have an atomic view of the directory as long as we have < ls limit
    // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files
    // in batches and may miss files being added/deleted. This could be more robust (iteratively
    // checking to see if we have all the files until we are sure), but the limit is currently
    // 1000 files/batch, far more than the number of store files under a single column family.
    Collection<String> familyNames = regionFs.getFamilies();
    if (familyNames != null) {
      for (String familyName: familyNames) {
        Object familyData = visitor.familyOpen(regionData, Bytes.toBytes(familyName));
        monitor.rethrowException();

        Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName);
        if (storeFiles == null) {
          if (LOG.isDebugEnabled()) {
            LOG.debug("No files under family: " + familyName);
          }
          continue;
        }

        // 2.1. build the snapshot reference for the store
        // iterate through all the store's files and create "references".
        addReferenceFiles(visitor, regionData, familyData, storeFiles, false);

        visitor.familyClose(regionData, familyData);
      }
    }
    visitor.regionClose(regionData);
  } catch (IOException e) {
    // the mob directory might not be created yet, so do nothing when it is a mob region
    if (!isMobRegion) {
      throw e;
    }
  }
}
 

static SnapshotRegionManifest buildManifestFromDisk(final Configuration conf,
    final FileSystem fs, final Path tableDir, final RegionInfo regionInfo) throws IOException {
  HRegionFileSystem regionFs = HRegionFileSystem.openRegionFromFileSystem(conf, fs,
        tableDir, regionInfo, true);
  SnapshotRegionManifest.Builder manifest = SnapshotRegionManifest.newBuilder();

  // 1. dump region meta info into the snapshot directory
  LOG.debug("Storing region-info for snapshot.");
  manifest.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));

  // 2. iterate through all the stores in the region
  LOG.debug("Creating references for hfiles");

  // This ensures that we have an atomic view of the directory as long as we have < ls limit
  // (batch size of the files in a directory) on the namenode. Otherwise, we get back the files in
  // batches and may miss files being added/deleted. This could be more robust (iteratively
  // checking to see if we have all the files until we are sure), but the limit is currently 1000
  // files/batch, far more than the number of store files under a single column family.
  Collection<String> familyNames = regionFs.getFamilies();
  if (familyNames != null) {
    for (String familyName: familyNames) {
      Collection<StoreFileInfo> storeFiles = regionFs.getStoreFiles(familyName, false);
      if (storeFiles == null) {
        LOG.debug("No files under family: " + familyName);
        continue;
      }

      // 2.1. build the snapshot reference for the store
      SnapshotRegionManifest.FamilyFiles.Builder family =
            SnapshotRegionManifest.FamilyFiles.newBuilder();
      family.setFamilyName(UnsafeByteOperations.unsafeWrap(Bytes.toBytes(familyName)));

      if (LOG.isDebugEnabled()) {
        LOG.debug("Adding snapshot references for " + storeFiles  + " hfiles");
      }

      // 2.2. iterate through all the store's files and create "references".
      int i = 0;
      int sz = storeFiles.size();
      for (StoreFileInfo storeFile: storeFiles) {
        // create "reference" to this store file.
        LOG.debug("Adding reference for file ("+ (++i) +"/" + sz + "): " + storeFile.getPath());
        SnapshotRegionManifest.StoreFile.Builder sfManifest =
              SnapshotRegionManifest.StoreFile.newBuilder();
        sfManifest.setName(storeFile.getPath().getName());
        family.addStoreFiles(sfManifest.build());
      }
      manifest.addFamilyFiles(family.build());
    }
  }
  return manifest.build();
}
 

@Test
public void testIsStoreFileSizeMatchFilesystemAndManifest() throws IOException {
  admin = UTIL.getAdmin();
  fs = UTIL.getTestFileSystem();
  UTIL.createTable(TABLE_NAME, FAMILY_NAME.getBytes());
  Table table = admin.getConnection().getTable(TABLE_NAME);
  UTIL.loadRandomRows(table, FAMILY_NAME.getBytes(), 3, 1000);
  admin.snapshot(SNAPSHOT_NAME, TABLE_NAME);

  Map<String, Long> storeFileInfoFromManifest = new HashMap<String, Long>();
  Map<String, Long> storeFileInfoFromFS = new HashMap<String, Long>();
  String storeFileName = "";
  long storeFilesize = 0L;
  Path snapshotDir = SnapshotDescriptionUtils
      .getCompletedSnapshotDir(SNAPSHOT_NAME, UTIL.getDefaultRootDirPath());
  SnapshotDescription snapshotDesc = SnapshotDescriptionUtils.readSnapshotInfo(fs, snapshotDir);
  SnapshotManifest snaphotManifest = SnapshotManifest.open(conf, fs, snapshotDir, snapshotDesc);
  List<SnapshotRegionManifest> regionManifest = snaphotManifest.getRegionManifests();
  for (int i = 0; i < regionManifest.size(); i++) {
    SnapshotRegionManifest.FamilyFiles family = regionManifest.get(i).getFamilyFiles(0);
    List<SnapshotRegionManifest.StoreFile> storeFiles = family.getStoreFilesList();
    for (int j = 0; j < storeFiles.size(); j++) {
      storeFileName = storeFiles.get(j).getName();
      storeFilesize = storeFiles.get(j).getFileSize();
      storeFileInfoFromManifest.put(storeFileName, storeFilesize);
    }
  }
  List<RegionInfo> regionsInfo = admin.getRegions(TABLE_NAME);
  Path path = CommonFSUtils.getTableDir(UTIL.getDefaultRootDirPath(), TABLE_NAME);
  for (RegionInfo regionInfo : regionsInfo) {
    HRegionFileSystem hRegionFileSystem =
        HRegionFileSystem.openRegionFromFileSystem(conf, fs, path, regionInfo, true);
    Collection<StoreFileInfo> storeFilesFS = hRegionFileSystem.getStoreFiles(FAMILY_NAME);
    Iterator<StoreFileInfo> sfIterator = storeFilesFS.iterator();
    while (sfIterator.hasNext()) {
      StoreFileInfo sfi = sfIterator.next();
      FileStatus[] fileStatus = CommonFSUtils.listStatus(fs, sfi.getPath());
      storeFileName = fileStatus[0].getPath().getName();
      storeFilesize = fileStatus[0].getLen();
      storeFileInfoFromFS.put(storeFileName, storeFilesize);
    }
  }
  Assert.assertEquals(storeFileInfoFromManifest, storeFileInfoFromFS);
}