Java Code Examples for org.apache.hadoop.hbase.util.FSUtils#getRegionDirFromRootDir()

/**
 * This method will check 3 places for finding the max sequence id file. One is the expected
 * place, another is the old place under the region directory, and the last one is the wrong one
 * we introduced in HBASE-20734. See HBASE-22617 for more details.
 * <p/>
 * Notice that, you should always call this method instead of
 * {@link #getMaxRegionSequenceId(FileSystem, Path)} until 4.0.0 release.
 * @deprecated Only for compatibility, will be removed in 4.0.0.
 */
@Deprecated
public static long getMaxRegionSequenceId(Configuration conf, RegionInfo region,
  IOExceptionSupplier<FileSystem> rootFsSupplier, IOExceptionSupplier<FileSystem> walFsSupplier)
  throws IOException {
  FileSystem rootFs = rootFsSupplier.get();
  FileSystem walFs = walFsSupplier.get();
  Path regionWALDir =
    CommonFSUtils.getWALRegionDir(conf, region.getTable(), region.getEncodedName());
  // This is the old place where we store max sequence id file
  Path regionDir = FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), region);
  // This is for HBASE-20734, where we use a wrong directory, see HBASE-22617 for more details.
  Path wrongRegionWALDir =
    CommonFSUtils.getWrongWALRegionDir(conf, region.getTable(), region.getEncodedName());
  long maxSeqId = getMaxRegionSequenceId(walFs, regionWALDir);
  maxSeqId = Math.max(maxSeqId, getMaxRegionSequenceId(rootFs, regionDir));
  maxSeqId = Math.max(maxSeqId, getMaxRegionSequenceId(walFs, wrongRegionWALDir));
  return maxSeqId;
}
 

/**
 * Check whether there is recovered.edits in the region dir
 * @param conf conf
 * @param regionInfo the region to check
 * @return true if recovered.edits exist in the region dir
 */
public static boolean hasRecoveredEdits(final Configuration conf, final RegionInfo regionInfo)
    throws IOException {
  // No recovered.edits for non default replica regions
  if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
    return false;
  }
  // Only default replica region can reach here, so we can use regioninfo
  // directly without converting it to default replica's regioninfo.
  Path regionWALDir =
    CommonFSUtils.getWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName());
  Path regionDir = FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(conf), regionInfo);
  Path wrongRegionWALDir =
    CommonFSUtils.getWrongWALRegionDir(conf, regionInfo.getTable(), regionInfo.getEncodedName());
  FileSystem walFs = CommonFSUtils.getWALFileSystem(conf);
  FileSystem rootFs = CommonFSUtils.getRootDirFileSystem(conf);
  NavigableSet<Path> files = getSplitEditFilesSorted(walFs, regionWALDir);
  if (!files.isEmpty()) {
    return true;
  }
  files = getSplitEditFilesSorted(rootFs, regionDir);
  if (!files.isEmpty()) {
    return true;
  }
  files = getSplitEditFilesSorted(walFs, wrongRegionWALDir);
  return !files.isEmpty();
}
 

/**
 * @return True if the Region exits in the filesystem.
 */
public static boolean exists(Configuration conf, FileSystem fs, RegionInfo info)
    throws IOException {
  Path rootDir = CommonFSUtils.getRootDir(conf);
  Path regionDir = FSUtils.getRegionDirFromRootDir(rootDir, info);
  return fs.exists(regionDir);
}
 

private void addStoreFileToKnownFamily(RegionInfo region) throws IOException {
  MasterFileSystem mfs = UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
  Path regionDir =
    FSUtils.getRegionDirFromRootDir(CommonFSUtils.getRootDir(mfs.getConfiguration()), region);
  Path familyDir = new Path(regionDir, Bytes.toString(UNKNOWN_FAMILY));
  StoreFileWriter writer =
      new StoreFileWriter.Builder(mfs.getConfiguration(), mfs.getFileSystem())
          .withOutputDir(familyDir).withFileContext(new HFileContextBuilder().build()).build();
  writer.close();
}
 

/**
 * @return the directory for a give {@code region}.
 */
public Path getRegionDir(RegionInfo region) {
  return FSUtils.getRegionDirFromRootDir(getRootDir(), region);
}
 

@Test
public void testRemoveRegionDirOnArchive() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  UTIL.createTable(tableName, TEST_FAM);

  final Admin admin = UTIL.getAdmin();

  // get the current store files for the region
  List<HRegion> servingRegions = UTIL.getHBaseCluster().getRegions(tableName);
  // make sure we only have 1 region serving this table
  assertEquals(1, servingRegions.size());
  HRegion region = servingRegions.get(0);

  // and load the table
  UTIL.loadRegion(region, TEST_FAM);

  // shutdown the table so we can manipulate the files
  admin.disableTable(tableName);

  FileSystem fs = UTIL.getTestFileSystem();

  // now attempt to depose the region
  Path rootDir = region.getRegionFileSystem().getTableDir().getParent();
  Path regionDir = FSUtils.getRegionDirFromRootDir(rootDir, region.getRegionInfo());

  HFileArchiver.archiveRegion(UTIL.getConfiguration(), fs, region.getRegionInfo());

  // check for the existence of the archive directory and some files in it
  Path archiveDir = HFileArchiveTestingUtil.getRegionArchiveDir(UTIL.getConfiguration(), region);
  assertTrue(fs.exists(archiveDir));

  // check to make sure the store directory was copied
  FileStatus[] stores = fs.listStatus(archiveDir, new PathFilter() {
    @Override
    public boolean accept(Path p) {
      if (p.getName().contains(HConstants.RECOVERED_EDITS_DIR)) {
        return false;
      }
      return true;
    }
  });
  assertTrue(stores.length == 1);

  // make sure we archived the store files
  FileStatus[] storeFiles = fs.listStatus(stores[0].getPath());
  assertTrue(storeFiles.length > 0);

  // then ensure the region's directory isn't present
  assertFalse(fs.exists(regionDir));

  UTIL.deleteTable(tableName);
}
 

private void testReplayWorksWithMemoryCompactionPolicy(MemoryCompactionPolicy policy) throws
  IOException {
  Configuration conf = new Configuration(TEST_UTIL.getConfiguration());
  // Set it so we flush every 1M or so.  Thats a lot.
  conf.setInt(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 1024*1024);
  conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(policy).toLowerCase());
  // The file of recovered edits has a column family of 'meta'.
  final String columnFamily = "meta";
  byte[][] columnFamilyAsByteArray = new byte[][] { Bytes.toBytes(columnFamily) };
  TableDescriptor tableDescriptor = TableDescriptorBuilder
    .newBuilder(TableName.valueOf(testName.getMethodName())).setColumnFamily(
      ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(columnFamily)).build())
    .build();
  RegionInfo hri = RegionInfoBuilder.newBuilder(tableDescriptor.getTableName()).build();
  final String encodedRegionName = hri.getEncodedName();
  Path hbaseRootDir = TEST_UTIL.getDataTestDir();
  FileSystem fs = FileSystem.get(TEST_UTIL.getConfiguration());
  Path tableDir = CommonFSUtils.getTableDir(hbaseRootDir, tableDescriptor.getTableName());
  HRegionFileSystem hrfs =
      new HRegionFileSystem(TEST_UTIL.getConfiguration(), fs, tableDir, hri);
  if (fs.exists(hrfs.getRegionDir())) {
    LOG.info("Region directory already exists. Deleting.");
    fs.delete(hrfs.getRegionDir(), true);
  }
  HRegion region = HBaseTestingUtility
      .createRegionAndWAL(hri, hbaseRootDir, conf, tableDescriptor, blockCache);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  List<String> storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // There should be no store files.
  assertTrue(storeFiles.isEmpty());
  region.close();
  Path regionDir = FSUtils.getRegionDirFromRootDir(hbaseRootDir, hri);
  Path recoveredEditsDir = WALSplitUtil.getRegionDirRecoveredEditsDir(regionDir);
  // This is a little fragile getting this path to a file of 10M of edits.
  Path recoveredEditsFile = new Path(
    System.getProperty("test.build.classes", "target/test-classes"),
      "0000000000000016310");
  // Copy this file under the region's recovered.edits dir so it is replayed on reopen.
  Path destination = new Path(recoveredEditsDir, recoveredEditsFile.getName());
  fs.copyToLocalFile(recoveredEditsFile, destination);
  assertTrue(fs.exists(destination));
  // Now the file 0000000000000016310 is under recovered.edits, reopen the region to replay.
  region = HRegion.openHRegion(region, null);
  assertEquals(encodedRegionName, region.getRegionInfo().getEncodedName());
  storeFiles = region.getStoreFileList(columnFamilyAsByteArray);
  // Our 0000000000000016310 is 10MB. Most of the edits are for one region. Lets assume that if
  // we flush at 1MB, that there are at least 3 flushed files that are there because of the
  // replay of edits.
  if(policy == MemoryCompactionPolicy.EAGER || policy == MemoryCompactionPolicy.ADAPTIVE) {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() >= 1);
  } else {
    assertTrue("Files count=" + storeFiles.size(), storeFiles.size() > 10);
  }
  // Now verify all edits made it into the region.
  int count = verifyAllEditsMadeItIn(fs, conf, recoveredEditsFile, region);
  LOG.info("Checked " + count + " edits made it in");
}