Java Code Examples for org.apache.hadoop.hbase.util.FSUtils#getRegionDirs()

public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName,
    final String family) throws IOException {
  // verify htd
  TableDescriptor htd = master.getTableDescriptors().get(tableName);
  assertTrue(htd != null);
  assertFalse(htd.hasColumnFamily(Bytes.toBytes(family)));

  // verify fs
  final FileSystem fs = master.getMasterFileSystem().getFileSystem();
  final Path tableDir =
    CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
  for (Path regionDir : FSUtils.getRegionDirs(fs, tableDir)) {
    final Path familyDir = new Path(regionDir, family);
    assertFalse(family + " family dir should not exist", fs.exists(familyDir));
  }
}
 

private void loadRegionsFromFS(final HashSet<String> mergedParentRegions) throws IOException {
  Path rootDir = master.getMasterFileSystem().getRootDir();
  FileSystem fs = master.getMasterFileSystem().getFileSystem();

  int numRegions = 0;
  List<Path> tableDirs = FSUtils.getTableDirs(fs, rootDir);
  for (Path tableDir : tableDirs) {
    List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
    for (Path regionDir : regionDirs) {
      String encodedRegionName = regionDir.getName();
      if (encodedRegionName == null) {
        LOG.warn("Failed get of encoded name from {}", regionDir);
        continue;
      }
      HbckRegionInfo hri = regionInfoMap.get(encodedRegionName);
      // If it is not in in-memory database and not a merged region,
      // report it as an orphan region.
      if (hri == null && !mergedParentRegions.contains(encodedRegionName)) {
        orphanRegionsOnFS.put(encodedRegionName, regionDir);
        continue;
      }
    }
    numRegions += regionDirs.size();
  }
  LOG.info("Loaded {} tables {} regions from filesyetem and found {} orphan regions",
      tableDirs.size(), numRegions, orphanRegionsOnFS.size());
}
 

/**
 * Create table and validate online schema modification
 * @param tableName Table name
 * @param modifyTable Modify table if true otherwise delete column family
 * @throws IOException in case of failures
 */
private void createTableAndValidateTableSchemaModification(TableName tableName,
    boolean modifyTable) throws Exception {
  Admin admin = TEST_UTIL.getAdmin();
  // Create table with two Cfs
  byte[] cf1 = Bytes.toBytes("cf1");
  byte[] cf2 = Bytes.toBytes("cf2");
  TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1))
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build();
  admin.createTable(tableDesc);

  Table t = TEST_UTIL.getConnection().getTable(tableName);
  // Insert few records and flush the table
  t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1")));
  t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));
  admin.flush(tableName);
  Path tableDir = CommonFSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName);
  List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir);
  assertEquals(1, regionDirs.size());
  List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
  assertEquals(2, familyDirs.size());

  // Insert record but dont flush the table
  t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2")));
  t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2")));

  if (modifyTable) {
    tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build();
    admin.modifyTable(tableDesc);
  } else {
    admin.deleteColumnFamily(tableName, cf2);
  }
  // After table modification or delete family there should be only one CF in FS
  familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0));
  assertEquals("CF dir count should be 1, but was " + familyDirs.size(), 1, familyDirs.size());
}
 

private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException {
  MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem();
  Set<String> families = new HashSet<>();
  Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tableName);
  for (Path regionDir : FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) {
    for (Path familyDir : FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) {
      families.add(familyDir.getName());
    }
  }
  return families;
}
 

private HRegionFileSystem getHRegionFS(Connection conn, Table table, Configuration conf)
    throws IOException {
  FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem();
  Path tableDir = CommonFSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), table.getName());
  List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir);
  assertEquals(1, regionDirs.size());
  List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDirs.get(0));
  assertEquals(2, familyDirs.size());
  RegionInfo hri =
    conn.getRegionLocator(table.getName()).getAllRegionLocations().get(0).getRegion();
  HRegionFileSystem regionFs = new HRegionFileSystem(conf, new HFileSystem(fs), tableDir, hri);
  return regionFs;
}
 

private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major)
    throws IOException {
  TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir);
  for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) {
    compactRegion(tableDir, htd, regionDir, compactOnce, major);
  }
}
 

/**
 * Create the input file for the given directories to compact.
 * The file is a TextFile with each line corrisponding to a
 * store files directory to compact.
 */
public static List<Path> createInputFile(final FileSystem fs, final FileSystem stagingFs,
    final Path path, final Set<Path> toCompactDirs) throws IOException {
  // Extract the list of store dirs
  List<Path> storeDirs = new LinkedList<>();
  for (Path compactDir: toCompactDirs) {
    if (isFamilyDir(fs, compactDir)) {
      storeDirs.add(compactDir);
    } else if (isRegionDir(fs, compactDir)) {
      storeDirs.addAll(FSUtils.getFamilyDirs(fs, compactDir));
    } else if (isTableDir(fs, compactDir)) {
      // Lookup regions
      for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) {
        storeDirs.addAll(FSUtils.getFamilyDirs(fs, regionDir));
      }
    } else {
      throw new IOException(
        "Specified path is not a table, region or family directory. path=" + compactDir);
    }
  }

  // Write Input File
  FSDataOutputStream stream = stagingFs.create(path);
  LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact.");
  try {
    final byte[] newLine = Bytes.toBytes("\n");
    for (Path storeDir: storeDirs) {
      stream.write(Bytes.toBytes(storeDir.toString()));
      stream.write(newLine);
    }
  } finally {
    stream.close();
  }
  return storeDirs;
}
 

@Test
public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception {
  TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks");
  String snapshotName = "foo";

  try {
    createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1);

    Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName);

    testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir);

    Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration());
    for (Path regionDir : FSUtils.getRegionDirs(fs,
      CommonFSUtils.getTableDir(rootDir, tableName))) {
      for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) {
        for (FileStatus status : fs.listStatus(storeDir)) {
          System.out.println(status.getPath());
          if (StoreFileInfo.isValid(status)) {
            Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(),
              tableName, regionDir.getName(), storeDir.getName());

            Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName());
            // assert back references directory is empty
            assertFalse("There is a back reference in " + path, fs.exists(path));

            path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName());
            // assert back references directory is empty
            assertFalse("There is a back reference in " + path, fs.exists(path));
          }
        }
      }
    }
  } finally {
    UTIL.getAdmin().deleteSnapshot(snapshotName);
    UTIL.deleteTable(tableName);
  }
}
 

private List<Path> getTableRegionsDirs(String table) throws IOException {
  String hbaseRoot = this.config.get(HConstants.HBASE_DIR);
  Path tableDir = HBCKFsUtils.getTableDir(new Path(hbaseRoot), TableName.valueOf(table));
  return FSUtils.getRegionDirs(fs, tableDir);
}
 

@Test
public void testRestartMaster() throws Exception {
  final String grantUserName = name.getMethodName();
  User grantUser = User.createUserForTesting(conf, grantUserName, new String[] {});
  String namespace = name.getMethodName();
  TableName table = TableName.valueOf(namespace, name.getMethodName() + ".1");
  TableName table2 = TableName.valueOf(namespace, name.getMethodName() + ".2");
  String snapshot = namespace + "t1";
  admin.createNamespace(NamespaceDescriptor.create(namespace).build());

  // create table2
  TestHDFSAclHelper.createTableAndPut(TEST_UTIL, table2);
  // make some region files in tmp dir and check if master archive these region correctly
  Path tmpTableDir = helper.getPathHelper().getTmpTableDir(table2);
  // make a empty region dir, this is an error region
  FS.mkdirs(new Path(tmpTableDir, "1"));
  // copy regions from data dir, this is a valid region
  for (Path regionDir : FSUtils.getRegionDirs(FS,
    helper.getPathHelper().getDataTableDir(table2))) {
    FSUtils.copyFilesParallel(FS, regionDir, FS,
      new Path(tmpTableDir, regionDir.getName() + "abc"), conf, 1);
  }
  assertEquals(4, FS.listStatus(tmpTableDir).length);

  // grant N(R)
  SecureTestUtil.grantOnNamespace(TEST_UTIL, grantUserName, namespace, READ);
  // restart cluster and tmp directory will not be deleted
  TEST_UTIL.getMiniHBaseCluster().shutdown();
  TEST_UTIL.restartHBaseCluster(1);
  TEST_UTIL.waitUntilNoRegionsInTransition();

  // reset the cached configs after restart
  conf = TEST_UTIL.getConfiguration();
  admin = TEST_UTIL.getAdmin();
  helper = new SnapshotScannerHDFSAclHelper(conf, admin.getConnection());

  Path tmpNsDir = helper.getPathHelper().getTmpNsDir(namespace);
  assertTrue(FS.exists(tmpNsDir));
  // check all regions in tmp table2 dir are archived
  assertEquals(0, FS.listStatus(tmpTableDir).length);

  // create table1 and snapshot
  TestHDFSAclHelper.createTableAndPut(TEST_UTIL, table);
  aclTable = TEST_UTIL.getConnection().getTable(PermissionStorage.ACL_TABLE_NAME);
  snapshotAndWait(snapshot, table);
  TestHDFSAclHelper.canUserScanSnapshot(TEST_UTIL, grantUser, snapshot, 6);
  deleteTable(table);
  deleteTable(table2);
}
 

public static void validateTableCreation(final HMaster master, final TableName tableName,
    final RegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException {
  // check filesystem
  final FileSystem fs = master.getMasterFileSystem().getFileSystem();
  final Path tableDir =
    CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName);
  assertTrue(fs.exists(tableDir));
  CommonFSUtils.logFileSystemState(fs, tableDir, LOG);
  List<Path> unwantedRegionDirs = FSUtils.getRegionDirs(fs, tableDir);
  for (int i = 0; i < regions.length; ++i) {
    Path regionDir = new Path(tableDir, regions[i].getEncodedName());
    assertTrue(regions[i] + " region dir does not exist", fs.exists(regionDir));
    assertTrue(unwantedRegionDirs.remove(regionDir));
    List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir);
    for (int j = 0; j < family.length; ++j) {
      final Path familyDir = new Path(regionDir, family[j]);
      if (hasFamilyDirs) {
        assertTrue(family[j] + " family dir does not exist", fs.exists(familyDir));
        assertTrue(allFamilyDirs.remove(familyDir));
      } else {
        // TODO: WARN: Modify Table/Families does not create a family dir
        if (!fs.exists(familyDir)) {
          LOG.warn(family[j] + " family dir does not exist");
        }
        allFamilyDirs.remove(familyDir);
      }
    }
    assertTrue("found extraneous families: " + allFamilyDirs, allFamilyDirs.isEmpty());
  }
  assertTrue("found extraneous regions: " + unwantedRegionDirs, unwantedRegionDirs.isEmpty());
  LOG.debug("Table directory layout is as expected.");

  // check meta
  assertTrue(MetaTableAccessor.tableExists(master.getConnection(), tableName));
  assertEquals(regions.length, countMetaRegions(master, tableName));

  // check htd
  TableDescriptor htd = master.getTableDescriptors().get(tableName);
  assertTrue("table descriptor not found", htd != null);
  for (int i = 0; i < family.length; ++i) {
    assertTrue("family not found " + family[i], htd.getColumnFamily(Bytes.toBytes(family[i])) != null);
  }
  assertEquals(family.length, htd.getColumnFamilyCount());
}