Java Code Examples for org.apache.hadoop.hbase.util.FSUtils#getRegionDirs()
The following examples show how to use
org.apache.hadoop.hbase.util.FSUtils#getRegionDirs() .
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MasterProcedureTestingUtility.java From hbase with Apache License 2.0 | 6 votes |
public static void validateColumnFamilyDeletion(final HMaster master, final TableName tableName, final String family) throws IOException { // verify htd TableDescriptor htd = master.getTableDescriptors().get(tableName); assertTrue(htd != null); assertFalse(htd.hasColumnFamily(Bytes.toBytes(family))); // verify fs final FileSystem fs = master.getMasterFileSystem().getFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); for (Path regionDir : FSUtils.getRegionDirs(fs, tableDir)) { final Path familyDir = new Path(regionDir, family); assertFalse(family + " family dir should not exist", fs.exists(familyDir)); } }
Example 2
Source File: HbckChore.java From hbase with Apache License 2.0 | 5 votes |
private void loadRegionsFromFS(final HashSet<String> mergedParentRegions) throws IOException { Path rootDir = master.getMasterFileSystem().getRootDir(); FileSystem fs = master.getMasterFileSystem().getFileSystem(); int numRegions = 0; List<Path> tableDirs = FSUtils.getTableDirs(fs, rootDir); for (Path tableDir : tableDirs) { List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir); for (Path regionDir : regionDirs) { String encodedRegionName = regionDir.getName(); if (encodedRegionName == null) { LOG.warn("Failed get of encoded name from {}", regionDir); continue; } HbckRegionInfo hri = regionInfoMap.get(encodedRegionName); // If it is not in in-memory database and not a merged region, // report it as an orphan region. if (hri == null && !mergedParentRegions.contains(encodedRegionName)) { orphanRegionsOnFS.put(encodedRegionName, regionDir); continue; } } numRegions += regionDirs.size(); } LOG.info("Loaded {} tables {} regions from filesyetem and found {} orphan regions", tableDirs.size(), numRegions, orphanRegionsOnFS.size()); }
Example 3
Source File: TestFromClientSide5.java From hbase with Apache License 2.0 | 5 votes |
/** * Create table and validate online schema modification * @param tableName Table name * @param modifyTable Modify table if true otherwise delete column family * @throws IOException in case of failures */ private void createTableAndValidateTableSchemaModification(TableName tableName, boolean modifyTable) throws Exception { Admin admin = TEST_UTIL.getAdmin(); // Create table with two Cfs byte[] cf1 = Bytes.toBytes("cf1"); byte[] cf2 = Bytes.toBytes("cf2"); TableDescriptor tableDesc = TableDescriptorBuilder.newBuilder(tableName) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf1)) .setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf2)).build(); admin.createTable(tableDesc); Table t = TEST_UTIL.getConnection().getTable(tableName); // Insert few records and flush the table t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val1"))); t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2"))); admin.flush(tableName); Path tableDir = CommonFSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), tableName); List<Path> regionDirs = FSUtils.getRegionDirs(TEST_UTIL.getTestFileSystem(), tableDir); assertEquals(1, regionDirs.size()); List<Path> familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0)); assertEquals(2, familyDirs.size()); // Insert record but dont flush the table t.put(new Put(ROW).addColumn(cf1, QUALIFIER, Bytes.toBytes("val2"))); t.put(new Put(ROW).addColumn(cf2, QUALIFIER, Bytes.toBytes("val2"))); if (modifyTable) { tableDesc = TableDescriptorBuilder.newBuilder(tableDesc).removeColumnFamily(cf2).build(); admin.modifyTable(tableDesc); } else { admin.deleteColumnFamily(tableName, cf2); } // After table modification or delete family there should be only one CF in FS familyDirs = FSUtils.getFamilyDirs(TEST_UTIL.getTestFileSystem(), regionDirs.get(0)); assertEquals("CF dir count should be 1, but was " + familyDirs.size(), 1, familyDirs.size()); }
Example 4
Source File: RestoreSnapshotFromClientSchemaChangeTestBase.java From hbase with Apache License 2.0 | 5 votes |
private Set<String> getFamiliesFromFS(final TableName tableName) throws IOException { MasterFileSystem mfs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Set<String> families = new HashSet<>(); Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), tableName); for (Path regionDir : FSUtils.getRegionDirs(mfs.getFileSystem(), tableDir)) { for (Path familyDir : FSUtils.getFamilyDirs(mfs.getFileSystem(), regionDir)) { families.add(familyDir.getName()); } } return families; }
Example 5
Source File: TestHRegionFileSystem.java From hbase with Apache License 2.0 | 5 votes |
private HRegionFileSystem getHRegionFS(Connection conn, Table table, Configuration conf) throws IOException { FileSystem fs = TEST_UTIL.getDFSCluster().getFileSystem(); Path tableDir = CommonFSUtils.getTableDir(TEST_UTIL.getDefaultRootDirPath(), table.getName()); List<Path> regionDirs = FSUtils.getRegionDirs(fs, tableDir); assertEquals(1, regionDirs.size()); List<Path> familyDirs = FSUtils.getFamilyDirs(fs, regionDirs.get(0)); assertEquals(2, familyDirs.size()); RegionInfo hri = conn.getRegionLocator(table.getName()).getAllRegionLocations().get(0).getRegion(); HRegionFileSystem regionFs = new HRegionFileSystem(conf, new HFileSystem(fs), tableDir, hri); return regionFs; }
Example 6
Source File: CompactionTool.java From hbase with Apache License 2.0 | 5 votes |
private void compactTable(final Path tableDir, final boolean compactOnce, final boolean major) throws IOException { TableDescriptor htd = FSTableDescriptors.getTableDescriptorFromFs(fs, tableDir); for (Path regionDir: FSUtils.getRegionDirs(fs, tableDir)) { compactRegion(tableDir, htd, regionDir, compactOnce, major); } }
Example 7
Source File: CompactionTool.java From hbase with Apache License 2.0 | 5 votes |
/** * Create the input file for the given directories to compact. * The file is a TextFile with each line corrisponding to a * store files directory to compact. */ public static List<Path> createInputFile(final FileSystem fs, final FileSystem stagingFs, final Path path, final Set<Path> toCompactDirs) throws IOException { // Extract the list of store dirs List<Path> storeDirs = new LinkedList<>(); for (Path compactDir: toCompactDirs) { if (isFamilyDir(fs, compactDir)) { storeDirs.add(compactDir); } else if (isRegionDir(fs, compactDir)) { storeDirs.addAll(FSUtils.getFamilyDirs(fs, compactDir)); } else if (isTableDir(fs, compactDir)) { // Lookup regions for (Path regionDir: FSUtils.getRegionDirs(fs, compactDir)) { storeDirs.addAll(FSUtils.getFamilyDirs(fs, regionDir)); } } else { throw new IOException( "Specified path is not a table, region or family directory. path=" + compactDir); } } // Write Input File FSDataOutputStream stream = stagingFs.create(path); LOG.info("Create input file=" + path + " with " + storeDirs.size() + " dirs to compact."); try { final byte[] newLine = Bytes.toBytes("\n"); for (Path storeDir: storeDirs) { stream.write(Bytes.toBytes(storeDir.toString())); stream.write(newLine); } } finally { stream.close(); } return storeDirs; }
Example 8
Source File: TableSnapshotInputFormatTestBase.java From hbase with Apache License 2.0 | 5 votes |
@Test public void testRestoreSnapshotDoesNotCreateBackRefLinks() throws Exception { TableName tableName = TableName.valueOf("testRestoreSnapshotDoesNotCreateBackRefLinks"); String snapshotName = "foo"; try { createTableAndSnapshot(UTIL, tableName, snapshotName, getStartRow(), getEndRow(), 1); Path tmpTableDir = UTIL.getDataTestDirOnTestFS(snapshotName); testRestoreSnapshotDoesNotCreateBackRefLinksInit(tableName, snapshotName,tmpTableDir); Path rootDir = CommonFSUtils.getRootDir(UTIL.getConfiguration()); for (Path regionDir : FSUtils.getRegionDirs(fs, CommonFSUtils.getTableDir(rootDir, tableName))) { for (Path storeDir : FSUtils.getFamilyDirs(fs, regionDir)) { for (FileStatus status : fs.listStatus(storeDir)) { System.out.println(status.getPath()); if (StoreFileInfo.isValid(status)) { Path archiveStoreDir = HFileArchiveUtil.getStoreArchivePath(UTIL.getConfiguration(), tableName, regionDir.getName(), storeDir.getName()); Path path = HFileLink.getBackReferencesDir(storeDir, status.getPath().getName()); // assert back references directory is empty assertFalse("There is a back reference in " + path, fs.exists(path)); path = HFileLink.getBackReferencesDir(archiveStoreDir, status.getPath().getName()); // assert back references directory is empty assertFalse("There is a back reference in " + path, fs.exists(path)); } } } } } finally { UTIL.getAdmin().deleteSnapshot(snapshotName); UTIL.deleteTable(tableName); } }
Example 9
Source File: FsRegionsMetaRecoverer.java From hbase-operator-tools with Apache License 2.0 | 4 votes |
private List<Path> getTableRegionsDirs(String table) throws IOException { String hbaseRoot = this.config.get(HConstants.HBASE_DIR); Path tableDir = HBCKFsUtils.getTableDir(new Path(hbaseRoot), TableName.valueOf(table)); return FSUtils.getRegionDirs(fs, tableDir); }
Example 10
Source File: TestSnapshotScannerHDFSAclController.java From hbase with Apache License 2.0 | 4 votes |
@Test public void testRestartMaster() throws Exception { final String grantUserName = name.getMethodName(); User grantUser = User.createUserForTesting(conf, grantUserName, new String[] {}); String namespace = name.getMethodName(); TableName table = TableName.valueOf(namespace, name.getMethodName() + ".1"); TableName table2 = TableName.valueOf(namespace, name.getMethodName() + ".2"); String snapshot = namespace + "t1"; admin.createNamespace(NamespaceDescriptor.create(namespace).build()); // create table2 TestHDFSAclHelper.createTableAndPut(TEST_UTIL, table2); // make some region files in tmp dir and check if master archive these region correctly Path tmpTableDir = helper.getPathHelper().getTmpTableDir(table2); // make a empty region dir, this is an error region FS.mkdirs(new Path(tmpTableDir, "1")); // copy regions from data dir, this is a valid region for (Path regionDir : FSUtils.getRegionDirs(FS, helper.getPathHelper().getDataTableDir(table2))) { FSUtils.copyFilesParallel(FS, regionDir, FS, new Path(tmpTableDir, regionDir.getName() + "abc"), conf, 1); } assertEquals(4, FS.listStatus(tmpTableDir).length); // grant N(R) SecureTestUtil.grantOnNamespace(TEST_UTIL, grantUserName, namespace, READ); // restart cluster and tmp directory will not be deleted TEST_UTIL.getMiniHBaseCluster().shutdown(); TEST_UTIL.restartHBaseCluster(1); TEST_UTIL.waitUntilNoRegionsInTransition(); // reset the cached configs after restart conf = TEST_UTIL.getConfiguration(); admin = TEST_UTIL.getAdmin(); helper = new SnapshotScannerHDFSAclHelper(conf, admin.getConnection()); Path tmpNsDir = helper.getPathHelper().getTmpNsDir(namespace); assertTrue(FS.exists(tmpNsDir)); // check all regions in tmp table2 dir are archived assertEquals(0, FS.listStatus(tmpTableDir).length); // create table1 and snapshot TestHDFSAclHelper.createTableAndPut(TEST_UTIL, table); aclTable = TEST_UTIL.getConnection().getTable(PermissionStorage.ACL_TABLE_NAME); snapshotAndWait(snapshot, table); TestHDFSAclHelper.canUserScanSnapshot(TEST_UTIL, grantUser, snapshot, 6); deleteTable(table); deleteTable(table2); }
Example 11
Source File: MasterProcedureTestingUtility.java From hbase with Apache License 2.0 | 4 votes |
public static void validateTableCreation(final HMaster master, final TableName tableName, final RegionInfo[] regions, boolean hasFamilyDirs, String... family) throws IOException { // check filesystem final FileSystem fs = master.getMasterFileSystem().getFileSystem(); final Path tableDir = CommonFSUtils.getTableDir(master.getMasterFileSystem().getRootDir(), tableName); assertTrue(fs.exists(tableDir)); CommonFSUtils.logFileSystemState(fs, tableDir, LOG); List<Path> unwantedRegionDirs = FSUtils.getRegionDirs(fs, tableDir); for (int i = 0; i < regions.length; ++i) { Path regionDir = new Path(tableDir, regions[i].getEncodedName()); assertTrue(regions[i] + " region dir does not exist", fs.exists(regionDir)); assertTrue(unwantedRegionDirs.remove(regionDir)); List<Path> allFamilyDirs = FSUtils.getFamilyDirs(fs, regionDir); for (int j = 0; j < family.length; ++j) { final Path familyDir = new Path(regionDir, family[j]); if (hasFamilyDirs) { assertTrue(family[j] + " family dir does not exist", fs.exists(familyDir)); assertTrue(allFamilyDirs.remove(familyDir)); } else { // TODO: WARN: Modify Table/Families does not create a family dir if (!fs.exists(familyDir)) { LOG.warn(family[j] + " family dir does not exist"); } allFamilyDirs.remove(familyDir); } } assertTrue("found extraneous families: " + allFamilyDirs, allFamilyDirs.isEmpty()); } assertTrue("found extraneous regions: " + unwantedRegionDirs, unwantedRegionDirs.isEmpty()); LOG.debug("Table directory layout is as expected."); // check meta assertTrue(MetaTableAccessor.tableExists(master.getConnection(), tableName)); assertEquals(regions.length, countMetaRegions(master, tableName)); // check htd TableDescriptor htd = master.getTableDescriptors().get(tableName); assertTrue("table descriptor not found", htd != null); for (int i = 0; i < family.length; ++i) { assertTrue("family not found " + family[i], htd.getColumnFamily(Bytes.toBytes(family[i])) != null); } assertEquals(family.length, htd.getColumnFamilyCount()); }