Java Code Examples for org.apache.hadoop.hbase.client.Admin#getRegions()

/**
 * Get region info from local cluster.
 */
Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
  ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
  Map<ServerName, List<String>> mm = new HashMap<>();
  for (ServerName hsi : regionServers) {
    // list all online regions from this region server
    List<RegionInfo> regions = admin.getRegions(hsi);
    List<String> regionNames = new ArrayList<>(regions.size());
    for (RegionInfo hri : regions) {
      regionNames.add(hri.getRegionNameAsString());
    }
    mm.put(hsi, regionNames);
  }
  return mm;
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Flush random region of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to flush");
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new RegionInfo[0]));
  getLogger().debug("Flushing region " + region.getRegionNameAsString());
  try {
    admin.flushRegion(region.getRegionName());
  } catch (Exception ex) {
    getLogger().warn("Flush failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }

  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Move random region of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to move");
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new RegionInfo[0]));
  getLogger().debug("Move random region {}", region.getRegionNameAsString());
  // Use facility over in MoveRegionsOfTableAction...
  MoveRegionsOfTableAction.moveRegion(admin, MoveRegionsOfTableAction.getServers(admin),
    region, getLogger());
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

public static void deleteFirstIndexRegion(SpliceWatcher spliceWatcher, Connection connection, String schemaName, String tableName, String indexName) throws Exception {
    SConfiguration config = HConfiguration.getConfiguration();
    HBaseTestingUtility testingUtility = new HBaseTestingUtility((Configuration) config.getConfigSource().unwrapDelegate());
    Admin admin = testingUtility.getAdmin();

    // Delete 2nd region of index
    long   conglomerateId = TableSplit.getConglomerateId(connection, schemaName, tableName, indexName);
    TableName iName = TableName.valueOf(config.getNamespace(),Long.toString(conglomerateId));
    List<RegionInfo> partitions = admin.getRegions(iName);
    for (RegionInfo partition : partitions) {
        byte[] startKey = partition.getStartKey();
        if (startKey.length == 0) {
            String encodedRegionName = partition.getEncodedName();
            spliceWatcher.execute(String.format("call syscs_util.delete_region('%s', '%s', '%s', '%s', false)",
                    schemaName, tableName, indexName, encodedRegionName));
            break;
        }
    }
}
 

@Test
public void testMerge()
    throws IOException, InterruptedException, ExecutionException, TimeoutException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  Admin admin = UTIL.getAdmin();
  admin.createTable(
    TableDescriptorBuilder.newBuilder(tableName)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(FAMILY)).build(),
    new byte[][] { Bytes.toBytes(0) });
  List<RegionInfo> regions = admin.getRegions(tableName);
  addStoreFileToKnownFamily(regions.get(0));
  admin.mergeRegionsAsync(regions.get(0).getEncodedNameAsBytes(),
    regions.get(1).getEncodedNameAsBytes(), false).get(30, TimeUnit.SECONDS);
}
 

protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
  throws IOException {
  Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
  Admin admin = TEST_UTIL.getAdmin();
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
  for (ServerName serverName : metrics.getServersName()) {
    for (RegionInfo region : admin.getRegions(serverName)) {
      TableName tableName = region.getTable();
      map.computeIfAbsent(tableName, k -> new TreeMap<>())
        .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
    }
  }
  return map;
}
 

public static void verifyReplicasCameOnline(TableName tableName, Admin admin,
    int regionReplication) throws IOException {
  List<RegionInfo> regions = admin.getRegions(tableName);
  HashSet<RegionInfo> set = new HashSet<>();
  for (RegionInfo hri : regions) {
    set.add(RegionReplicaUtil.getRegionInfoForDefaultReplica(hri));
    for (int i = 0; i < regionReplication; i++) {
      RegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i);
      if (!regions.contains(replica)) {
        Assert.fail(replica + " is not contained in the list of online regions");
      }
    }
  }
  assertEquals(getSplitKeys().length + 1, set.size());
}
 

@Test
public void testNonExistentRegionReplica() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  final byte[] FAMILYNAME = Bytes.toBytes("fam");
  FileSystem fs = HTU.getTestFileSystem();
  Admin admin = HTU.getAdmin();
  Configuration conf = HTU.getConfiguration();
  Path rootDir = HTU.getDataTestDirOnTestFS();

  TableDescriptorBuilder.ModifyableTableDescriptor htd =
    new TableDescriptorBuilder.ModifyableTableDescriptor(tableName);
  htd.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(FAMILYNAME));
  admin.createTable(htd);
  HTU.waitUntilNoRegionsInTransition(60000);

  // Create new HRI with non-default region replica id
  RegionInfo hri = RegionInfoBuilder.newBuilder(htd.getTableName())
    .setStartKey(Bytes.toBytes("A")).setEndKey(Bytes.toBytes("B"))
    .setRegionId(System.currentTimeMillis()).setReplicaId(2).build();
  HRegionFileSystem regionFs = HRegionFileSystem.createRegionOnFileSystem(conf, fs,
    CommonFSUtils.getTableDir(rootDir, hri.getTable()), hri);
  Path regionDir = regionFs.getRegionDir();
  try {
    HRegionFileSystem.loadRegionInfoFileContent(fs, regionDir);
  } catch (IOException e) {
    LOG.info("Caught expected IOE due missing .regioninfo file, due: " + e.getMessage() + " skipping region open.");
    // We should only have 1 region online
    List<RegionInfo> regions = admin.getRegions(tableName);
    LOG.info("Regions: " + regions);
    if (regions.size() != 1) {
      fail("Table " + tableName + " should have only one region, but got more: " + regions);
    }
    return;
  }
  fail("Should have thrown IOE when attempting to open a non-existing region.");
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();
  boolean major = RandomUtils.nextInt(0, 100) < majorRatio;

  getLogger().info("Performing action: Compact random region of table "
    + tableName + ", major=" + major);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to compact");
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
    regions.toArray(new RegionInfo[0]));

  try {
    if (major) {
      getLogger().debug("Major compacting region " + region.getRegionNameAsString());
      admin.majorCompactRegion(region.getRegionName());
    } else {
      getLogger().debug("Compacting region " + region.getRegionNameAsString());
      admin.compactRegion(region.getRegionName());
    }
  } catch (Exception ex) {
    getLogger().warn("Compaction failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Merge random adjacent regions of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.size() < 2) {
    getLogger().info("Table " + tableName + " doesn't have enough regions to merge");
    return;
  }

  int i = RandomUtils.nextInt(0, regions.size() - 1);
  RegionInfo a = regions.get(i++);
  RegionInfo b = regions.get(i);
  getLogger().debug("Merging " + a.getRegionNameAsString() + " and " + b.getRegionNameAsString());

  // Don't try the merge if we're stopping
  if (context.isStopping()) {
    return;
  }

  try {
    admin.mergeRegionsAsync(a.getEncodedNameAsBytes(), b.getEncodedNameAsBytes(), false);
  } catch (Exception ex) {
    getLogger().warn("Merge failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

@Override
public void perform() throws Exception {
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }

  Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
  ServerName[] servers = getServers(admin);

  getLogger().info("Performing action: Move regions of table {}", tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table {} doesn't have regions to move", tableName);
    return;
  }

  Collections.shuffle(regions);

  long start = System.currentTimeMillis();
  for (RegionInfo regionInfo : regions) {
    // Don't try the move if we're stopping
    if (context.isStopping()) {
      return;
    }

    moveRegion(admin, servers, regionInfo, getLogger());
    if (sleepTime > 0) {
      Thread.sleep(sleepTime);
    }

    // put a limit on max num regions. Otherwise, this won't finish
    // with a sleep time of 10sec, 100 regions will finish in 16min
    if (System.currentTimeMillis() - start > maxTime) {
      break;
    }
  }
}
 

@Override
public void perform() throws Exception {
  HBaseTestingUtility util = context.getHBaseIntegrationTestingUtility();
  Admin admin = util.getAdmin();

  getLogger().info("Performing action: Split random region of table " + tableName);
  List<RegionInfo> regions = admin.getRegions(tableName);
  if (regions == null || regions.isEmpty()) {
    getLogger().info("Table " + tableName + " doesn't have regions to split");
    return;
  }
  // Don't try the split if we're stopping
  if (context.isStopping()) {
    return;
  }

  RegionInfo region = PolicyBasedChaosMonkey.selectRandomItem(
      regions.toArray(new RegionInfo[0]));
  getLogger().debug("Splitting region " + region.getRegionNameAsString());
  try {
    admin.splitRegionAsync(region.getRegionName()).get();
  } catch (Exception ex) {
    getLogger().warn("Split failed, might be caused by other chaos: " + ex.getMessage());
  }
  if (sleepTime > 0) {
    Thread.sleep(sleepTime);
  }
}
 

public void mergeRegions(String tblName, int targetRegions) throws Exception {
  TableName table = TableName.valueOf(tblName);
  Path tableDir = getTablePath(table);
  try(Connection conn = ConnectionFactory.createConnection(conf)) {
    Admin admin = conn.getAdmin();
    LongAdder counter = new LongAdder();
    LongAdder lastTimeProgessed = new LongAdder();
    //need to get all regions for the table, regardless of region state
    List<RegionInfo> regions = admin.getRegions(table);
    Map<Future, Pair<RegionInfo, RegionInfo>> regionsMerging = new ConcurrentHashMap<>();
    long roundsNoProgress = 0;
    while (regions.size() > targetRegions) {
      LOG.info("Iteration: {}", counter);
      RegionInfo previous = null;
      int regionSize = regions.size();
      LOG.info("Attempting to merge {} regions to reach the target {} ...", regionSize, targetRegions);
      //to request merge, regions must be OPEN, though
      regions = getOpenRegions(conn, table);
      for (RegionInfo current : regions) {
        if (!current.isSplit()) {
          if (previous != null && canMerge(tableDir, previous, current, regionsMerging.values())) {
            Future f = admin.mergeRegionsAsync(current.getEncodedNameAsBytes(),
                previous.getEncodedNameAsBytes(), true);
            Pair<RegionInfo, RegionInfo> regionPair = new Pair<>(previous, current);
            regionsMerging.put(f,regionPair);
            previous = null;
            if ((regionSize - regionsMerging.size()) <= targetRegions) {
              break;
            }
          } else {
            previous = current;
          }
        }
        else{
          LOG.debug("Skipping split region: {}", current.getEncodedName());
        }
      }
      counter.increment();
      LOG.info("Sleeping for {} seconds before next iteration...", (sleepBetweenCycles/1000));
      Thread.sleep(sleepBetweenCycles);
      regionsMerging.forEach((f, currentPair)-> {
        if (f.isDone()) {
          LOG.info("Merged regions {} and {} together.",
            currentPair.getFirst().getEncodedName(),
            currentPair.getSecond().getEncodedName());
          regionsMerging.remove(f);
          lastTimeProgessed.reset();
          lastTimeProgessed.add(counter.longValue());
        } else {
          LOG.warn("Merge of regions {} and {} isn't completed yet.",
            currentPair.getFirst(),
            currentPair.getSecond());
        }
      });
      roundsNoProgress = counter.longValue() - lastTimeProgessed.longValue();
      if(roundsNoProgress == this.maxRoundsStuck){
        LOG.warn("Reached {} iterations without progressing with new merges. Aborting...",
          roundsNoProgress);
        break;
      }

      //again, get all regions, regardless of the state,
      // in order to avoid breaking the loop prematurely
      regions = admin.getRegions(table);
    }
  }
}
 

@Test
public void testTableCreation() throws Exception {

  conf.set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, StochasticLoadBalancer.class.getName());

  LOG.info("Starting up cluster");
  UTIL.startMiniCluster(SLAVES);
  while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
    Threads.sleep(1);
  }
  Admin admin = UTIL.getAdmin();
  admin.balancerSwitch(false, true);

  String tableName = "testFNImport";
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(TableName.valueOf(tableName));

  tableDescriptor.setColumnFamily(
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(
      HConstants.CATALOG_FAMILY));
  admin.createTable(tableDescriptor, Bytes.toBytes("a"), Bytes.toBytes("z"), REGION_NUM);
  UTIL.waitTableAvailable(tableDescriptor.getTableName());
  admin.balancerSwitch(true, true);

  LOG.info("Shutting down cluster");
  UTIL.shutdownMiniHBaseCluster();

  Thread.sleep(2000);
  LOG.info("Starting cluster again with FN Balancer");
  UTIL.getConfiguration().set(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
      FavoredStochasticBalancer.class.getName());
  UTIL.restartHBaseCluster(SLAVES);
  HMaster master = UTIL.getMiniHBaseCluster().getMaster();
  while (!master.isInitialized()) {
    Threads.sleep(1);
  }
  UTIL.waitTableAvailable(tableDescriptor.getTableName());
  UTIL.waitUntilNoRegionsInTransition(10000);
  assertTrue(master.isBalancerOn());

  FavoredNodesManager fnm = master.getFavoredNodesManager();
  assertNotNull(fnm);

  admin = UTIL.getAdmin();
  List<RegionInfo> regionsOfTable = admin.getRegions(TableName.valueOf(tableName));
  for (RegionInfo rInfo : regionsOfTable) {
    assertNotNull(rInfo);
    assertNotNull(fnm);
    List<ServerName> fns = fnm.getFavoredNodes(rInfo);
    LOG.info("FNS {} {}", rInfo, fns);
    assertNotNull(rInfo.toString(), fns);
    Set<ServerName> favNodes = Sets.newHashSet(fns);
    assertNotNull(favNodes);
    assertEquals("Required no of favored nodes not found.", FAVORED_NODES_NUM, favNodes.size());
    for (ServerName fn : favNodes) {
      assertEquals("StartCode invalid for:" + fn, ServerName.NON_STARTCODE, fn.getStartcode());
    }
  }
}
 

@Override
void perform() throws IOException {
  TableDescriptor selected = selectTable(enabledTables);
  if (selected == null ) {
    return;
  }

  Admin admin = connection.getAdmin();
  TableName tableName = selected.getTableName();
  try (Table table = connection.getTable(tableName)){
    ArrayList<RegionInfo> regionInfos = new ArrayList<>(admin.getRegions(
        selected.getTableName()));
    int numRegions = regionInfos.size();
    // average number of rows to be added per action to each region
    int average_rows = 1;
    int numRows = average_rows * numRegions;
    LOG.info("Adding " + numRows + " rows to table: " + selected);
    for (int i = 0; i < numRows; i++){
      // nextInt(Integer.MAX_VALUE)) to return positive numbers only
      byte[] rowKey = Bytes.toBytes(
          "row-" + String.format("%010d", RandomUtils.nextInt()));
      ColumnFamilyDescriptor cfd = selectFamily(selected);
      if (cfd == null){
        return;
      }
      byte[] family = cfd.getName();
      byte[] qualifier = Bytes.toBytes("col-" + RandomUtils.nextInt() % 10);
      byte[] value = Bytes.toBytes("val-" + RandomStringUtils.randomAlphanumeric(10));
      Put put = new Put(rowKey);
      put.addColumn(family, qualifier, value);
      table.put(put);
    }
    TableDescriptor freshTableDesc = admin.getDescriptor(tableName);
    Assert.assertTrue(
      "After insert, Table: " + tableName + " in not enabled", admin.isTableEnabled(tableName));
    enabledTables.put(tableName, freshTableDesc);
    LOG.info("Added " + numRows + " rows to table: " + selected);
  } catch (Exception e) {
    LOG.warn("Caught exception in action: " + this.getClass());
    throw e;
  } finally {
    admin.close();
  }
}
 

/**
 * Verifies that the given tables each have a single region and are on
 * different region servers. If they are on the same server moves tableName2
 * to the other region server.
 */
private void ensureTablesOnDifferentRegionServers(String tableName1, String tableName2) throws Exception  {
	byte[] table1 = Bytes.toBytes(tableName1);
	byte[] table2 = Bytes.toBytes(tableName2);
	Admin admin = driver.getConnectionQueryServices(getUrl(), TEST_PROPERTIES).getAdmin();
	HBaseTestingUtility util = getUtility();
	MiniHBaseCluster cluster = util.getHBaseCluster();
	HMaster master = cluster.getMaster();
	AssignmentManager am = master.getAssignmentManager();
  
	// verify there is only a single region for data table
	List<RegionInfo> tableRegions = admin.getRegions(TableName.valueOf(table1));
	assertEquals("Expected single region for " + table1, tableRegions.size(), 1);
	RegionInfo hri1 = tableRegions.get(0);
  
	// verify there is only a single region for index table
	tableRegions = admin.getRegions(TableName.valueOf(table2));
	RegionInfo hri2 = tableRegions.get(0);
	assertEquals("Expected single region for " + table2, tableRegions.size(), 1);
  
	ServerName serverName1 = am.getRegionStates().getRegionServerOfRegion(hri1);
	ServerName serverName2 = am.getRegionStates().getRegionServerOfRegion(hri2);
  
	// if data table and index table are on same region server, move the index table to the other region server
	if (serverName1.equals(serverName2)) {
	    HRegionServer server1 = util.getHBaseCluster().getRegionServer(0);
	    HRegionServer server2 = util.getHBaseCluster().getRegionServer(1);
	    HRegionServer dstServer = null;
	    HRegionServer srcServer = null;
	    if (server1.getServerName().equals(serverName2)) {
	        dstServer = server2;
	        srcServer = server1;
	    } else {
	        dstServer = server1;
	        srcServer = server2;
	    }
	    byte[] encodedRegionNameInBytes = hri2.getEncodedNameAsBytes();
	    admin.move(encodedRegionNameInBytes, Bytes.toBytes(dstServer.getServerName().getServerName()));
	    while (dstServer.getOnlineRegion(hri2.getRegionName()) == null
	            || dstServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
	            || srcServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
	            || master.getAssignmentManager().getRegionStates().isRegionInTransition(hri2)) {
	        // wait for the move to be finished
	        Thread.sleep(1);
	    }
	}
  
	hri1 = admin.getRegions(TableName.valueOf(table1)).get(0);
	serverName1 = am.getRegionStates().getRegionServerOfRegion(hri1);
	hri2 = admin.getRegions(TableName.valueOf(table2)).get(0);
	serverName2 = am.getRegionStates().getRegionServerOfRegion(hri2);

	// verify index and data tables are on different servers
	assertNotEquals("Tables " + tableName1 + " and " + tableName2 + " should be on different region servers", serverName1, serverName2);
}