Java Code Examples for org.apache.hadoop.hbase.client.RegionInfo#getTable()

@Override
public void preCommitStoreFile(final ObserverContext<RegionCoprocessorEnvironment> ctx,
    final byte[] family, final List<Pair<Path, Path>> pairs) throws IOException {
  Configuration cfg = ctx.getEnvironment().getConfiguration();
  if (pairs == null || pairs.isEmpty() || !BackupManager.isBackupEnabled(cfg)) {
    LOG.debug("skipping recording bulk load in preCommitStoreFile since backup is disabled");
    return;
  }
  try (Connection connection = ConnectionFactory.createConnection(cfg);
      BackupSystemTable tbl = new BackupSystemTable(connection)) {
    List<TableName> fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL);
    RegionInfo info = ctx.getEnvironment().getRegionInfo();
    TableName tableName = info.getTable();
    if (!fullyBackedUpTables.contains(tableName)) {
      if (LOG.isTraceEnabled()) {
        LOG.trace(tableName + " has not gone thru full backup");
      }
      return;
    }
    tbl.writeFilesForBulkLoadPreCommit(tableName, info.getEncodedNameAsBytes(), family, pairs);
    return;
  }
}
 

/**
 * All regions have an assignment.
 */
protected void assertImmediateAssignment(List<RegionInfo> regions, List<ServerName> servers,
    Map<RegionInfo, ServerName> assignments) throws IOException {
  for (RegionInfo region : regions) {
    assertTrue(assignments.containsKey(region));
    ServerName server = assignments.get(region);
    TableName tableName = region.getTable();

    String groupName =
        tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
    assertTrue(StringUtils.isNotEmpty(groupName));
    RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
    assertTrue("Region is not correctly assigned to group servers.",
      gInfo.containsServer(server.getAddress()));
  }
}
 

/**
 * Tests the bulk assignment used during cluster startup.
 * <p/>
 * Round-robin. Should yield a balanced cluster so same invariant as the load balancer holds, all
 * servers holding either floor(avg) or ceiling(avg).
 */
@Test
public void testBulkAssignment() throws Exception {
  List<RegionInfo> regions = randomRegions(25);
  Map<ServerName, List<RegionInfo>> assignments =
      loadBalancer.roundRobinAssignment(regions, servers);
  // test empty region/servers scenario
  // this should not throw an NPE
  loadBalancer.roundRobinAssignment(regions, Collections.emptyList());
  // test regular scenario
  assertTrue(assignments.keySet().size() == servers.size());
  for (ServerName sn : assignments.keySet()) {
    List<RegionInfo> regionAssigned = assignments.get(sn);
    for (RegionInfo region : regionAssigned) {
      TableName tableName = region.getTable();
      String groupName =
          tableDescs.get(tableName).getRegionServerGroup().orElse(RSGroupInfo.DEFAULT_GROUP);
      assertTrue(StringUtils.isNotEmpty(groupName));
      RSGroupInfo gInfo = getMockedGroupInfoManager().getRSGroup(groupName);
      assertTrue("Region is not correctly assigned to group servers.",
        gInfo.containsServer(sn.getAddress()));
    }
  }
  ArrayListMultimap<String, ServerAndLoad> loadMap = convertToGroupBasedMap(assignments);
  assertClusterAsBalanced(loadMap);
}
 

/**
 * A 'full' WAL transaction involves starting an mvcc transaction followed by an append, an
 * optional sync, and then a call to complete the mvcc transaction. This method does it all. Good
 * for case of adding a single edit or marker to the WAL.
 * <p/>
 * This write is for internal use only. Not for external client consumption.
 * @return WALKeyImpl that was added to the WAL.
 */
private static WALKeyImpl doFullMarkerAppendTransaction(final WAL wal,
  final NavigableMap<byte[], Integer> replicationScope, final RegionInfo hri, final WALEdit edit,
  final MultiVersionConcurrencyControl mvcc,
  final Map<String, byte[]> extendedAttributes, final boolean sync) throws IOException {
  // TODO: Pass in current time to use?
  WALKeyImpl walKey = new WALKeyImpl(hri.getEncodedNameAsBytes(), hri.getTable(),
    System.currentTimeMillis(), mvcc, replicationScope, extendedAttributes);
  long trx = MultiVersionConcurrencyControl.NONE;
  try {
    trx = wal.appendMarker(hri, walKey, edit);
    if (sync) {
      wal.sync(trx);
    }
    // Call complete only here because these are markers only. They are not for clients to read.
    mvcc.complete(walKey.getWriteEntry());
  } catch (IOException ioe) {
    if (walKey.getWriteEntry() != null) {
      mvcc.complete(walKey.getWriteEntry());
    }
    throw ioe;
  }
  return walKey;
}
 

private Map<TableName, Map<ServerName, List<RegionInfo>>> getRSGroupAssignmentsByTable(
    String groupName) throws IOException {
  Map<TableName, Map<ServerName, List<RegionInfo>>> result = Maps.newHashMap();
  Set<TableName> tablesInGroupCache = new HashSet<>();
  for (Map.Entry<RegionInfo, ServerName> entry :
      masterServices.getAssignmentManager().getRegionStates()
      .getRegionAssignments().entrySet()) {
    RegionInfo region = entry.getKey();
    TableName tn = region.getTable();
    ServerName server = entry.getValue();
    if (isTableInGroup(tn, groupName, tablesInGroupCache)) {
      result.computeIfAbsent(tn, k -> new HashMap<>())
          .computeIfAbsent(server, k -> new ArrayList<>()).add(region);
    }
  }
  RSGroupInfo rsGroupInfo = getRSGroupInfo(groupName);
  for (ServerName serverName : masterServices.getServerManager().getOnlineServers().keySet()) {
    if (rsGroupInfo.containsServer(serverName.getAddress())) {
      for (Map<ServerName, List<RegionInfo>> map : result.values()) {
        map.computeIfAbsent(serverName, k -> Collections.emptyList());
      }
    }
  }

  return result;
}
 

private void prepareTableToReopenRegionsMap(
    final Map<TableName, List<byte[]>> tableToReopenRegionsMap,
    final byte[] regionName, final int regionStoreRefCount) {

  final RegionInfo regionInfo = hMaster.getAssignmentManager().getRegionInfo(regionName);
  final TableName tableName = regionInfo.getTable();
  if (TableName.isMetaTableName(tableName)) {
    // Do not reopen regions of meta table even if it has
    // high store file reference count
    return;
  }
  LOG.warn("Region {} for Table {} has high storeFileRefCount {}, considering it for reopen..",
    regionInfo.getRegionNameAsString(), tableName, regionStoreRefCount);
  tableToReopenRegionsMap
      .computeIfAbsent(tableName, (key) -> new ArrayList<>()).add(regionName);
}
 

/**
 * Compact a region on the master.
 *
 * @param controller the RPC controller
 * @param request the request
 * @throws ServiceException
 */
@Override
@QosPriority(priority=HConstants.ADMIN_QOS)
public CompactRegionResponse compactRegion(final RpcController controller,
  final CompactRegionRequest request) throws ServiceException {
  try {
    master.checkInitialized();
    byte[] regionName = request.getRegion().getValue().toByteArray();
    TableName tableName = RegionInfo.getTable(regionName);
    // TODO: support CompactType.MOB
    // if the region is a mob region, do the mob file compaction.
    if (MobUtils.isMobRegionName(tableName, regionName)) {
      checkHFileFormatVersionForMob();
      //TODO: support CompactType.MOB
      // HBASE-23571
      LOG.warn("CompactType.MOB is not supported yet, will run regular compaction."+
          " Refer to HBASE-23571.");
      return super.compactRegion(controller, request);
    } else {
      return super.compactRegion(controller, request);
    }
  } catch (IOException ie) {
    throw new ServiceException(ie);
  }
}
 

public SplitTableRegionProcedure(final MasterProcedureEnv env,
    final RegionInfo regionToSplit, final byte[] splitRow) throws IOException {
  super(env, regionToSplit);
  preflightChecks(env, true);
  // When procedure goes to run in its prepare step, it also does these checkOnline checks. Here
  // we fail-fast on construction. There it skips the split with just a warning.
  checkOnline(env, regionToSplit);
  this.bestSplitRow = splitRow;
  checkSplittable(env, regionToSplit, bestSplitRow);
  final TableName table = regionToSplit.getTable();
  final long rid = getDaughterRegionIdTimestamp(regionToSplit);
  this.daughterOneRI = RegionInfoBuilder.newBuilder(table)
      .setStartKey(regionToSplit.getStartKey())
      .setEndKey(bestSplitRow)
      .setSplit(false)
      .setRegionId(rid)
      .build();
  this.daughterTwoRI = RegionInfoBuilder.newBuilder(table)
      .setStartKey(bestSplitRow)
      .setEndKey(regionToSplit.getEndKey())
      .setSplit(false)
      .setRegionId(rid)
      .build();
  TableDescriptor htd = env.getMasterServices().getTableDescriptors().get(getTableName());
  if(htd.getRegionSplitPolicyClassName() != null) {
    // Since we don't have region reference here, creating the split policy instance without it.
    // This can be used to invoke methods which don't require Region reference. This instantiation
    // of a class on Master-side though it only makes sense on the RegionServer-side is
    // for Phoenix Local Indexing. Refer HBASE-12583 for more information.
    Class<? extends RegionSplitPolicy> clazz =
        RegionSplitPolicy.getSplitPolicyClass(htd, env.getMasterConfiguration());
    this.splitPolicy = ReflectionUtils.newInstance(clazz, env.getMasterConfiguration());
  }
}
 

protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
  throws IOException {
  Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
  Admin admin = TEST_UTIL.getAdmin();
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
  for (ServerName serverName : metrics.getServersName()) {
    for (RegionInfo region : admin.getRegions(serverName)) {
      TableName tableName = region.getTable();
      map.computeIfAbsent(tableName, k -> new TreeMap<>())
        .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
    }
  }
  return map;
}
 

/**
 * This method implements Admin getRegionInfo. On RegionServer, it is
 * able to return RegionInfo and detail. On Master, it just returns
 * RegionInfo. On Master it has been hijacked to return Mob detail.
 * Master implementation is good for querying full region name if
 * you only have the encoded name (useful around region replicas
 * for example which do not have a row in hbase:meta).
 */
@Override
@QosPriority(priority=HConstants.ADMIN_QOS)
public GetRegionInfoResponse getRegionInfo(final RpcController controller,
  final GetRegionInfoRequest request) throws ServiceException {
  RegionInfo ri = null;
  try {
    ri = getRegionInfo(request.getRegion());
  } catch(UnknownRegionException ure) {
    throw new ServiceException(ure);
  }
  GetRegionInfoResponse.Builder builder = GetRegionInfoResponse.newBuilder();
  if (ri != null) {
    builder.setRegionInfo(ProtobufUtil.toRegionInfo(ri));
  } else {
    // Is it a MOB name? These work differently.
    byte [] regionName = request.getRegion().getValue().toByteArray();
    TableName tableName = RegionInfo.getTable(regionName);
    if (MobUtils.isMobRegionName(tableName, regionName)) {
      // a dummy region info contains the compaction state.
      RegionInfo mobRegionInfo = MobUtils.getMobRegionInfo(tableName);
      builder.setRegionInfo(ProtobufUtil.toRegionInfo(mobRegionInfo));
      if (request.hasCompactionState() && request.getCompactionState()) {
        builder.setCompactionState(master.getMobCompactionState(tableName));
      }
    } else {
      // If unknown RegionInfo and not a MOB region, it is unknown.
      throw new ServiceException(new UnknownRegionException(Bytes.toString(regionName)));
    }
  }
  return builder.build();
}
 

@Override
public void postBulkLoadHFile(ObserverContext<RegionCoprocessorEnvironment> ctx,
  List<Pair<byte[], String>> stagingFamilyPaths, Map<byte[], List<Path>> finalPaths)
      throws IOException {
  Configuration cfg = ctx.getEnvironment().getConfiguration();
  if (finalPaths == null) {
    // there is no need to record state
    return;
  }
  if (!BackupManager.isBackupEnabled(cfg)) {
    LOG.debug("skipping recording bulk load in postBulkLoadHFile since backup is disabled");
    return;
  }
  try (Connection connection = ConnectionFactory.createConnection(cfg);
      BackupSystemTable tbl = new BackupSystemTable(connection)) {
    List<TableName> fullyBackedUpTables = tbl.getTablesForBackupType(BackupType.FULL);
    RegionInfo info = ctx.getEnvironment().getRegionInfo();
    TableName tableName = info.getTable();
    if (!fullyBackedUpTables.contains(tableName)) {
      if (LOG.isTraceEnabled()) {
        LOG.trace(tableName + " has not gone thru full backup");
      }
      return;
    }
    tbl.writePathsPostBulkLoad(tableName, info.getEncodedNameAsBytes(), finalPaths);
  } catch (IOException ioe) {
    LOG.error("Failed to get tables which have been fully backed up", ioe);
  }
}
 

public void publishWriteTiming(ServerName serverName, RegionInfo region,
    ColumnFamilyDescriptor column, long msTime) {
  RegionTaskResult rtr = new RegionTaskResult(region, region.getTable(), serverName, column);
  rtr.setWriteSuccess();
  rtr.setWriteLatency(msTime);
  List<RegionTaskResult> rtrs = regionMap.get(region.getRegionNameAsString());
  rtrs.add(rtr);
  // Note that write success count will be equal to total column family write successes.
  incWriteSuccessCount();
  LOG.info("Write to {} on {} {} in {}ms",
    region.getRegionNameAsString(), serverName, column.getNameAsString(), msTime);
}
 

public void publishReadTiming(ServerName serverName, RegionInfo region,
    ColumnFamilyDescriptor column, long msTime) {
  RegionTaskResult rtr = new RegionTaskResult(region, region.getTable(), serverName, column);
  rtr.setReadSuccess();
  rtr.setReadLatency(msTime);
  List<RegionTaskResult> rtrs = regionMap.get(region.getRegionNameAsString());
  rtrs.add(rtr);
  // Note that read success count will be equal to total column family read successes.
  incReadSuccessCount();
  LOG.info("Read from {} on {} {} in {}ms", region.getRegionNameAsString(), serverName,
      column.getNameAsString(), msTime);
}
 

private TableName getTableName(Region region) {
  RegionInfo regionInfo = region.getRegionInfo();
  if (regionInfo != null) {
    return regionInfo.getTable();
  }
  return null;
}
 

/**
 * Returns <code>true</code> if the current user is allowed the given action
 * over at least one of the column qualifiers in the given column families.
 */
private boolean hasFamilyQualifierPermission(User user,
    Action perm,
    RegionCoprocessorEnvironment env,
    Map<byte[], ? extends Collection<byte[]>> familyMap)
  throws IOException {
  RegionInfo hri = env.getRegion().getRegionInfo();
  TableName tableName = hri.getTable();

  if (user == null) {
    return false;
  }

  if (familyMap != null && familyMap.size() > 0) {
    // at least one family must be allowed
    for (Map.Entry<byte[], ? extends Collection<byte[]>> family :
        familyMap.entrySet()) {
      if (family.getValue() != null && !family.getValue().isEmpty()) {
        for (byte[] qualifier : family.getValue()) {
          if (getAuthManager().authorizeUserTable(user, tableName,
                family.getKey(), qualifier, perm)) {
            return true;
          }
        }
      } else {
        if (getAuthManager().authorizeUserFamily(user, tableName, family.getKey(), perm)) {
          return true;
        }
      }
    }
  } else if (LOG.isDebugEnabled()) {
    LOG.debug("Empty family map passed for permission check");
  }

  return false;
}
 

private void addRegion(RegionInfo regionInfo) {
  // Process the region name to region info map
  regionNameToRegionInfoMap.put(regionInfo.getRegionNameAsString(), regionInfo);

  // Process the table to region map
  TableName tableName = regionInfo.getTable();
  List<RegionInfo> regionList = tableToRegionMap.get(tableName);
  if (regionList == null) {
    regionList = new ArrayList<>();
  }
  // Add the current region info into the tableToRegionMap
  regionList.add(regionInfo);
  tableToRegionMap.put(tableName, regionList);
}
 

@Override
public TableName getTableName() {
  RegionInfo hri = getRegionInfo();
  return hri != null ? hri.getTable() : null;
}
 

/**
 * Initialize the region assignment snapshot by scanning the hbase:meta table
 * @throws IOException
 */
public void initialize() throws IOException {
  LOG.info("Start to scan the hbase:meta for the current region assignment " +
    "snappshot");
  // TODO: at some point this code could live in the MetaTableAccessor
  ClientMetaTableAccessor.Visitor v = new ClientMetaTableAccessor.Visitor() {
    @Override
    public boolean visit(Result result) throws IOException {
      try {
        if (result ==  null || result.isEmpty()) return true;
        RegionLocations rl = CatalogFamilyFormat.getRegionLocations(result);
        if (rl == null) return true;
        RegionInfo hri = rl.getRegionLocation(0).getRegion();
        if (hri == null) return true;
        if (hri.getTable() == null) return true;
        if (disabledTables.contains(hri.getTable())) {
          return true;
        }
        // Are we to include split parents in the list?
        if (excludeOfflinedSplitParents && hri.isSplit()) return true;
        HRegionLocation[] hrls = rl.getRegionLocations();

        // Add the current assignment to the snapshot for all replicas
        for (int i = 0; i < hrls.length; i++) {
          if (hrls[i] == null) continue;
          hri = hrls[i].getRegion();
          if (hri == null) continue;
          addAssignment(hri, hrls[i].getServerName());
          addRegion(hri);
        }

        hri = rl.getRegionLocation(0).getRegion();
        // the code below is to handle favored nodes
        byte[] favoredNodes = result.getValue(HConstants.CATALOG_FAMILY,
            FavoredNodeAssignmentHelper.FAVOREDNODES_QUALIFIER);
        if (favoredNodes == null) return true;
        // Add the favored nodes into assignment plan
        ServerName[] favoredServerList =
            FavoredNodeAssignmentHelper.getFavoredNodesList(favoredNodes);
        // Add the favored nodes into assignment plan
        existingAssignmentPlan.updateFavoredNodesMap(hri,
            Arrays.asList(favoredServerList));

        /*
         * Typically there should be FAVORED_NODES_NUM favored nodes for a region in meta. If
         * there is less than FAVORED_NODES_NUM, lets use as much as we can but log a warning.
         */
        if (favoredServerList.length != FavoredNodeAssignmentHelper.FAVORED_NODES_NUM) {
          LOG.warn("Insufficient favored nodes for region " + hri + " fn: " + Arrays
              .toString(favoredServerList));
        }
        for (int i = 0; i < favoredServerList.length; i++) {
          if (i == PRIMARY.ordinal()) addPrimaryAssignment(hri, favoredServerList[i]);
          if (i == SECONDARY.ordinal()) addSecondaryAssignment(hri, favoredServerList[i]);
          if (i == TERTIARY.ordinal()) addTeritiaryAssignment(hri, favoredServerList[i]);
        }
        return true;
      } catch (RuntimeException e) {
        LOG.error("Catche remote exception " + e.getMessage() +
            " when processing" + result);
        throw e;
      }
    }
  };
  // Scan hbase:meta to pick up user regions
  MetaTableAccessor.fullScanRegions(connection, v);
  //regionToRegionServerMap = regions;
  LOG.info("Finished to scan the hbase:meta for the current region assignment" +
    "snapshot");
}
 

/**
 * Find a RS that has regions of a table.
 * @param hasMetaRegion when true, the returned RS has hbase:meta region as well
 */
private HRegionServer findRSToKill(boolean hasMetaRegion) throws Exception {
  List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
  List<RegionInfo> regions = null;
  HRegionServer hrs = null;

  for (RegionServerThread rst : rsts) {
    hrs = rst.getRegionServer();
    while (rst.isAlive() && !hrs.isOnline()) {
      Thread.sleep(100);
    }
    if (!rst.isAlive()) {
      continue;
    }
    boolean isCarryingMeta = false;
    boolean foundTableRegion = false;
    regions = ProtobufUtil.getOnlineRegions(hrs.getRSRpcServices());
    for (RegionInfo region : regions) {
      if (region.isMetaRegion()) {
        isCarryingMeta = true;
      }
      if (region.getTable() == tableName) {
        foundTableRegion = true;
      }
      if (foundTableRegion && (isCarryingMeta || !hasMetaRegion)) {
        break;
      }
    }
    if (isCarryingMeta && hasMetaRegion) {
      // clients ask for a RS with META
      if (!foundTableRegion) {
        HRegionServer destRS = hrs;
        // the RS doesn't have regions of the specified table so we need move one to this RS
        List<RegionInfo> tableRegions = TEST_UTIL.getAdmin().getRegions(tableName);
        RegionInfo hri = tableRegions.get(0);
        TEST_UTIL.getAdmin().move(hri.getEncodedNameAsBytes(), destRS.getServerName());
        // wait for region move completes
        RegionStates regionStates =
            TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates();
        TEST_UTIL.waitFor(45000, 200, new Waiter.Predicate<Exception>() {
          @Override
          public boolean evaluate() throws Exception {
            ServerName sn = regionStates.getRegionServerOfRegion(hri);
            return (sn != null && sn.equals(destRS.getServerName()));
          }
        });
      }
      return hrs;
    } else if (hasMetaRegion || isCarryingMeta) {
      continue;
    }
    if (foundTableRegion) {
      break;
    }
  }

  return hrs;
}
 

/**
 * Create a new HFileLink
 *
 * <p>It also adds a back-reference to the hfile back-reference directory
 * to simplify the reference-count and the cleaning process.
 *
 * @param conf {@link Configuration} to read for the archive directory name
 * @param fs {@link FileSystem} on which to write the HFileLink
 * @param dstFamilyPath - Destination path (table/region/cf/)
 * @param hfileRegionInfo - Linked HFile Region Info
 * @param hfileName - Linked HFile name
 * @param createBackRef - Whether back reference should be created. Defaults to true.
 * @return true if the file is created, otherwise the file exists.
 * @throws IOException on file or parent directory creation failure
 */
public static boolean create(final Configuration conf, final FileSystem fs,
    final Path dstFamilyPath, final RegionInfo hfileRegionInfo,
    final String hfileName, final boolean createBackRef) throws IOException {
  TableName linkedTable = hfileRegionInfo.getTable();
  String linkedRegion = hfileRegionInfo.getEncodedName();
  return create(conf, fs, dstFamilyPath, linkedTable, linkedRegion, hfileName, createBackRef);
}