org.apache.hadoop.hbase.MetaTableAccessor Java Examples

The following examples show how to use org.apache.hadoop.hbase.MetaTableAccessor. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: VisibilityController.java    From hbase with Apache License 2.0 6 votes vote down vote up
/********************************* Master related hooks **********************************/

  @Override
  public void postStartMaster(ObserverContext<MasterCoprocessorEnvironment> ctx) throws IOException {
    // Need to create the new system table for labels here
    if (!MetaTableAccessor.tableExists(ctx.getEnvironment().getConnection(), LABELS_TABLE_NAME)) {
      TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
        new TableDescriptorBuilder.ModifyableTableDescriptor(LABELS_TABLE_NAME);
      ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
        new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(LABELS_TABLE_FAMILY);
      familyDescriptor.setBloomFilterType(BloomType.NONE);
      // We will cache all the labels. No need of normal
      // table block cache.
      familyDescriptor.setBlockCacheEnabled(false);
      tableDescriptor.setColumnFamily(familyDescriptor);
      // Let the "labels" table having only one region always. We are not expecting too many labels in
      // the system.
      tableDescriptor.setValue(HTableDescriptor.SPLIT_POLICY,
          DisabledRegionSplitPolicy.class.getName());
      try (Admin admin = ctx.getEnvironment().getConnection().getAdmin()) {
        admin.createTable(tableDescriptor);
      }
    }
  }
 
Example #2
Source File: RegionStateStore.java    From hbase with Apache License 2.0 6 votes vote down vote up
public void visitMeta(final RegionStateVisitor visitor) throws IOException {
  MetaTableAccessor.fullScanRegions(master.getConnection(),
    new ClientMetaTableAccessor.Visitor() {
      final boolean isDebugEnabled = LOG.isDebugEnabled();

      @Override
      public boolean visit(final Result r) throws IOException {
        if (r != null && !r.isEmpty()) {
          long st = 0;
          if (LOG.isTraceEnabled()) {
            st = System.currentTimeMillis();
          }
          visitMetaEntry(visitor, r);
          if (LOG.isTraceEnabled()) {
            long et = System.currentTimeMillis();
            LOG.trace("[T] LOAD META PERF " + StringUtils.humanTimeDiff(et - st));
          }
        } else if (isDebugEnabled) {
          LOG.debug("NULL result from meta - ignoring but this is strange.");
        }
        return true;
      }
    });
}
 
Example #3
Source File: CatalogJanitor.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * For testing against a cluster.
 * Doesn't have a MasterServices context so does not report on good vs bad servers.
 */
public static void main(String [] args) throws IOException {
  checkLog4jProperties();
  ReportMakingVisitor visitor = new ReportMakingVisitor(null);
  Configuration configuration = HBaseConfiguration.create();
  configuration.setBoolean("hbase.defaults.for.version.skip", true);
  try (Connection connection = ConnectionFactory.createConnection(configuration)) {
    /* Used to generate an overlap.
    */
    Get g = new Get(Bytes.toBytes("t2,40,1564119846424.1db8c57d64e0733e0f027aaeae7a0bf0."));
    g.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
    try (Table t = connection.getTable(TableName.META_TABLE_NAME)) {
      Result r = t.get(g);
      byte [] row = g.getRow();
      row[row.length - 2] <<= row[row.length - 2];
      Put p = new Put(g.getRow());
      p.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER,
          r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER));
      t.put(p);
    }
    MetaTableAccessor.scanMetaForTableRegions(connection, visitor, null);
    Report report = visitor.getReport();
    LOG.info(report != null? report.toString(): "empty");
  }
}
 
Example #4
Source File: TestMasterOperationsForRegionReplicas.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testCreateTableWithSingleReplica() throws Exception {
  final int numRegions = 3;
  final int numReplica = 1;
  final TableName tableName = TableName.valueOf(name.getMethodName());
  try {
    TableDescriptor desc =
      TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(numReplica)
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of("family")).build();
    ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions);
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    TEST_UTIL.waitUntilNoRegionsInTransition();

    validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection());
    List<RegionInfo> hris = MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName);
    assertEquals(numRegions * numReplica, hris.size());
  } finally {
    ADMIN.disableTable(tableName);
    ADMIN.deleteTable(tableName);
  }
}
 
Example #5
Source File: OfflineMetaRebuildTestCore.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected RegionInfo createRegion(Configuration conf, final Table htbl,
    byte[] startKey, byte[] endKey) throws IOException {
  Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
  RegionInfo hri = RegionInfoBuilder.newBuilder(htbl.getName())
      .setStartKey(startKey)
      .setEndKey(endKey)
      .build();

  LOG.info("manually adding regioninfo and hdfs data: " + hri.toString());
  Path rootDir = CommonFSUtils.getRootDir(conf);
  FileSystem fs = rootDir.getFileSystem(conf);
  Path p = new Path(CommonFSUtils.getTableDir(rootDir, htbl.getName()),
      hri.getEncodedName());
  fs.mkdirs(p);
  Path riPath = new Path(p, HRegionFileSystem.REGION_INFO_FILE);
  FSDataOutputStream out = fs.create(riPath);
  out.write(RegionInfo.toDelimitedByteArray(hri));
  out.close();

  // add to meta.
  MetaTableAccessor.addRegionToMeta(TEST_UTIL.getConnection(), hri);
  meta.close();
  return hri;
}
 
Example #6
Source File: TestMaster.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Return the region and current deployment for the region containing the given row. If the region
 * cannot be found, returns null. If it is found, but not currently deployed, the second element
 * of the pair may be null.
 */
private Pair<RegionInfo, ServerName> getTableRegionForRow(HMaster master, TableName tableName,
    byte[] rowKey) throws IOException {
  final AtomicReference<Pair<RegionInfo, ServerName>> result = new AtomicReference<>(null);

  ClientMetaTableAccessor.Visitor visitor = new ClientMetaTableAccessor.Visitor() {
    @Override
    public boolean visit(Result data) throws IOException {
      if (data == null || data.size() <= 0) {
        return true;
      }
      Pair<RegionInfo, ServerName> pair = new Pair<>(CatalogFamilyFormat.getRegionInfo(data),
        CatalogFamilyFormat.getServerName(data, 0));
      if (!pair.getFirst().getTable().equals(tableName)) {
        return false;
      }
      result.set(pair);
      return true;
    }
  };

  MetaTableAccessor.scanMeta(master.getConnection(), visitor, tableName, rowKey, 1);
  return result.get();
}
 
Example #7
Source File: TestSimpleRegionNormalizerOnCluster.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testRegionNormalizationMerge() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  try {
    final int currentRegionCount = createTableBegsMerge(tableName);
    assertFalse(admin.normalizerSwitch(true));
    assertTrue(admin.normalize());
    waitForTableMerge(tableName, currentRegionCount - 1);
    assertEquals(
      tableName + " should have merged.",
      currentRegionCount - 1,
      MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
  } finally {
    dropIfExists(tableName);
  }
}
 
Example #8
Source File: AbstractPeerProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected final void setLastPushedSequenceIdForTable(MasterProcedureEnv env, TableName tableName,
    Map<String, Long> lastSeqIds) throws IOException, ReplicationException {
  TableStateManager tsm = env.getMasterServices().getTableStateManager();
  ReplicationQueueStorage queueStorage = env.getReplicationPeerManager().getQueueStorage();
  Connection conn = env.getMasterServices().getConnection();
  if (!needSetLastPushedSequenceId(tsm, tableName)) {
    LOG.debug("Skip settting last pushed sequence id for {}", tableName);
    return;
  }
  for (Pair<String, Long> name2Barrier : MetaTableAccessor
    .getTableEncodedRegionNameAndLastBarrier(conn, tableName)) {
    LOG.trace("Update last pushed sequence id for {}, {}", tableName, name2Barrier);
    addToMap(lastSeqIds, name2Barrier.getFirst(), name2Barrier.getSecond().longValue() - 1,
      queueStorage);
  }
}
 
Example #9
Source File: TestAdmin1.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testSplitShouldNotHappenIfSplitIsDisabledForTable() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  TableDescriptor htd = TableDescriptorBuilder.newBuilder(tableName)
    .setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName())
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
  Table table = TEST_UTIL.createTable(htd, null);
  for (int i = 0; i < 10; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    byte[] q1 = Bytes.toBytes("q1");
    byte[] v1 = Bytes.toBytes("v1");
    p.addColumn(Bytes.toBytes("f"), q1, v1);
    table.put(p);
  }
  ADMIN.flush(tableName);
  try {
    ADMIN.split(tableName, Bytes.toBytes("row5"));
    Threads.sleep(10000);
  } catch (Exception e) {
    // Nothing to do.
  }
  // Split should not happen.
  List<RegionInfo> allRegions =
    MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName, true);
  assertEquals(1, allRegions.size());
}
 
Example #10
Source File: TestMetaFixer.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Just make sure running fixMeta does right thing for the case
 * of a single-region Table where the region gets dropped.
 * There is nothing much we can do. We can't restore what
 * we don't know about (at least from a read of hbase:meta).
 */
@Test
public void testOneRegionTable() throws IOException {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  deleteRegion(services, ris.get(0));
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.isEmpty());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixHoles(report);
  report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertEquals(0, ris.size());
}
 
Example #11
Source File: DeleteTableProcedure.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * There may be items for this table still up in hbase:meta in the case where the info:regioninfo
 * column was empty because of some write error. Remove ALL rows from hbase:meta that have to do
 * with this table. See HBASE-12980.
 */
private static void cleanRegionsInMeta(final MasterProcedureEnv env, final TableName tableName)
    throws IOException {
  Connection connection = env.getMasterServices().getConnection();
  Scan tableScan = MetaTableAccessor.getScanForTableName(connection, tableName);
  try (Table metaTable = connection.getTable(TableName.META_TABLE_NAME)) {
    List<Delete> deletes = new ArrayList<>();
    try (ResultScanner resScanner = metaTable.getScanner(tableScan)) {
      for (Result result : resScanner) {
        deletes.add(new Delete(result.getRow()));
      }
    }
    if (!deletes.isEmpty()) {
      LOG.warn("Deleting some vestigial " + deletes.size() + " rows of " + tableName + " from "
          + TableName.META_TABLE_NAME);
      metaTable.delete(deletes);
    }
  }
}
 
Example #12
Source File: TableStateManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Return all tables in given states.
 * @param states filter by states
 * @return tables in given states
 */
Set<TableName> getTablesInStates(TableState.State... states) throws IOException {
  // Only be called in region normalizer, will not use cache.
  final Set<TableName> rv = Sets.newHashSet();
  MetaTableAccessor.fullScanTables(master.getConnection(), new ClientMetaTableAccessor.Visitor() {
    @Override
    public boolean visit(Result r) throws IOException {
      TableState tableState = CatalogFamilyFormat.getTableState(r);
      if (tableState != null && tableState.inStates(states)) {
        rv.add(tableState.getTableName());
      }
      return true;
    }
  });
  return rv;
}
 
Example #13
Source File: TableStateManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void updateMetaState(TableName tableName, TableState.State newState) throws IOException {
  if (tableName.equals(TableName.META_TABLE_NAME)) {
    if (TableState.State.DISABLING.equals(newState) ||
        TableState.State.DISABLED.equals(newState)) {
      throw new IllegalArgumentIOException("Cannot disable meta table; " + newState);
    }
    // Otherwise, just return; no need to set ENABLED on meta -- it is always ENABLED.
    return;
  }
  boolean succ = false;
  try {
    MetaTableAccessor.updateTableState(master.getConnection(), tableName, newState);
    tableName2State.put(tableName, newState);
    succ = true;
  } finally {
    if (!succ) {
      this.tableName2State.remove(tableName);
    }
  }
  metaStateUpdated(tableName, newState);
}
 
Example #14
Source File: NamespaceStateManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Initialize namespace state cache by scanning meta table.
 */
private void initialize() throws IOException {
  List<NamespaceDescriptor> namespaces = this.master.getClusterSchema().getNamespaces();
  for (NamespaceDescriptor namespace : namespaces) {
    addNamespace(namespace.getName());
    List<TableName> tables = this.master.listTableNamesByNamespace(namespace.getName());
    for (TableName table : tables) {
      if (table.isSystemTable()) {
        continue;
      }
      List<RegionInfo> regions =
          MetaTableAccessor.getTableRegions(this.master.getConnection(), table, true);
      addTable(table, regions.size());
    }
  }
  LOG.info("Finished updating state of " + nsStateCache.size() + " namespaces. ");
  initialized = true;
}
 
Example #15
Source File: TestRegionMergeTransactionOnCluster.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void waitAndVerifyRegionNum(HMaster master, TableName tablename,
    int expectedRegionNum) throws Exception {
  List<Pair<RegionInfo, ServerName>> tableRegionsInMeta;
  List<RegionInfo> tableRegionsInMaster;
  long timeout = System.currentTimeMillis() + waitTime;
  while (System.currentTimeMillis() < timeout) {
    tableRegionsInMeta =
        MetaTableAccessor.getTableRegionsAndLocations(TEST_UTIL.getConnection(), tablename);
    tableRegionsInMaster =
        master.getAssignmentManager().getRegionStates().getRegionsOfTable(tablename);
    LOG.info(Objects.toString(tableRegionsInMaster));
    LOG.info(Objects.toString(tableRegionsInMeta));
    int tableRegionsInMetaSize = tableRegionsInMeta.size();
    int tableRegionsInMasterSize = tableRegionsInMaster.size();
    if (tableRegionsInMetaSize == expectedRegionNum
        && tableRegionsInMasterSize == expectedRegionNum) {
      break;
    }
    Thread.sleep(250);
  }

  tableRegionsInMeta = MetaTableAccessor.getTableRegionsAndLocations(
      TEST_UTIL.getConnection(), tablename);
  LOG.info("Regions after merge:" + Joiner.on(',').join(tableRegionsInMeta));
  assertEquals(expectedRegionNum, tableRegionsInMeta.size());
}
 
Example #16
Source File: HMaster.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Check hbase:namespace table is assigned. If not, startup will hang looking for the ns table
 * <p/>
 * This is for rolling upgrading, later we will migrate the data in ns table to the ns family of
 * meta table. And if this is a new cluster, this method will return immediately as there will be
 * no namespace table/region.
 * @return True if namespace table is up/online.
 */
private boolean waitForNamespaceOnline() throws IOException {
  TableState nsTableState =
    MetaTableAccessor.getTableState(getConnection(), TableName.NAMESPACE_TABLE_NAME);
  if (nsTableState == null || nsTableState.isDisabled()) {
    // this means we have already migrated the data and disabled or deleted the namespace table,
    // or this is a new deploy which does not have a namespace table from the beginning.
    return true;
  }
  List<RegionInfo> ris =
    this.assignmentManager.getRegionStates().getRegionsOfTable(TableName.NAMESPACE_TABLE_NAME);
  if (ris.isEmpty()) {
    // maybe this will not happen any more, but anyway, no harm to add a check here...
    return true;
  }
  // Else there are namespace regions up in meta. Ensure they are assigned before we go on.
  for (RegionInfo ri : ris) {
    if (!isRegionOnline(ri)) {
      return false;
    }
  }
  return true;
}
 
Example #17
Source File: IndexSplitTransaction.java    From phoenix with Apache License 2.0 6 votes vote down vote up
private void offlineParentInMetaAndputMetaEntries(Connection conn,
    HRegionInfo parent, HRegionInfo splitA, HRegionInfo splitB,
    ServerName serverName, List<Mutation> metaEntries) throws IOException {
  List<Mutation> mutations = metaEntries;
  HRegionInfo copyOfParent = new HRegionInfo(parent);
  copyOfParent.setOffline(true);
  copyOfParent.setSplit(true);

  //Put for parent
  Put putParent = MetaTableAccessor.makePutFromRegionInfo(copyOfParent);
  MetaTableAccessor.addDaughtersToPut(putParent, splitA, splitB);
  mutations.add(putParent);
  
  //Puts for daughters
  Put putA = MetaTableAccessor.makePutFromRegionInfo(splitA);
  Put putB = MetaTableAccessor.makePutFromRegionInfo(splitB);

  addLocation(putA, serverName, 1); //these are new regions, openSeqNum = 1 is fine.
  addLocation(putB, serverName, 1);
  mutations.add(putA);
  mutations.add(putB);
  MetaTableAccessor.mutateMetaTable(conn, mutations);
}
 
Example #18
Source File: TestRegionMergeTransactionOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
private PairOfSameType<RegionInfo> requestMergeRegion(
    HMaster master, TableName tablename,
    int regionAnum, int regionBnum) throws Exception {
  List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
      .getTableRegionsAndLocations(
          TEST_UTIL.getConnection(), tablename);
  RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
  RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
  ADMIN.mergeRegionsAsync(
    regionA.getEncodedNameAsBytes(),
    regionB.getEncodedNameAsBytes(), false);
  return new PairOfSameType<>(regionA, regionB);
}
 
Example #19
Source File: MasterRpcServices.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Update state of the region in meta only. This is required by hbck in some situations to cleanup
 * stuck assign/ unassign regions procedures for the table.
 *
 * @return previous states of the regions
 */
@Override
public SetRegionStateInMetaResponse setRegionStateInMeta(RpcController controller,
  SetRegionStateInMetaRequest request) throws ServiceException {
  SetRegionStateInMetaResponse.Builder builder = SetRegionStateInMetaResponse.newBuilder();
  try {
    for (RegionSpecifierAndState s : request.getStatesList()) {
      RegionSpecifier spec = s.getRegionSpecifier();
      String encodedName;
      if (spec.getType() == RegionSpecifierType.ENCODED_REGION_NAME) {
        encodedName = spec.getValue().toStringUtf8();
      } else {
        // TODO: actually, a full region name can save a lot on meta scan, improve later.
        encodedName = RegionInfo.encodeRegionName(spec.getValue().toByteArray());
      }
      RegionInfo info = this.master.getAssignmentManager().loadRegionFromMeta(encodedName);
      LOG.trace("region info loaded from meta table: {}", info);
      RegionState prevState =
        this.master.getAssignmentManager().getRegionStates().getRegionState(info);
      RegionState.State newState = RegionState.State.convert(s.getState());
      LOG.info("{} set region={} state from {} to {}", master.getClientIdAuditPrefix(), info,
        prevState.getState(), newState);
      Put metaPut = MetaTableAccessor.makePutFromRegionInfo(info, System.currentTimeMillis());
      metaPut.addColumn(HConstants.CATALOG_FAMILY, HConstants.STATE_QUALIFIER,
        Bytes.toBytes(newState.name()));
      List<Put> putList = new ArrayList<>();
      putList.add(metaPut);
      MetaTableAccessor.putsToMetaTable(this.master.getConnection(), putList);
      // Loads from meta again to refresh AM cache with the new region state
      this.master.getAssignmentManager().loadRegionFromMeta(encodedName);
      builder.addStates(RegionSpecifierAndState.newBuilder().setRegionSpecifier(spec)
        .setState(prevState.getState().convert()));
    }
  } catch (Exception e) {
    throw new ServiceException(e);
  }
  return builder.build();
}
 
Example #20
Source File: TableNamespaceManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void start() throws IOException {
  TableState nsTableState = MetaTableAccessor.getTableState(masterServices.getConnection(),
    TableName.NAMESPACE_TABLE_NAME);
  if (nsTableState != null && nsTableState.isEnabled()) {
    migrateNamespaceTable();
  }
  loadNamespaceIntoCache();
}
 
Example #21
Source File: TestSimpleRegionNormalizerOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * create a table with 5 regions, having region sizes so as to provoke a merge
 * of the smallest regions.
 * <ul>
 *   <li>total table size: 13</li>
 *   <li>average region size: 2.6</li>
 *   <li>sum of sizes of first two regions < average</li>
 * </ul>
 */
private static int createTableBegsMerge(final TableName tableName) throws IOException {
  // create 5 regions with sizes to trigger merge of small regions
  final List<HRegion> generatedRegions = generateTestData(tableName, 1, 1, 3, 3, 5);
  assertEquals(5, MetaTableAccessor.getRegionCount(TEST_UTIL.getConnection(), tableName));
  admin.flush(tableName);

  final TableDescriptor td = TableDescriptorBuilder.newBuilder(admin.getDescriptor(tableName))
    .setNormalizationEnabled(true)
    .build();
  admin.modifyTable(td);

  // make sure relatively accurate region statistics are available for the test table. use
  // the last/largest region as clue.
  LOG.debug("waiting for region statistics to settle.");
  TEST_UTIL.waitFor(TimeUnit.MINUTES.toMillis(1), new ExplainingPredicate<IOException>() {
    @Override public String explainFailure() {
      return "expected largest region to be >= 4mb.";
    }
    @Override public boolean evaluate() {
      return generatedRegions.stream()
        .mapToDouble(val -> getRegionSizeMB(master, val.getRegionInfo()))
        .allMatch(val -> val > 0)
        && getRegionSizeMB(master, generatedRegions.get(4).getRegionInfo()) >= 4.0;
    }
  });
  return 5;
}
 
Example #22
Source File: TableMapReduceUtil.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Ensures that the given number of reduce tasks for the given job
 * configuration does not exceed the number of regions for the given table.
 *
 * @param table  The table to get the region count for.
 * @param job  The current job configuration to adjust.
 * @throws IOException When retrieving the table details fails.
 */
// Used by tests.
public static void limitNumReduceTasks(String table, JobConf job)
throws IOException {
  int regions =
    MetaTableAccessor.getRegionCount(HBaseConfiguration.create(job), TableName.valueOf(table));
  if (job.getNumReduceTasks() > regions)
    job.setNumReduceTasks(regions);
}
 
Example #23
Source File: RegionSplitter.java    From hbase with Apache License 2.0 5 votes vote down vote up
static void createPresplitTable(TableName tableName, SplitAlgorithm splitAlgo,
        String[] columnFamilies, Configuration conf)
throws IOException, InterruptedException {
  final int splitCount = conf.getInt("split.count", 0);
  Preconditions.checkArgument(splitCount > 1, "Split count must be > 1");

  Preconditions.checkArgument(columnFamilies.length > 0,
      "Must specify at least one column family. ");
  LOG.debug("Creating table " + tableName + " with " + columnFamilies.length
      + " column families.  Presplitting to " + splitCount + " regions");

  TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
  for (String cf : columnFamilies) {
    builder.setColumnFamily(ColumnFamilyDescriptorBuilder.of(cf));
  }
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    Admin admin = connection.getAdmin();
    try {
      Preconditions.checkArgument(!admin.tableExists(tableName),
        "Table already exists: " + tableName);
      admin.createTable(builder.build(), splitAlgo.split(splitCount));
    } finally {
      admin.close();
    }
    LOG.debug("Table created!  Waiting for regions to show online in META...");
    if (!conf.getBoolean("split.verify", true)) {
      // NOTE: createTable is synchronous on the table, but not on the regions
      int onlineRegions = 0;
      while (onlineRegions < splitCount) {
        onlineRegions = MetaTableAccessor.getRegionCount(connection, tableName);
        LOG.debug(onlineRegions + " of " + splitCount + " regions online...");
        if (onlineRegions < splitCount) {
          Thread.sleep(10 * 1000); // sleep
        }
      }
    }
    LOG.debug("Finished creating table with " + splitCount + " regions");
  }
}
 
Example #24
Source File: CompatUtil.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public static List<RegionInfo> getMergeRegions(Connection conn, byte[] regionName) 
        throws IOException {
    Pair<RegionInfo, RegionInfo> regionPair = 
            MetaTableAccessor.getRegionsFromMergeQualifier(conn,regionName);
    List<RegionInfo> regionList = new ArrayList<RegionInfo>(2);
    regionList.add(regionPair.getFirst());
    regionList.add(regionPair.getSecond());
    return regionList;
}
 
Example #25
Source File: DisableTableProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Action before any real action of disabling table. Set the exception in the procedure instead
 * of throwing it.  This approach is to deal with backward compatible with 1.0.
 * @param env MasterProcedureEnv
 */
private boolean prepareDisable(final MasterProcedureEnv env) throws IOException {
  boolean canTableBeDisabled = true;
  if (tableName.equals(TableName.META_TABLE_NAME)) {
    setFailure("master-disable-table",
      new ConstraintException("Cannot disable " + this.tableName));
    canTableBeDisabled = false;
  } else if (!MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
    setFailure("master-disable-table", new TableNotFoundException(tableName));
    canTableBeDisabled = false;
  } else if (!skipTableStateCheck) {
    // There could be multiple client requests trying to disable or enable
    // the table at the same time. Ensure only the first request is honored
    // After that, no other requests can be accepted until the table reaches
    // DISABLED or ENABLED.
    //
    // Note: in 1.0 release, we called TableStateManager.setTableStateIfInStates() to set
    // the state to DISABLING from ENABLED. The implementation was done before table lock
    // was implemented. With table lock, there is no need to set the state here (it will
    // set the state later on). A quick state check should be enough for us to move forward.
    TableStateManager tsm = env.getMasterServices().getTableStateManager();
    TableState ts = tsm.getTableState(tableName);
    if (!ts.isEnabled()) {
      LOG.info("Not ENABLED, state={}, skipping disable; {}", ts.getState(), this);
      setFailure("master-disable-table", new TableNotEnabledException(ts.toString()));
      canTableBeDisabled = false;
    }
  }

  // We are done the check. Future actions in this procedure could be done asynchronously.
  releaseSyncLatch();

  return canTableBeDisabled;
}
 
Example #26
Source File: TestReplicationBarrierCleaner.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testDeleteRowForDeletedRegion() throws IOException, ReplicationException {
  TableName tableName = TableName.valueOf(name.getMethodName());
  RegionInfo region = RegionInfoBuilder.newBuilder(tableName).build();
  addBarrier(region, 40, 50, 60);
  fillCatalogFamily(region);

  String peerId = "1";
  ReplicationQueueStorage queueStorage = create(59L);
  @SuppressWarnings("unchecked")
  ReplicationPeerManager peerManager = create(queueStorage, Lists.newArrayList(peerId));
  ReplicationBarrierCleaner cleaner = create(peerManager);

  // we have something in catalog family, so only delete 40
  cleaner.chore();
  assertArrayEquals(new long[] { 50, 60 },
    MetaTableAccessor.getReplicationBarrier(UTIL.getConnection(), region.getRegionName()));
  verify(queueStorage, never()).removeLastSequenceIds(anyString(), anyList());

  // No catalog family, then we should remove the whole row
  clearCatalogFamily(region);
  cleaner.chore();
  try (Table table = UTIL.getConnection().getTable(TableName.META_TABLE_NAME)) {
    assertFalse(table
      .exists(new Get(region.getRegionName()).addFamily(HConstants.REPLICATION_BARRIER_FAMILY)));
  }
  verify(queueStorage, times(1)).removeLastSequenceIds(peerId,
    Arrays.asList(region.getEncodedName()));
}
 
Example #27
Source File: CreateTableProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Add the specified set of regions to the hbase:meta table.
 */
private static void addRegionsToMeta(final MasterProcedureEnv env,
    final TableDescriptor tableDescriptor,
    final List<RegionInfo> regionInfos) throws IOException {
  MetaTableAccessor.addRegionsToMeta(env.getMasterServices().getConnection(),
    regionInfos, tableDescriptor.getRegionReplication());
}
 
Example #28
Source File: CreateTableProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
private boolean prepareCreate(final MasterProcedureEnv env) throws IOException {
  final TableName tableName = getTableName();
  if (MetaTableAccessor.tableExists(env.getMasterServices().getConnection(), tableName)) {
    setFailure("master-create-table", new TableExistsException(getTableName()));
    return false;
  }

  // check that we have at least 1 CF
  if (tableDescriptor.getColumnFamilyCount() == 0) {
    setFailure("master-create-table", new DoNotRetryIOException(
      "Table " + getTableName().toString() + " should have at least one column family."));
    return false;
  }
  if (!tableName.isSystemTable()) {
    // do not check rs group for system tables as we may block the bootstrap.
    Supplier<String> forWhom = () -> "table " + tableName;
    RSGroupInfo rsGroupInfo = MasterProcedureUtil.checkGroupExists(
      env.getMasterServices().getRSGroupInfoManager()::getRSGroup,
      tableDescriptor.getRegionServerGroup(), forWhom);
    if (rsGroupInfo == null) {
      // we do not set rs group info on table, check if we have one on namespace
      String namespace = tableName.getNamespaceAsString();
      NamespaceDescriptor nd = env.getMasterServices().getClusterSchema().getNamespace(namespace);
      forWhom = () -> "table " + tableName + "(inherit from namespace)";
      rsGroupInfo = MasterProcedureUtil.checkGroupExists(
        env.getMasterServices().getRSGroupInfoManager()::getRSGroup,
        MasterProcedureUtil.getNamespaceGroup(nd), forWhom);
    }
    MasterProcedureUtil.checkGroupNotEmpty(rsGroupInfo, forWhom);
  }

  return true;
}
 
Example #29
Source File: TestSlowLogAccessor.java    From hbase with Apache License 2.0 5 votes vote down vote up
private Connection waitForSlowLogTableCreation() {
  Connection connection =
    HBASE_TESTING_UTILITY.getMiniHBaseCluster().getRegionServer(0).getConnection();
  Assert.assertNotEquals(-1, HBASE_TESTING_UTILITY.waitFor(2000, () -> {
    try {
      return MetaTableAccessor.tableExists(connection, SlowLogTableAccessor.SLOW_LOG_TABLE_NAME);
    } catch (IOException e) {
      return false;
    }
  }));
  return connection;
}
 
Example #30
Source File: RegionReplicaInfo.java    From hbase with Apache License 2.0 5 votes vote down vote up
private RegionReplicaInfo(final Result result, final HRegionLocation location) {
  this.row = result != null ? result.getRow() : null;
  this.regionInfo = location != null ? location.getRegion() : null;
  this.regionState = (result != null && regionInfo != null)
    ? RegionStateStore.getRegionState(result, regionInfo)
    : null;
  this.serverName = location != null ? location.getServerName() : null;
  this.seqNum = (location != null) ? location.getSeqNum() : HConstants.NO_SEQNUM;
  this.targetServerName = (result != null && regionInfo != null)
    ? MetaTableAccessor.getTargetServerName(result, regionInfo.getReplicaId())
    : null;
  this.mergeRegionInfo = (result != null)
    ? MetaTableAccessor.getMergeRegionsWithName(result.rawCells())
    : null;

  if (result != null) {
    PairOfSameType<RegionInfo> daughterRegions = MetaTableAccessor.getDaughterRegions(result);
    this.splitRegionInfo = new LinkedHashMap<>();
    if (daughterRegions.getFirst() != null) {
      splitRegionInfo.put(HConstants.SPLITA_QUALIFIER_STR, daughterRegions.getFirst());
    }
    if (daughterRegions.getSecond() != null) {
      splitRegionInfo.put(HConstants.SPLITB_QUALIFIER_STR, daughterRegions.getSecond());
    }
  } else {
    this.splitRegionInfo = null;
  }
}