org.apache.hadoop.hbase.util.PairOfSameType Java Examples

The following examples show how to use org.apache.hadoop.hbase.util.PairOfSameType. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: MetricsMasterWrapperImpl.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public PairOfSameType<Integer> getRegionCounts() {
  try {
    if (!master.isInitialized()) {
      return new PairOfSameType<>(0, 0);
    }
    Integer onlineRegionCount = 0;
    Integer offlineRegionCount = 0;

    List<TableDescriptor> descriptors = master.listTableDescriptors(null, null,
        null, false);

    for (TableDescriptor htDesc : descriptors) {
      TableName tableName = htDesc.getTableName();
      Map<RegionState.State, List<RegionInfo>> tableRegions =
          master.getAssignmentManager().getRegionStates()
              .getRegionByStateOfTable(tableName);
      onlineRegionCount += tableRegions.get(RegionState.State.OPEN).size();
      offlineRegionCount += tableRegions.get(RegionState.State.OFFLINE).size();
    }
    return new PairOfSameType<>(onlineRegionCount, offlineRegionCount);
  } catch (IOException e) {
    return new PairOfSameType<>(0, 0);
  }
}
 
Example #2
Source File: TestCatalogJanitorInMemoryStates.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Splits a region
 * @param r Region to split.
 * @return List of region locations
 */
private List<HRegionLocation> splitRegion(final RegionInfo r)
    throws IOException, InterruptedException, ExecutionException {
  List<HRegionLocation> locations = new ArrayList<>();
  // Split this table in two.
  Admin admin = TEST_UTIL.getAdmin();
  Connection connection = TEST_UTIL.getConnection();
  admin.splitRegionAsync(r.getEncodedNameAsBytes()).get();
  admin.close();
  PairOfSameType<RegionInfo> regions = waitOnDaughters(r);
  if (regions != null) {
    try (RegionLocator rl = connection.getRegionLocator(r.getTable())) {
      locations.add(rl.getRegionLocation(regions.getFirst().getEncodedNameAsBytes()));
      locations.add(rl.getRegionLocation(regions.getSecond().getEncodedNameAsBytes()));
    }
    return locations;
  }
  return locations;
}
 
Example #3
Source File: IndexSplitTransaction.java    From phoenix with Apache License 2.0 5 votes vote down vote up
public PairOfSameType<HRegion> stepsAfterPONR(final Server server,
    final RegionServerServices services, PairOfSameType<HRegion> regions)
    throws IOException {
  openDaughters(server, services, regions.getFirst(), regions.getSecond());
  transitionZKNode(server, services, regions.getFirst(), regions.getSecond());
  return regions;
}
 
Example #4
Source File: CatalogJanitor.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * If daughters no longer hold reference to the parents, delete the parent.
 * @param parent RegionInfo of split offlined parent
 * @param rowContent Content of <code>parent</code> row in
 * <code>metaRegionName</code>
 * @return True if we removed <code>parent</code> from meta table and from
 * the filesystem.
 */
boolean cleanParent(final RegionInfo parent, Result rowContent)
throws IOException {
  // Check whether it is a merged region and if it is clean of references.
  if (MetaTableAccessor.hasMergeRegions(rowContent.rawCells())) {
    // Wait until clean of merge parent regions first
    return false;
  }
  // Run checks on each daughter split.
  PairOfSameType<RegionInfo> daughters = MetaTableAccessor.getDaughterRegions(rowContent);
  Pair<Boolean, Boolean> a = checkDaughterInFs(parent, daughters.getFirst());
  Pair<Boolean, Boolean> b = checkDaughterInFs(parent, daughters.getSecond());
  if (hasNoReferences(a) && hasNoReferences(b)) {
    String daughterA = daughters.getFirst() != null?
        daughters.getFirst().getShortNameToLog(): "null";
    String daughterB = daughters.getSecond() != null?
        daughters.getSecond().getShortNameToLog(): "null";
    LOG.debug("Deleting region " + parent.getShortNameToLog() +
      " because daughters -- " + daughterA + ", " + daughterB +
      " -- no longer hold references");
    ProcedureExecutor<MasterProcedureEnv> pe = this.services.getMasterProcedureExecutor();
    pe.submitProcedure(new GCRegionProcedure(pe.getEnvironment(), parent));
    // Remove from in-memory states
    this.services.getAssignmentManager().getRegionStates().deleteRegion(parent);
    this.services.getServerManager().removeRegion(parent);
    return true;
  }
  return false;
}
 
Example #5
Source File: RegionReplicaInfo.java    From hbase with Apache License 2.0 5 votes vote down vote up
private RegionReplicaInfo(final Result result, final HRegionLocation location) {
  this.row = result != null ? result.getRow() : null;
  this.regionInfo = location != null ? location.getRegion() : null;
  this.regionState = (result != null && regionInfo != null)
    ? RegionStateStore.getRegionState(result, regionInfo)
    : null;
  this.serverName = location != null ? location.getServerName() : null;
  this.seqNum = (location != null) ? location.getSeqNum() : HConstants.NO_SEQNUM;
  this.targetServerName = (result != null && regionInfo != null)
    ? MetaTableAccessor.getTargetServerName(result, regionInfo.getReplicaId())
    : null;
  this.mergeRegionInfo = (result != null)
    ? MetaTableAccessor.getMergeRegionsWithName(result.rawCells())
    : null;

  if (result != null) {
    PairOfSameType<RegionInfo> daughterRegions = MetaTableAccessor.getDaughterRegions(result);
    this.splitRegionInfo = new LinkedHashMap<>();
    if (daughterRegions.getFirst() != null) {
      splitRegionInfo.put(HConstants.SPLITA_QUALIFIER_STR, daughterRegions.getFirst());
    }
    if (daughterRegions.getSecond() != null) {
      splitRegionInfo.put(HConstants.SPLITB_QUALIFIER_STR, daughterRegions.getSecond());
    }
  } else {
    this.splitRegionInfo = null;
  }
}
 
Example #6
Source File: TestCatalogJanitorInMemoryStates.java    From hbase with Apache License 2.0 5 votes vote down vote up
private PairOfSameType<RegionInfo> waitOnDaughters(final RegionInfo r)
    throws IOException {
  long start = System.currentTimeMillis();
  PairOfSameType<RegionInfo> pair = null;
  try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
       Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
    Result result = null;
    RegionInfo region = null;
    while ((System.currentTimeMillis() - start) < 60000) {
      result = metaTable.get(new Get(r.getRegionName()));
      if (result == null) {
        break;
      }
      region = CatalogFamilyFormat.getRegionInfo(result);
      if (region.isSplitParent()) {
        LOG.debug(region.toString() + " IS a parent!");
        pair = MetaTableAccessor.getDaughterRegions(result);
        break;
      }
      Threads.sleep(100);
    }

    if (pair.getFirst() == null || pair.getSecond() == null) {
      throw new IOException("Failed to get daughters, for parent region: " + r);
    }
    return pair;
  }
}
 
Example #7
Source File: TestRegionMergeTransactionOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testWholesomeMerge() throws Exception {
  LOG.info("Starting " + name.getMethodName());
  final TableName tableName = TableName.valueOf(name.getMethodName());

  try {
    // Create table and load data.
    Table table = createTableAndLoadData(MASTER, tableName);
    // Merge 1st and 2nd region
    mergeRegionsAndVerifyRegionNum(MASTER, tableName, 0, 1, INITIAL_REGION_NUM - 1);

    // Merge 2nd and 3th region
    PairOfSameType<RegionInfo> mergedRegions =
      mergeRegionsAndVerifyRegionNum(MASTER, tableName, 1, 2, INITIAL_REGION_NUM - 2);

    verifyRowCount(table, ROWSIZE);

    // Randomly choose one of the two merged regions
    RegionInfo hri = RandomUtils.nextBoolean() ? mergedRegions.getFirst() : mergedRegions.getSecond();
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    AssignmentManager am = cluster.getMaster().getAssignmentManager();
    RegionStates regionStates = am.getRegionStates();

    // We should not be able to assign it again
    am.assign(hri);
    assertFalse("Merged region can't be assigned", regionStates.isRegionInTransition(hri));

    // We should not be able to unassign it either
    am.unassign(hri);
    assertFalse("Merged region can't be unassigned", regionStates.isRegionInTransition(hri));

    table.close();
  } finally {
    TEST_UTIL.deleteTable(tableName);
  }
}
 
Example #8
Source File: TestRegionMergeTransactionOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
private PairOfSameType<RegionInfo> mergeRegionsAndVerifyRegionNum(
    HMaster master, TableName tablename,
    int regionAnum, int regionBnum, int expectedRegionNum) throws Exception {
  PairOfSameType<RegionInfo> mergedRegions =
    requestMergeRegion(master, tablename, regionAnum, regionBnum);
  waitAndVerifyRegionNum(master, tablename, expectedRegionNum);
  return mergedRegions;
}
 
Example #9
Source File: TestRegionMergeTransactionOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
private PairOfSameType<RegionInfo> requestMergeRegion(
    HMaster master, TableName tablename,
    int regionAnum, int regionBnum) throws Exception {
  List<Pair<RegionInfo, ServerName>> tableRegions = MetaTableAccessor
      .getTableRegionsAndLocations(
          TEST_UTIL.getConnection(), tablename);
  RegionInfo regionA = tableRegions.get(regionAnum).getFirst();
  RegionInfo regionB = tableRegions.get(regionBnum).getFirst();
  ADMIN.mergeRegionsAsync(
    regionA.getEncodedNameAsBytes(),
    regionB.getEncodedNameAsBytes(), false);
  return new PairOfSameType<>(regionA, regionB);
}
 
Example #10
Source File: IndexSplitTransaction.java    From phoenix with Apache License 2.0 4 votes vote down vote up
/**
 * Prepare the regions and region files.
 * @param server Hosting server instance.  Can be null when testing (won't try
 * and update in zk if a null server)
 * @param services Used to online/offline regions.
 * @throws IOException If thrown, transaction failed.
 *    Call {@link #rollback(Server, RegionServerServices)}
 * @return Regions created
 */
/* package */PairOfSameType<HRegion> createDaughters(final Server server,
    final RegionServerServices services) throws IOException {
  LOG.info("Starting split of region " + this.parent);
  if ((server != null && server.isStopped()) ||
      (services != null && services.isStopping())) {
    throw new IOException("Server is stopped or stopping");
  }
  assert !this.parent.lock.writeLock().isHeldByCurrentThread():
    "Unsafe to hold write lock while performing RPCs";

  // Coprocessor callback
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplit();
  }

  // Coprocessor callback
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplit(this.splitrow);
  }

  // If true, no cluster to write meta edits to or to update znodes in.
  boolean testing = server == null? true:
      server.getConfiguration().getBoolean("hbase.testing.nocluster", false);
  this.fileSplitTimeout = testing ? this.fileSplitTimeout :
      server.getConfiguration().getLong("hbase.regionserver.fileSplitTimeout",
        this.fileSplitTimeout);

  PairOfSameType<HRegion> daughterRegions = stepsBeforePONR(server, services, testing);

  List<Mutation> metaEntries = new ArrayList<Mutation>();
  if (this.parent.getCoprocessorHost() != null) {
    if (this.parent.getCoprocessorHost().
        preSplitBeforePONR(this.splitrow, metaEntries)) {
      throw new IOException("Coprocessor bypassing region "
          + this.parent.getRegionNameAsString() + " split.");
    }
    try {
      for (Mutation p : metaEntries) {
        HRegionInfo.parseRegionName(p.getRow());
      }
    } catch (IOException e) {
      LOG.error("Row key of mutation from coprossor is not parsable as region name."
          + "Mutations from coprocessor should only for hbase:meta table.");
      throw e;
    }
  }

  // This is the point of no return.  Adding subsequent edits to .META. as we
  // do below when we do the daughter opens adding each to .META. can fail in
  // various interesting ways the most interesting of which is a timeout
  // BUT the edits all go through (See HBASE-3872).  IF we reach the PONR
  // then subsequent failures need to crash out this regionserver; the
  // server shutdown processing should be able to fix-up the incomplete split.
  // The offlined parent will have the daughters as extra columns.  If
  // we leave the daughter regions in place and do not remove them when we
  // crash out, then they will have their references to the parent in place
  // still and the server shutdown fixup of .META. will point to these
  // regions.
  // We should add PONR JournalEntry before offlineParentInMeta,so even if
  // OfflineParentInMeta timeout,this will cause regionserver exit,and then
  // master ServerShutdownHandler will fix daughter & avoid data loss. (See
  // HBase-4562).
  this.journal.add(JournalEntry.PONR);

  // Edit parent in meta.  Offlines parent region and adds splita and splitb
  // as an atomic update. See HBASE-7721. This update to META makes the region
  // will determine whether the region is split or not in case of failures.
  // If it is successful, master will roll-forward, if not, master will rollback
  // and assign the parent region.
  if (!testing) {
    if (metaEntries == null || metaEntries.isEmpty()) {
      MetaTableAccessor.splitRegion(server.getConnection(), parent.getRegionInfo(),
              daughterRegions.getFirst().getRegionInfo(),
              daughterRegions.getSecond().getRegionInfo(), server.getServerName(), 1);
    } else {
      offlineParentInMetaAndputMetaEntries(server.getConnection(),
        parent.getRegionInfo(), daughterRegions.getFirst().getRegionInfo(), daughterRegions
            .getSecond().getRegionInfo(), server.getServerName(), metaEntries);
    }
  }
  return daughterRegions;
}
 
Example #11
Source File: MetaTableAccessor.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Returns the daughter regions by reading the corresponding columns of the catalog table Result.
 * @param data a Result object from the catalog table scan
 * @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
 */
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
  RegionInfo splitA = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
  RegionInfo splitB = CatalogFamilyFormat.getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
  return new PairOfSameType<>(splitA, splitB);
}
 
Example #12
Source File: TestEndToEndSplitTransaction.java    From hbase with Apache License 2.0 4 votes vote down vote up
/**
 * Blocks until the region split is complete in hbase:meta and region server opens the daughters
 */
public static void blockUntilRegionSplit(Configuration conf, long timeout,
    final byte[] regionName, boolean waitForDaughters) throws IOException, InterruptedException {
  long start = System.currentTimeMillis();
  log("blocking until region is split:" + Bytes.toStringBinary(regionName));
  RegionInfo daughterA = null, daughterB = null;
  try (Connection conn = ConnectionFactory.createConnection(conf);
      Table metaTable = conn.getTable(TableName.META_TABLE_NAME)) {
    Result result = null;
    RegionInfo region = null;
    while ((System.currentTimeMillis() - start) < timeout) {
      result = metaTable.get(new Get(regionName));
      if (result == null) {
        break;
      }

      region = CatalogFamilyFormat.getRegionInfo(result);
      if (region.isSplitParent()) {
        log("found parent region: " + region.toString());
        PairOfSameType<RegionInfo> pair = MetaTableAccessor.getDaughterRegions(result);
        daughterA = pair.getFirst();
        daughterB = pair.getSecond();
        break;
      }
      Threads.sleep(100);
    }
    if (daughterA == null || daughterB == null) {
      throw new IOException("Failed to get daughters, daughterA=" + daughterA + ", daughterB=" +
        daughterB + ", timeout=" + timeout + ", result=" + result + ", regionName=" +
        Bytes.toString(regionName) + ", region=" + region);
    }

    // if we are here, this means the region split is complete or timed out
    if (waitForDaughters) {
      long rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsInMeta(conn, rem, daughterA);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsInMeta(conn, rem, daughterB);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsOpened(conf, rem, daughterA);

      rem = timeout - (System.currentTimeMillis() - start);
      blockUntilRegionIsOpened(conf, rem, daughterB);

      // Compacting the new region to make sure references can be cleaned up
      compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
        TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterA.getRegionName());
      compactAndBlockUntilDone(TEST_UTIL.getAdmin(),
        TEST_UTIL.getMiniHBaseCluster().getRegionServer(0), daughterB.getRegionName());

      removeCompactedFiles(conn, timeout, daughterA);
      removeCompactedFiles(conn, timeout, daughterB);
    }
  }
}
 
Example #13
Source File: MetricsMasterSourceImpl.java    From hbase with Apache License 2.0 4 votes vote down vote up
@Override
public void getMetrics(MetricsCollector metricsCollector, boolean all) {

  MetricsRecordBuilder metricsRecordBuilder = metricsCollector.addRecord(metricsName);

  // masterWrapper can be null because this function is called inside of init.
  // If the master is already stopped or has initiated a shutdown, no point in registering the
  // metrics again.
  if (masterWrapper != null && masterWrapper.isRunning()) {

    // Pair<online region number, offline region number>
    PairOfSameType<Integer> regionNumberPair = masterWrapper.getRegionCounts();

    metricsRecordBuilder
        .addGauge(Interns.info(MERGE_PLAN_COUNT_NAME, MERGE_PLAN_COUNT_DESC),
            masterWrapper.getMergePlanCount())
        .addGauge(Interns.info(SPLIT_PLAN_COUNT_NAME, SPLIT_PLAN_COUNT_DESC),
            masterWrapper.getSplitPlanCount())
        .addGauge(Interns.info(MASTER_ACTIVE_TIME_NAME,
            MASTER_ACTIVE_TIME_DESC), masterWrapper.getActiveTime())
        .addGauge(Interns.info(MASTER_START_TIME_NAME,
            MASTER_START_TIME_DESC), masterWrapper.getStartTime())
        .addGauge(Interns.info(MASTER_FINISHED_INITIALIZATION_TIME_NAME,
                MASTER_FINISHED_INITIALIZATION_TIME_DESC),
                masterWrapper.getMasterInitializationTime())
        .addGauge(Interns.info(AVERAGE_LOAD_NAME, AVERAGE_LOAD_DESC),
            masterWrapper.getAverageLoad())
        .addGauge(Interns.info(ONLINE_REGION_COUNT_NAME, ONLINE_REGION_COUNT_DESC),
            regionNumberPair.getFirst())
        .addGauge(Interns.info(OFFLINE_REGION_COUNT_NAME, OFFLINE_REGION_COUNT_DESC),
            regionNumberPair.getSecond())
        .tag(Interns.info(LIVE_REGION_SERVERS_NAME, LIVE_REGION_SERVERS_DESC),
              masterWrapper.getRegionServers())
        .addGauge(Interns.info(NUM_REGION_SERVERS_NAME,
            NUMBER_OF_REGION_SERVERS_DESC), masterWrapper.getNumRegionServers())
        .tag(Interns.info(DEAD_REGION_SERVERS_NAME, DEAD_REGION_SERVERS_DESC),
              masterWrapper.getDeadRegionServers())
        .addGauge(Interns.info(NUM_DEAD_REGION_SERVERS_NAME,
            NUMBER_OF_DEAD_REGION_SERVERS_DESC),
            masterWrapper.getNumDeadRegionServers())
        .tag(Interns.info(ZOOKEEPER_QUORUM_NAME, ZOOKEEPER_QUORUM_DESC),
            masterWrapper.getZookeeperQuorum())
        .tag(Interns.info(SERVER_NAME_NAME, SERVER_NAME_DESC), masterWrapper.getServerName())
        .tag(Interns.info(CLUSTER_ID_NAME, CLUSTER_ID_DESC), masterWrapper.getClusterId())
        .tag(Interns.info(IS_ACTIVE_MASTER_NAME, IS_ACTIVE_MASTER_DESC),
            String.valueOf(masterWrapper.getIsActiveMaster()));
  }

  metricsRegistry.snapshot(metricsRecordBuilder, all);
  if(metricsAdapter != null) {
    metricsAdapter.snapshotAllMetrics(registry, metricsRecordBuilder);
  }
}
 
Example #14
Source File: IndexSplitTransaction.java    From phoenix with Apache License 2.0 3 votes vote down vote up
/**
 * Run the transaction.
 * @param server Hosting server instance.  Can be null when testing (won't try
 * and update in zk if a null server)
 * @param services Used to online/offline regions.
 * @throws IOException If thrown, transaction failed.
 *          Call {@link #rollback(Server, RegionServerServices)}
 * @return Regions created
 * @throws IOException
 * @see #rollback(Server, RegionServerServices)
 */
public PairOfSameType<HRegion> execute(final Server server,
    final RegionServerServices services)
throws IOException {
  PairOfSameType<HRegion> regions = createDaughters(server, services);
  if (this.parent.getCoprocessorHost() != null) {
    this.parent.getCoprocessorHost().preSplitAfterPONR();
  }
  return stepsAfterPONR(server, services, regions);
}
 
Example #15
Source File: HBCKMetaTableAccessor.java    From hbase-operator-tools with Apache License 2.0 2 votes vote down vote up
/**
 * Returns the daughter regions by reading the corresponding columns of the catalog table
 * Result.
 * (Copied from MetaTableAccessor)
 * @param data a Result object from the catalog table scan
 * @return pair of RegionInfo or PairOfSameType(null, null) if region is not a split parent
 */
public static PairOfSameType<RegionInfo> getDaughterRegions(Result data) {
  RegionInfo splitA = getRegionInfo(data, HConstants.SPLITA_QUALIFIER);
  RegionInfo splitB = getRegionInfo(data, HConstants.SPLITB_QUALIFIER);
  return new PairOfSameType<>(splitA, splitB);
}
 
Example #16
Source File: MetricsMasterWrapper.java    From hbase with Apache License 2.0 2 votes vote down vote up
/**
 * Get the online and offline region counts
 *
 * @return pair of count for online regions and offline regions
 */
PairOfSameType<Integer> getRegionCounts();