org.apache.hadoop.hbase.RegionMetrics Java Examples

The following examples show how to use org.apache.hadoop.hbase.RegionMetrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HbaseSessions.java    From hugegraph with Apache License 2.0 6 votes vote down vote up
public long storeSize(String table) throws IOException {
    long total = 0;
    try (Admin admin = this.hbase.getAdmin()) {
        for (ServerName rs : admin.getRegionServers()) {
            // NOTE: we can use getLoad() before hbase 2.0
            //ServerLoad load = admin.getClusterStatus().getLoad(rs);
            //total += load.getStorefileSizeMB() * Bytes.MB;
            //total += load.getMemStoreSizeMB() * Bytes.MB;
            TableName tableName = TableName.valueOf(this.namespace, table);
            for (RegionMetrics m : admin.getRegionMetrics(rs, tableName)) {
                total += m.getStoreFileSize().get(Size.Unit.BYTE);
                total += m.getMemStoreSize().get(Size.Unit.BYTE);
            }
        }
    }
    return total;
}
 
Example #2
Source File: TestUtils.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static RegionMetrics createRegionMetrics(String regionName, long readRequestCount,
  long filteredReadRequestCount, long writeRequestCount, Size storeFileSize,
  Size uncompressedStoreFileSize, int storeFileCount, Size memStoreSize, float locality,
  long compactedCellCount, long compactingCellCount, String lastMajorCompactionTime) {

  FastDateFormat df = FastDateFormat.getInstance("yyyy-MM-dd HH:mm:ss");
  try {
    return RegionMetricsBuilder.newBuilder(Bytes.toBytes(regionName))
      .setReadRequestCount(readRequestCount)
      .setFilteredReadRequestCount(filteredReadRequestCount)
      .setWriteRequestCount(writeRequestCount).setStoreFileSize(storeFileSize)
      .setUncompressedStoreFileSize(uncompressedStoreFileSize).setStoreFileCount(storeFileCount)
      .setMemStoreSize(memStoreSize).setDataLocality(locality)
      .setCompactedCellCount(compactedCellCount).setCompactingCellCount(compactingCellCount)
      .setLastMajorCompactionTimestamp(df.parse(lastMajorCompactionTime).getTime()).build();
  } catch (ParseException e) {
    throw new IllegalArgumentException(e);
  }
}
 
Example #3
Source File: TestRegionServerReadRequestMetrics.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static long getReadRequest(ServerMetrics serverMetrics, RegionMetrics regionMetrics,
    Metric metric) {
  switch (metric) {
    case REGION_READ:
      return regionMetrics.getReadRequestCount();
    case SERVER_READ:
      return serverMetrics.getRegionMetrics().get(regionMetrics.getRegionName())
          .getReadRequestCount();
    case FILTERED_REGION_READ:
      return regionMetrics.getFilteredReadRequestCount();
    case FILTERED_SERVER_READ:
      return serverMetrics.getRegionMetrics().get(regionMetrics.getRegionName())
          .getFilteredReadRequestCount();
    default:
      throw new IllegalStateException();
  }
}
 
Example #4
Source File: TestStochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
private ServerMetrics mockServerMetricsWithCpRequests(ServerName server,
                                                      List<RegionInfo> regionsOnServer,
                                                      long cpRequestCount) {
  ServerMetrics serverMetrics = mock(ServerMetrics.class);
  Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(RegionInfo info : regionsOnServer){
    RegionMetrics rl = mock(RegionMetrics.class);
    when(rl.getReadRequestCount()).thenReturn(0L);
    when(rl.getCpRequestCount()).thenReturn(cpRequestCount);
    when(rl.getWriteRequestCount()).thenReturn(0L);
    when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
    when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
    regionLoadMap.put(info.getRegionName(), rl);
  }
  when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
  return serverMetrics;
}
 
Example #5
Source File: TestClassLoading.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * return the subset of all regionservers
 * (actually returns set of ServerLoads)
 * which host some region in a given table.
 * used by assertAllRegionServers() below to
 * test reporting of loaded coprocessors.
 * @param tableName : given table.
 * @return subset of all servers.
 */
Map<ServerName, ServerMetrics> serversForTable(String tableName) {
  Map<ServerName, ServerMetrics> serverLoadHashMap = new HashMap<>();
  for(Map.Entry<ServerName, ServerMetrics> server:
      TEST_UTIL.getMiniHBaseCluster().getMaster().getServerManager().
          getOnlineServers().entrySet()) {
    for(Map.Entry<byte[], RegionMetrics> region:
        server.getValue().getRegionMetrics().entrySet()) {
      if (region.getValue().getNameAsString().equals(tableName)) {
        // this server hosts a region of tableName: add this server..
        serverLoadHashMap.put(server.getKey(),server.getValue());
        // .. and skip the rest of the regions that it hosts.
        break;
      }
    }
  }
  return serverLoadHashMap;
}
 
Example #6
Source File: RegionsRecoveryChore.java    From hbase with Apache License 2.0 6 votes vote down vote up
private Map<TableName, List<byte[]>> getTableToRegionsByRefCount(
    final Map<ServerName, ServerMetrics> serverMetricsMap) {

  final Map<TableName, List<byte[]>> tableToReopenRegionsMap = new HashMap<>();
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      // For each region, each compacted store file can have different ref counts
      // We need to find maximum of all such ref counts and if that max count of compacted
      // store files is beyond a threshold value, we should reopen the region.
      // Here, we take max ref count of all compacted store files and not the cumulative
      // count of all compacted store files
      final int maxCompactedStoreFileRefCount = regionMetrics
        .getMaxCompactedStoreFileRefCount();

      if (maxCompactedStoreFileRefCount > storeFileRefCountThreshold) {
        final byte[] regionName = regionMetrics.getRegionName();
        prepareTableToReopenRegionsMap(tableToReopenRegionsMap, regionName,
          maxCompactedStoreFileRefCount);
      }
    }
  }
  return tableToReopenRegionsMap;

}
 
Example #7
Source File: StochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Store the current region loads.
 */
private synchronized void updateRegionLoad() {
  // We create a new hashmap so that regions that are no longer there are removed.
  // However we temporarily need the old loads so we can use them to keep the rolling average.
  Map<String, Deque<BalancerRegionLoad>> oldLoads = loads;
  loads = new HashMap<>();

  clusterStatus.getLiveServerMetrics().forEach((ServerName sn, ServerMetrics sm) -> {
    sm.getRegionMetrics().forEach((byte[] regionName, RegionMetrics rm) -> {
      String regionNameAsString = RegionInfo.getRegionNameAsString(regionName);
      Deque<BalancerRegionLoad> rLoads = oldLoads.get(regionNameAsString);
      if (rLoads == null) {
        rLoads = new ArrayDeque<>(numRegionLoadsToRemember + 1);
      } else if (rLoads.size() >= numRegionLoadsToRemember) {
        rLoads.remove();
      }
      rLoads.add(new BalancerRegionLoad(rm));
      loads.put(regionNameAsString, rLoads);
    });
  });

  for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
    cost.setLoads(loads);
  }
}
 
Example #8
Source File: TestRSGroupBasedLoadBalancerWithStochasticLoadBalancerAsInternal.java    From hbase with Apache License 2.0 6 votes vote down vote up
private ServerMetrics mockServerMetricsWithReadRequests(ServerName server,
    List<RegionInfo> regionsOnServer, long readRequestCount) {
  ServerMetrics serverMetrics = mock(ServerMetrics.class);
  Map<byte[], RegionMetrics> regionLoadMap = new TreeMap<>(Bytes.BYTES_COMPARATOR);
  for(RegionInfo info : regionsOnServer){
    RegionMetrics rl = mock(RegionMetrics.class);
    when(rl.getReadRequestCount()).thenReturn(readRequestCount);
    when(rl.getCpRequestCount()).thenReturn(0L);
    when(rl.getWriteRequestCount()).thenReturn(0L);
    when(rl.getMemStoreSize()).thenReturn(Size.ZERO);
    when(rl.getStoreFileSize()).thenReturn(Size.ZERO);
    regionLoadMap.put(info.getRegionName(), rl);
  }
  when(serverMetrics.getRegionMetrics()).thenReturn(regionLoadMap);
  return serverMetrics;
}
 
Example #9
Source File: TestAsyncClusterAdminApi.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void compareRegionLoads(Collection<RegionMetrics> regionLoadCluster,
    Collection<RegionMetrics> regionLoads) {

  assertEquals("No of regionLoads from clusterStatus and regionloads from RS doesn't match",
    regionLoadCluster.size(), regionLoads.size());

  for (RegionMetrics loadCluster : regionLoadCluster) {
    boolean matched = false;
    for (RegionMetrics load : regionLoads) {
      if (Bytes.equals(loadCluster.getRegionName(), load.getRegionName())) {
        matched = true;
        continue;
      }
    }
    assertTrue("The contents of region load from cluster and server should match", matched);
  }
}
 
Example #10
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<List<RegionMetrics>> getRegionMetrics(ServerName serverName,
    TableName tableName) {
  Preconditions.checkNotNull(tableName,
    "tableName is null. If you don't specify a tableName, use getRegionLoads() instead");
  return getRegionMetrics(RequestConverter.buildGetRegionLoadRequest(tableName), serverName);
}
 
Example #11
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithLessThreshold() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of only 1 table
  Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen only 1 region with refCount > 400
  Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(1))
    .getRegionInfo(Mockito.any());
}
 
Example #12
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithoutStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.unset("hbase.regions.recovery.store.file.ref.count");
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that by default the feature is turned off so no regions
  // should be reopened
  Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());

  // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
  Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(0))
    .getRegionInfo(Mockito.any());
}
 
Example #13
Source File: TestRegionServerReadRequestMetrics.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void testReadRequests(byte[] regionName, int expectedReadRequests) throws Exception {
  for (ServerName serverName : serverNames) {
    ServerMetrics serverMetrics = admin.getClusterMetrics(
      EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().get(serverName);
    Map<byte[], RegionMetrics> regionMetrics = serverMetrics.getRegionMetrics();
    RegionMetrics regionMetric = regionMetrics.get(regionName);
    if (regionMetric != null) {
      LOG.debug("server read request is "
          + serverMetrics.getRegionMetrics().get(regionName).getReadRequestCount()
          + ", region read request is " + regionMetric.getReadRequestCount());
      assertEquals(3, serverMetrics.getRegionMetrics().get(regionName).getReadRequestCount());
      assertEquals(3, regionMetric.getReadRequestCount());
    }
  }
}
 
Example #14
Source File: RegionSizeCalculator.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void init(RegionLocator regionLocator, Admin admin)
    throws IOException {
  if (!enabled(admin.getConfiguration())) {
    LOG.info("Region size calculation disabled.");
    return;
  }

  if (regionLocator.getName().isSystemTable()) {
    LOG.info("Region size calculation disabled for system tables.");
    return;
  }

  LOG.info("Calculating region sizes for table \"" + regionLocator.getName() + "\".");

  // Get the servers which host regions of the table
  Set<ServerName> tableServers = getRegionServersOfTable(regionLocator);

  for (ServerName tableServerName : tableServers) {
    for (RegionMetrics regionLoad : admin.getRegionMetrics(
      tableServerName,regionLocator.getName())) {

      byte[] regionId = regionLoad.getRegionName();
      long regionSizeBytes
        = ((long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE)) * MEGABYTE;

      sizeMap.put(regionId, regionSizeBytes);

      if (LOG.isDebugEnabled()) {
        LOG.debug("Region " + regionLoad.getNameAsString() + " has size " + regionSizeBytes);
      }
    }
  }
  LOG.debug("Region sizes calculated");
}
 
Example #15
Source File: TestRegionSizeCalculator.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Creates mock of region with given name and size.
 *
 * @param  fileSizeMb number of megabytes occupied by region in file store in megabytes
 * */
private RegionMetrics mockRegion(String regionName, int fileSizeMb) {
  RegionMetrics region = Mockito.mock(RegionMetrics.class);
  when(region.getRegionName()).thenReturn(Bytes.toBytes(regionName));
  when(region.getNameAsString()).thenReturn(regionName);
  when(region.getStoreFileSize()).thenReturn(new Size(fileSizeMb, Size.Unit.MEGABYTE));
  return region;
}
 
Example #16
Source File: HbaseMetrics.java    From hugegraph with Apache License 2.0 5 votes vote down vote up
private static Map<String, Object> formatMetrics(
                                   List<RegionMetrics> regions) {
    Map<String, Object> metrics = InsertionOrderUtil.newMap();
    for (RegionMetrics region : regions) {
        metrics.put(region.getNameAsString(), formatMetrics(region));
    }
    return metrics;
}
 
Example #17
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<List<RegionMetrics>> getRegionMetrics(GetRegionLoadRequest request,
    ServerName serverName) {
  return this.<List<RegionMetrics>> newAdminCaller()
      .action((controller, stub) -> this
          .<GetRegionLoadRequest, GetRegionLoadResponse, List<RegionMetrics>>
            adminCall(controller, stub, request, (s, c, req, done) ->
              s.getRegionLoad(controller, req, done), RegionMetricsBuilder::toRegionMetrics))
      .serverName(serverName).call();
}
 
Example #18
Source File: RegionModeStrategy.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  List<Record> ret = new ArrayList<>();
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    long lastReportTimestamp = sm.getLastReportTimestamp();
    for (RegionMetrics rm : sm.getRegionMetrics().values()) {
      ret.add(createRecord(sm, rm, lastReportTimestamp));
    }
  }
  return ret;
}
 
Example #19
Source File: TestUtils.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static ServerMetrics createServerMetrics(ServerName serverName, long reportTimestamp,
  Size usedHeapSize, Size maxHeapSize, long requestCountPerSecond,
  List<RegionMetrics> regionMetricsList, List<UserMetrics> userMetricsList) {

  return ServerMetricsBuilder.newBuilder(serverName)
    .setReportTimestamp(reportTimestamp)
    .setUsedHeapSize(usedHeapSize)
    .setMaxHeapSize(maxHeapSize)
    .setRequestCountPerSecond(requestCountPerSecond)
    .setRegionMetrics(regionMetricsList)
    .setUserMetrics(userMetricsList).build();
}
 
Example #20
Source File: HServerLoad.java    From spliceengine with GNU Affero General Public License v3.0 5 votes vote down vote up
@Override
public Set<PartitionLoad> getPartitionLoads(){
    Map<byte[], RegionMetrics> regionsLoad=load.getRegionMetrics();
    Set<PartitionLoad> loads = new HashSet<>(regionsLoad.size(),0.9f);
    for(Map.Entry<byte[], RegionMetrics> regionLoad:regionsLoad.entrySet()){
        String name = Bytes.toString(regionLoad.getKey());
        RegionMetrics rm = regionLoad.getValue();
        PartitionLoad pl = new HPartitionLoad(name, (long) rm.getStoreFileSize().get(Size.Unit.BYTE), (long) rm.getMemStoreSize().get(Size.Unit.BYTE));
        loads.add(pl);
    }
    return loads;
}
 
Example #21
Source File: TestSimpleRegionNormalizer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("MockitoCast")
private void setupMocksForNormalizer(Map<byte[], Integer> regionSizes,
  List<RegionInfo> regionInfoList) {
  masterServices = Mockito.mock(MasterServices.class, RETURNS_DEEP_STUBS);

  // for simplicity all regions are assumed to be on one server; doesn't matter to us
  ServerName sn = ServerName.valueOf("localhost", 0, 0L);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionsOfTable(any())).thenReturn(regionInfoList);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionServerOfRegion(any())).thenReturn(sn);
  when(masterServices.getAssignmentManager().getRegionStates()
    .getRegionState(any(RegionInfo.class))).thenReturn(
      RegionState.createForTesting(null, RegionState.State.OPEN));

  for (Map.Entry<byte[], Integer> region : regionSizes.entrySet()) {
    RegionMetrics regionLoad = Mockito.mock(RegionMetrics.class);
    when(regionLoad.getRegionName()).thenReturn(region.getKey());
    when(regionLoad.getStoreFileSize())
      .thenReturn(new Size(region.getValue(), Size.Unit.MEGABYTE));

    // this is possibly broken with jdk9, unclear if false positive or not
    // suppress it for now, fix it when we get to running tests on 9
    // see: http://errorprone.info/bugpattern/MockitoCast
    when((Object) masterServices.getServerManager().getLoad(sn)
      .getRegionMetrics().get(region.getKey())).thenReturn(regionLoad);
  }

  when(masterServices.isSplitOrMergeEnabled(any())).thenReturn(true);

  normalizer = new SimpleRegionNormalizer();
  normalizer.setConf(conf);
  normalizer.setMasterServices(masterServices);
}
 
Example #22
Source File: HbaseMetrics.java    From hugegraph with Apache License 2.0 5 votes vote down vote up
private static Map<String, Object> formatMetrics(
                                   ServerMetrics serverMetrics,
                                   List<RegionMetrics> regions) {
    Map<String, Object> metrics = InsertionOrderUtil.newMap();
    metrics.put("max_heap_size",
                serverMetrics.getMaxHeapSize().get(Size.Unit.MEGABYTE));
    metrics.put("used_heap_size",
                serverMetrics.getUsedHeapSize().get(Size.Unit.MEGABYTE));
    metrics.put("heap_size_unit", "MB");
    metrics.put("request_count", serverMetrics.getRequestCount());
    metrics.put("request_count_per_second",
                serverMetrics.getRequestCountPerSecond());
    metrics.put("regions", formatMetrics(regions));
    return metrics;
}
 
Example #23
Source File: HbaseMetrics.java    From hugegraph with Apache License 2.0 5 votes vote down vote up
private static Map<String, Object> formatMetrics(RegionMetrics region) {
    Map<String, Object> metrics = InsertionOrderUtil.newMap();
    metrics.put("mem_store_size",
                region.getMemStoreSize().get(Size.Unit.MEGABYTE));
    metrics.put("file_store_size",
                region.getStoreFileSize().get(Size.Unit.MEGABYTE));
    metrics.put("store_size_unit", "MB");
    return metrics;
}
 
Example #24
Source File: BalancerRegionLoad.java    From hbase with Apache License 2.0 5 votes vote down vote up
BalancerRegionLoad(RegionMetrics regionMetrics) {
  readRequestsCount = regionMetrics.getReadRequestCount();
  cpRequestsCount = regionMetrics.getCpRequestCount();
  writeRequestsCount = regionMetrics.getWriteRequestCount();
  memStoreSizeMB = (int) regionMetrics.getMemStoreSize().get(Size.Unit.MEGABYTE);
  storefileSizeMB = (int) regionMetrics.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #25
Source File: SimpleRegionNormalizer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * @return size of region in MB and if region is not found than -1
 */
private long getRegionSizeMB(RegionInfo hri) {
  ServerName sn =
    masterServices.getAssignmentManager().getRegionStates().getRegionServerOfRegion(hri);
  RegionMetrics regionLoad =
    masterServices.getServerManager().getLoad(sn).getRegionMetrics().get(hri.getRegionName());
  if (regionLoad == null) {
    LOG.debug("{} was not found in RegionsLoad", hri.getRegionNameAsString());
    return -1;
  }
  return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #26
Source File: ServerManager.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Updates last flushed sequence Ids for the regions on server sn
 * @param sn
 * @param hsl
 */
private void updateLastFlushedSequenceIds(ServerName sn, ServerMetrics hsl) {
  for (Entry<byte[], RegionMetrics> entry : hsl.getRegionMetrics().entrySet()) {
    byte[] encodedRegionName = Bytes.toBytes(RegionInfo.encodeRegionName(entry.getKey()));
    Long existingValue = flushedSequenceIdByRegion.get(encodedRegionName);
    long l = entry.getValue().getCompletedSequenceId();
    // Don't let smaller sequence ids override greater sequence ids.
    if (LOG.isTraceEnabled()) {
      LOG.trace(Bytes.toString(encodedRegionName) + ", existingValue=" + existingValue +
        ", completeSequenceId=" + l);
    }
    if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue)) {
      flushedSequenceIdByRegion.put(encodedRegionName, l);
    } else if (l != HConstants.NO_SEQNUM && l < existingValue) {
      LOG.warn("RegionServer " + sn + " indicates a last flushed sequence id ("
          + l + ") that is less than the previous last flushed sequence id ("
          + existingValue + ") for region " + Bytes.toString(entry.getKey()) + " Ignoring.");
    }
    ConcurrentNavigableMap<byte[], Long> storeFlushedSequenceId =
        computeIfAbsent(storeFlushedSequenceIdsByRegion, encodedRegionName,
          () -> new ConcurrentSkipListMap<>(Bytes.BYTES_COMPARATOR));
    for (Entry<byte[], Long> storeSeqId : entry.getValue().getStoreSequenceId().entrySet()) {
      byte[] family = storeSeqId.getKey();
      existingValue = storeFlushedSequenceId.get(family);
      l = storeSeqId.getValue();
      if (LOG.isTraceEnabled()) {
        LOG.trace(Bytes.toString(encodedRegionName) + ", family=" + Bytes.toString(family) +
          ", existingValue=" + existingValue + ", completeSequenceId=" + l);
      }
      // Don't let smaller sequence ids override greater sequence ids.
      if (existingValue == null || (l != HConstants.NO_SEQNUM && l > existingValue.longValue())) {
        storeFlushedSequenceId.put(family, l);
      }
    }
  }
}
 
Example #27
Source File: TestAsyncClusterAdminApi.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void checkRegionsAndRegionLoads(Collection<RegionInfo> regions,
    Collection<RegionMetrics> regionLoads) {

  assertEquals("No of regions and regionloads doesn't match", regions.size(), regionLoads.size());

  Map<byte[], RegionMetrics> regionLoadMap = Maps.newTreeMap(Bytes.BYTES_COMPARATOR);
  for (RegionMetrics regionLoad : regionLoads) {
    regionLoadMap.put(regionLoad.getRegionName(), regionLoad);
  }
  for (RegionInfo info : regions) {
    assertTrue("Region not in regionLoadMap region:" + info.getRegionNameAsString()
        + " regionMap: " + regionLoadMap, regionLoadMap.containsKey(info.getRegionName()));
  }
}
 
Example #28
Source File: TestSimpleRegionNormalizerOnCluster.java    From hbase with Apache License 2.0 5 votes vote down vote up
private static double getRegionSizeMB(final MasterServices masterServices,
  final RegionInfo regionInfo) {
  final ServerName sn = masterServices.getAssignmentManager()
    .getRegionStates()
    .getRegionServerOfRegion(regionInfo);
  final RegionMetrics regionLoad = masterServices.getServerManager()
    .getLoad(sn)
    .getRegionMetrics()
    .get(regionInfo.getRegionName());
  if (regionLoad == null) {
    LOG.debug("{} was not found in RegionsLoad", regionInfo.getRegionNameAsString());
    return -1;
  }
  return regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE);
}
 
Example #29
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of 2 tables
  Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen total 3 regions that have refCount > 300
  Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(3))
    .getRegionInfo(Mockito.any());
}
 
Example #30
Source File: TestFromClientSide3.java    From hbase with Apache License 2.0 5 votes vote down vote up
private int getStoreFileCount(Admin admin, ServerName serverName, RegionInfo region)
    throws IOException {
  for (RegionMetrics metrics : admin.getRegionMetrics(serverName, region.getTable())) {
    if (Bytes.equals(region.getRegionName(), metrics.getRegionName())) {
      return metrics.getStoreFileCount();
    }
  }
  return 0;
}