Java Code Examples for org.apache.hadoop.hbase.ClusterMetrics

The following examples show how to use org.apache.hadoop.hbase.ClusterMetrics. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: hbase-operator-tools   Source File: HBCK2.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Check for HBCK support.
 * Expects created connection.
 * @param supportedVersions list of zero or more supported versions.
 */
void checkHBCKSupport(ClusterConnection connection, String cmd, String ... supportedVersions)
    throws IOException {
  if (skipCheck) {
    LOG.info("Skipped {} command version check; 'skip' set", cmd);
    return;
  }
  try (Admin admin = connection.getAdmin()) {
    String serverVersion = admin.
        getClusterMetrics(EnumSet.of(ClusterMetrics.Option.HBASE_VERSION)).getHBaseVersion();
    String [] thresholdVersions = supportedVersions == null || supportedVersions.length == 0?
        MINIMUM_HBCK2_VERSION: supportedVersions;
    boolean supported = Version.check(serverVersion, thresholdVersions);
    if (!supported) {
      throw new UnsupportedOperationException(cmd + " not supported on server version=" +
          serverVersion + "; needs at least a server that matches or exceeds " +
          Arrays.toString(thresholdVersions));
    }
  }
}
 
Example 2
Source Project: hbase   Source File: StochasticLoadBalancer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void setClusterMetrics(ClusterMetrics st) {
  super.setClusterMetrics(st);
  updateRegionLoad();
  for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
    cost.setClusterMetrics(st);
  }

  // update metrics size
  try {
    // by-table or ensemble mode
    int tablesCount = isByTable ? services.getTableDescriptors().getAll().size() : 1;
    int functionsCount = getCostFunctionNames().length;

    updateMetricsSize(tablesCount * (functionsCount + 1)); // +1 for overall
  } catch (Exception e) {
    LOG.error("failed to get the size of all tables", e);
  }
}
 
Example 3
Source Project: hbase   Source File: TestReplicationStatusAfterLagging.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testReplicationStatusAfterLagging() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  // add some values to cluster 1
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  UTIL2.startMiniHBaseCluster();
  Thread.sleep(10000);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertTrue(loadSource.getTimestampOfLastShippedOp() > 0);
  assertEquals(0, loadSource.getReplicationLag());
}
 
Example 4
@Test
public void testReplicationStatusSourceStartedTargetStoppedNewOp() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  // add some values to source cluster
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  Thread.sleep(10000);
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertTrue(loadSource.getReplicationLag() > 0);
  assertFalse(loadSource.isRecovered());
}
 
Example 5
@Test
public void testReplicationStatusSourceStartedTargetStoppedNoOps() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  Thread.sleep(10000);
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertFalse(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertEquals(0, loadSource.getReplicationLag());
  assertFalse(loadSource.isRecovered());
}
 
Example 6
Source Project: hbase   Source File: BaseTestHBaseFsck.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get region info from local cluster.
 */
Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
  ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
  Map<ServerName, List<String>> mm = new HashMap<>();
  for (ServerName hsi : regionServers) {
    // list all online regions from this region server
    List<RegionInfo> regions = admin.getRegions(hsi);
    List<String> regionNames = new ArrayList<>(regions.size());
    for (RegionInfo hri : regions) {
      regionNames.add(hri.getRegionNameAsString());
    }
    mm.put(hsi, regionNames);
  }
  return mm;
}
 
Example 7
Source Project: hbase   Source File: Action.java    License: Apache License 2.0 6 votes vote down vote up
/** Returns current region servers - active master */
protected ServerName[] getCurrentServers() throws IOException {
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  Collection<ServerName> regionServers = clusterStatus.getLiveServerMetrics().keySet();
  int count = regionServers.size();
  if (count <= 0) {
    return new ServerName [] {};
  }
  ServerName master = clusterStatus.getMasterName();
  Set<ServerName> masters = new HashSet<>();
  masters.add(master);
  masters.addAll(clusterStatus.getBackupMasterNames());
  ArrayList<ServerName> tmp = new ArrayList<>(count);
  tmp.addAll(regionServers);
  tmp.removeAll(masters);

  if(skipMetaRS){
    ServerName metaServer = cluster.getServerHoldingMeta();
    tmp.remove(metaServer);
  }

  return tmp.toArray(new ServerName[0]);
}
 
Example 8
Source Project: hbase   Source File: RestartRsHoldingMetaAction.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void perform() throws Exception {
  getLogger().info("Performing action: Restart region server holding META");
  ServerName server = cluster.getServerHoldingMeta();
  if (server == null) {
    getLogger().warn("No server is holding hbase:meta right now.");
    return;
  }
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  if (server.equals(clusterStatus.getMasterName())) {
    // Master holds the meta, so restart the master.
    restartMaster(server, sleepTime);
  } else {
    restartRs(server, sleepTime);
  }
}
 
Example 9
Source Project: hbase   Source File: RegionServerModeStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);
  // Aggregation by LONG_REGION_SERVER field
  Map<String, Record> retMap =
      ModeStrategyUtils.aggregateRecords(records, Field.LONG_REGION_SERVER).stream()
          .collect(Collectors.toMap(r -> r.get(Field.LONG_REGION_SERVER).asString(), r -> r));

  // Add USED_HEAP_SIZE field and MAX_HEAP_SIZE field
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    Record record = retMap.get(sm.getServerName().getServerName());
    if (record == null) {
      continue;
    }

    Record newRecord = Record.builder().putAll(record)
      .put(Field.USED_HEAP_SIZE, sm.getUsedHeapSize())
      .put(Field.MAX_HEAP_SIZE, sm.getMaxHeapSize()).build();

    retMap.put(sm.getServerName().getServerName(), newRecord);
  }

  return new ArrayList<>(retMap.values());
}
 
Example 10
Source Project: hbase   Source File: TableModeStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);

  // Aggregation by NAMESPACE field and TABLE field
  return records.stream()
    .collect(Collectors.groupingBy(r -> {
      String namespace = r.get(Field.NAMESPACE).asString();
      String table = r.get(Field.TABLE).asString();
      return TableName.valueOf(namespace, table);
    }))
    .entrySet().stream()
    .flatMap(
      e -> e.getValue().stream()
        .reduce(Record::combine)
        .map(Stream::of)
        .orElse(Stream.empty()))
    .collect(Collectors.toList());
}
 
Example 11
Source Project: hbase   Source File: TopScreenModel.java    License: Apache License 2.0 6 votes vote down vote up
private void refreshSummary(ClusterMetrics clusterMetrics) {
  String currentTime = ISO_8601_EXTENDED_TIME_FORMAT
    .format(System.currentTimeMillis());
  String version = clusterMetrics.getHBaseVersion();
  String clusterId = clusterMetrics.getClusterId();
  int liveServers = clusterMetrics.getLiveServerMetrics().size();
  int deadServers = clusterMetrics.getDeadServerNames().size();
  int regionCount = clusterMetrics.getRegionCount();
  int ritCount = clusterMetrics.getRegionStatesInTransition().size();
  double averageLoad = clusterMetrics.getAverageLoad();
  long aggregateRequestPerSecond = clusterMetrics.getLiveServerMetrics().entrySet().stream()
    .mapToLong(e -> e.getValue().getRequestCountPerSecond()).sum();

  summary = new Summary(currentTime, version, clusterId, liveServers + deadServers,
    liveServers, deadServers, regionCount, ritCount, averageLoad, aggregateRequestPerSecond);
}
 
Example 12
Source Project: hbase   Source File: RegionLocationFinder.java    License: Apache License 2.0 5 votes vote down vote up
public void setClusterMetrics(ClusterMetrics status) {
  long currentTime = EnvironmentEdgeManager.currentTime();
  this.status = status;
  if (currentTime > lastFullRefresh + (CACHE_TIME / 2)) {
    // Only count the refresh if it includes user tables ( eg more than meta and namespace ).
    lastFullRefresh = scheduleFullRefresh()?currentTime:lastFullRefresh;
  }

}
 
Example 13
Source Project: hbase   Source File: BaseLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public synchronized void setClusterMetrics(ClusterMetrics st) {
  this.clusterStatus = st;
  if (useRegionFinder) {
    regionFinder.setClusterMetrics(st);
  }
}
 
Example 14
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
public void postGetClusterMetrics(ClusterMetrics status) throws IOException {
  execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
    @Override
    public void call(MasterObserver observer) throws IOException {
      observer.postGetClusterMetrics(this, status);
    }
  });
}
 
Example 15
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 5 votes vote down vote up
public ClusterMetrics getClusterMetrics(EnumSet<Option> options) throws IOException {
  if (cpHost != null) {
    cpHost.preGetClusterMetrics();
  }
  ClusterMetrics status = getClusterMetricsWithoutCoprocessor(options);
  if (cpHost != null) {
    cpHost.postGetClusterMetrics(status);
  }
  return status;
}
 
Example 16
Source Project: hbase   Source File: RSGroupBasedLoadBalancer.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void setClusterMetrics(ClusterMetrics sm) {
  this.clusterStatus = sm;
  if (internalBalancer != null) {
    internalBalancer.setClusterMetrics(sm);
  }
}
 
Example 17
Source Project: hbase   Source File: TestReplicationStatusSink.java    License: Apache License 2.0 5 votes vote down vote up
private ReplicationLoadSink getLatestSinkMetric(Admin admin, ServerName server)
    throws IOException {
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
  ServerMetrics sm = metrics.getLiveServerMetrics().get(server);
  return sm.getReplicationLoadSink();
}
 
Example 18
Source Project: hbase   Source File: TestReplicationStatus.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Wait until Master shows metrics counts for ReplicationLoadSourceList that are
 * greater than <code>greaterThan</code> for <code>serverName</code> before
 * returning. We want to avoid case where RS hasn't yet updated Master before
 * allowing test proceed.
 * @param greaterThan size of replicationLoadSourceList must be greater before we proceed
 */
private List<ReplicationLoadSource> waitOnMetricsReport(int greaterThan, ServerName serverName)
    throws IOException {
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> list =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  while(list.size() <= greaterThan) {
    Threads.sleep(1000);
  }
  return list;
}
 
Example 19
Source Project: hbase   Source File: TestMasterReplication.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Tests the replication scenario 0 -> 0. By default
 * {@link BaseReplicationEndpoint#canReplicateToSameCluster()} returns false, so the
 * ReplicationSource should terminate, and no further logs should get enqueued
 */
@Test
public void testLoopedReplication() throws Exception {
  LOG.info("testLoopedReplication");
  startMiniClusters(1);
  createTableOnClusters(table);
  addPeer("1", 0, 0);
  Thread.sleep(SLEEP_TIME);

  // wait for source to terminate
  final ServerName rsName = utilities[0].getHBaseCluster().getRegionServer(0).getServerName();
  Waiter.waitFor(baseConfiguration, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      ClusterMetrics clusterStatus = utilities[0].getAdmin()
          .getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
      ServerMetrics serverLoad = clusterStatus.getLiveServerMetrics().get(rsName);
      List<ReplicationLoadSource> replicationLoadSourceList =
          serverLoad.getReplicationLoadSourceList();
      return replicationLoadSourceList.isEmpty();
    }
  });

  Table[] htables = getHTablesOnClusters(tableName);
  putAndWait(row, famName, htables[0], htables[0]);
  rollWALAndWait(utilities[0], table.getTableName(), row);
  ZKWatcher zkw = utilities[0].getZooKeeperWatcher();
  String queuesZnode = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode,
    ZNodePaths.joinZNode("replication", "rs"));
  List<String> listChildrenNoWatch =
      ZKUtil.listChildrenNoWatch(zkw, ZNodePaths.joinZNode(queuesZnode, rsName.toString()));
  assertEquals(0, listChildrenNoWatch.size());
}
 
Example 20
@Test
public void testReplicationStatusSourceStartedTargetStoppedWithRecovery() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  // add some values to cluster 1
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  Thread.sleep(10000);
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  Thread.sleep(10000);
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(2, loadSources.size());
  boolean foundRecovery = false;
  boolean foundNormal = false;
  for (ReplicationLoadSource loadSource : loadSources) {
    if (loadSource.isRecovered()) {
      foundRecovery = true;
      assertTrue(loadSource.hasEditsSinceRestart());
      assertEquals(0, loadSource.getTimestampOfLastShippedOp());
      assertTrue(loadSource.getReplicationLag() > 0);
    } else {
      foundNormal = true;
      assertFalse(loadSource.hasEditsSinceRestart());
      assertEquals(0, loadSource.getTimestampOfLastShippedOp());
      assertEquals(0, loadSource.getReplicationLag());
    }
  }
  assertTrue("No normal queue found.", foundNormal);
  assertTrue("No recovery queue found.", foundRecovery);
}
 
Example 21
Source Project: hbase   Source File: TestAsyncAdminMasterSwitch.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testSwitch() throws IOException, InterruptedException {
  assertEquals(TEST_UTIL.getHBaseCluster().getRegionServerThreads().size(),
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME)).join()
      .getServersName().size());
  TEST_UTIL.getMiniHBaseCluster().stopMaster(0).join();
  assertTrue(TEST_UTIL.getMiniHBaseCluster().waitForActiveAndReadyMaster(30000));
  // make sure that we could still call master
  assertEquals(TEST_UTIL.getHBaseCluster().getRegionServerThreads().size(),
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME)).join()
      .getServersName().size());
}
 
Example 22
/**
 * Test that if the master fails, the load balancer maintains its
 * state (running or not) when the next master takes over
 *
 * @throws Exception
 */
@Test
public void testMasterFailoverBalancerPersistence() throws Exception {
  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(3).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

  assertTrue(cluster.waitForActiveAndReadyMaster());
  HMaster active = cluster.getMaster();
  // check that the balancer is on by default for the active master
  ClusterMetrics clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is still running on new master
  clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  // turn off the load balancer
  active.balanceSwitch(false);

  // once more, kill active master and wait for new active master to show up
  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is not running on the new master
  clusterStatus = active.getClusterMetrics();
  assertFalse(clusterStatus.getBalancerOn());

  // Stop the cluster
  TEST_UTIL.shutdownMiniCluster();
}
 
Example 23
Source Project: hbase   Source File: TestRegionsRecoveryChore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of 2 tables
  Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen total 3 regions that have refCount > 300
  Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(3))
    .getRegionInfo(Mockito.any());
}
 
Example 24
Source Project: hbase   Source File: TestRegionsRecoveryChore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithLessThreshold() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of only 1 table
  Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen only 1 region with refCount > 400
  Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(1))
    .getRegionInfo(Mockito.any());
}
 
Example 25
Source Project: hbase   Source File: TestRegionsRecoveryChore.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithoutStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.unset("hbase.regions.recovery.store.file.ref.count");
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that by default the feature is turned off so no regions
  // should be reopened
  Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());

  // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
  Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(0))
    .getRegionInfo(Mockito.any());
}
 
Example 26
Source Project: hbase   Source File: TestRSGroupsBase.java    License: Apache License 2.0 5 votes vote down vote up
protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
  throws IOException {
  Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
  Admin admin = TEST_UTIL.getAdmin();
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
  for (ServerName serverName : metrics.getServersName()) {
    for (RegionInfo region : admin.getRegions(serverName)) {
      TableName tableName = region.getTable();
      map.computeIfAbsent(tableName, k -> new TreeMap<>())
        .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
    }
  }
  return map;
}
 
Example 27
Source Project: hbase   Source File: TestRSGroupsBase.java    License: Apache License 2.0 5 votes vote down vote up
protected int getNumServers() throws IOException {
  ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
  ServerName masterName = status.getMasterName();
  int count = 0;
  for (ServerName sn : status.getLiveServerMetrics().keySet()) {
    if (!sn.equals(masterName)) {
      count++;
    }
  }
  return count;
}
 
Example 28
Source Project: hbase   Source File: IntegrationTestMTTR.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected boolean doAction() throws Exception {
  Admin admin = null;
  try {
    admin = util.getAdmin();
    ClusterMetrics status = admin.getClusterMetrics();
    return status != null;
  } finally {
    if (admin != null) {
      admin.close();
    }
  }
}
 
Example 29
Source Project: hbase   Source File: Action.java    License: Apache License 2.0 5 votes vote down vote up
protected void unbalanceRegions(ClusterMetrics clusterStatus,
    List<ServerName> fromServers, List<ServerName> toServers,
    double fractionOfRegions) throws Exception {
  List<byte[]> victimRegions = new LinkedList<>();
  for (Map.Entry<ServerName, ServerMetrics> entry
    : clusterStatus.getLiveServerMetrics().entrySet()) {
    ServerName sn = entry.getKey();
    ServerMetrics serverLoad = entry.getValue();
    // Ugh.
    List<byte[]> regions = new LinkedList<>(serverLoad.getRegionMetrics().keySet());
    int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
    getLogger().debug("Removing {} regions from {}", victimRegionCount, sn);
    for (int i = 0; i < victimRegionCount; ++i) {
      int victimIx = RandomUtils.nextInt(0, regions.size());
      String regionId = RegionInfo.encodeRegionName(regions.remove(victimIx));
      victimRegions.add(Bytes.toBytes(regionId));
    }
  }

  getLogger().info("Moving {} regions from {} servers to {} different servers",
    victimRegions.size(), fromServers.size(), toServers.size());
  Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
  for (byte[] victimRegion : victimRegions) {
    // Don't keep moving regions if we're
    // trying to stop the monkey.
    if (context.isStopping()) {
      break;
    }
    int targetIx = RandomUtils.nextInt(0, toServers.size());
    admin.move(victimRegion, toServers.get(targetIx));
  }
}
 
Example 30
Source Project: hbase   Source File: UnbalanceRegionsAction.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public void perform() throws Exception {
  getLogger().info("Unbalancing regions");
  ClusterMetrics status = this.cluster.getClusterMetrics();
  List<ServerName> victimServers = new LinkedList<>(status.getLiveServerMetrics().keySet());
  int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size());
  List<ServerName> targetServers = new ArrayList<>(targetServerCount);
  for (int i = 0; i < targetServerCount; ++i) {
    int victimIx = RandomUtils.nextInt(0, victimServers.size());
    targetServers.add(victimServers.remove(victimIx));
  }
  unbalanceRegions(status, victimServers, targetServers, fractionOfRegions);
}