org.apache.hadoop.hbase.ClusterMetrics Java Examples

The following examples show how to use org.apache.hadoop.hbase.ClusterMetrics. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RestartRsHoldingMetaAction.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public void perform() throws Exception {
  getLogger().info("Performing action: Restart region server holding META");
  ServerName server = cluster.getServerHoldingMeta();
  if (server == null) {
    getLogger().warn("No server is holding hbase:meta right now.");
    return;
  }
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  if (server.equals(clusterStatus.getMasterName())) {
    // Master holds the meta, so restart the master.
    restartMaster(server, sleepTime);
  } else {
    restartRs(server, sleepTime);
  }
}
 
Example #2
Source File: Action.java    From hbase with Apache License 2.0 6 votes vote down vote up
/** Returns current region servers - active master */
protected ServerName[] getCurrentServers() throws IOException {
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  Collection<ServerName> regionServers = clusterStatus.getLiveServerMetrics().keySet();
  int count = regionServers.size();
  if (count <= 0) {
    return new ServerName [] {};
  }
  ServerName master = clusterStatus.getMasterName();
  Set<ServerName> masters = new HashSet<>();
  masters.add(master);
  masters.addAll(clusterStatus.getBackupMasterNames());
  ArrayList<ServerName> tmp = new ArrayList<>(count);
  tmp.addAll(regionServers);
  tmp.removeAll(masters);

  if(skipMetaRS){
    ServerName metaServer = cluster.getServerHoldingMeta();
    tmp.remove(metaServer);
  }

  return tmp.toArray(new ServerName[0]);
}
 
Example #3
Source File: BaseTestHBaseFsck.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Get region info from local cluster.
 */
Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
  ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
  Map<ServerName, List<String>> mm = new HashMap<>();
  for (ServerName hsi : regionServers) {
    // list all online regions from this region server
    List<RegionInfo> regions = admin.getRegions(hsi);
    List<String> regionNames = new ArrayList<>(regions.size());
    for (RegionInfo hri : regions) {
      regionNames.add(hri.getRegionNameAsString());
    }
    mm.put(hsi, regionNames);
  }
  return mm;
}
 
Example #4
Source File: RegionServerModeStrategy.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);
  // Aggregation by LONG_REGION_SERVER field
  Map<String, Record> retMap =
      ModeStrategyUtils.aggregateRecords(records, Field.LONG_REGION_SERVER).stream()
          .collect(Collectors.toMap(r -> r.get(Field.LONG_REGION_SERVER).asString(), r -> r));

  // Add USED_HEAP_SIZE field and MAX_HEAP_SIZE field
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    Record record = retMap.get(sm.getServerName().getServerName());
    if (record == null) {
      continue;
    }

    Record newRecord = Record.builder().putAll(record)
      .put(Field.USED_HEAP_SIZE, sm.getUsedHeapSize())
      .put(Field.MAX_HEAP_SIZE, sm.getMaxHeapSize()).build();

    retMap.put(sm.getServerName().getServerName(), newRecord);
  }

  return new ArrayList<>(retMap.values());
}
 
Example #5
Source File: TestReplicationStatusSourceStartedTargetStoppedNoOps.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testReplicationStatusSourceStartedTargetStoppedNoOps() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  Thread.sleep(10000);
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertFalse(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertEquals(0, loadSource.getReplicationLag());
  assertFalse(loadSource.isRecovered());
}
 
Example #6
Source File: TestReplicationStatusSourceStartedTargetStoppedNewOp.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testReplicationStatusSourceStartedTargetStoppedNewOp() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  // add some values to source cluster
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  Thread.sleep(10000);
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertTrue(loadSource.getReplicationLag() > 0);
  assertFalse(loadSource.isRecovered());
}
 
Example #7
Source File: TestReplicationStatusAfterLagging.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testReplicationStatusAfterLagging() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  // add some values to cluster 1
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  UTIL2.startMiniHBaseCluster();
  Thread.sleep(10000);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertTrue(loadSource.getTimestampOfLastShippedOp() > 0);
  assertEquals(0, loadSource.getReplicationLag());
}
 
Example #8
Source File: TableModeStrategy.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);

  // Aggregation by NAMESPACE field and TABLE field
  return records.stream()
    .collect(Collectors.groupingBy(r -> {
      String namespace = r.get(Field.NAMESPACE).asString();
      String table = r.get(Field.TABLE).asString();
      return TableName.valueOf(namespace, table);
    }))
    .entrySet().stream()
    .flatMap(
      e -> e.getValue().stream()
        .reduce(Record::combine)
        .map(Stream::of)
        .orElse(Stream.empty()))
    .collect(Collectors.toList());
}
 
Example #9
Source File: HBCK2.java    From hbase-operator-tools with Apache License 2.0 6 votes vote down vote up
/**
 * Check for HBCK support.
 * Expects created connection.
 * @param supportedVersions list of zero or more supported versions.
 */
void checkHBCKSupport(ClusterConnection connection, String cmd, String ... supportedVersions)
    throws IOException {
  if (skipCheck) {
    LOG.info("Skipped {} command version check; 'skip' set", cmd);
    return;
  }
  try (Admin admin = connection.getAdmin()) {
    String serverVersion = admin.
        getClusterMetrics(EnumSet.of(ClusterMetrics.Option.HBASE_VERSION)).getHBaseVersion();
    String [] thresholdVersions = supportedVersions == null || supportedVersions.length == 0?
        MINIMUM_HBCK2_VERSION: supportedVersions;
    boolean supported = Version.check(serverVersion, thresholdVersions);
    if (!supported) {
      throw new UnsupportedOperationException(cmd + " not supported on server version=" +
          serverVersion + "; needs at least a server that matches or exceeds " +
          Arrays.toString(thresholdVersions));
    }
  }
}
 
Example #10
Source File: TopScreenModel.java    From hbase with Apache License 2.0 6 votes vote down vote up
private void refreshSummary(ClusterMetrics clusterMetrics) {
  String currentTime = ISO_8601_EXTENDED_TIME_FORMAT
    .format(System.currentTimeMillis());
  String version = clusterMetrics.getHBaseVersion();
  String clusterId = clusterMetrics.getClusterId();
  int liveServers = clusterMetrics.getLiveServerMetrics().size();
  int deadServers = clusterMetrics.getDeadServerNames().size();
  int regionCount = clusterMetrics.getRegionCount();
  int ritCount = clusterMetrics.getRegionStatesInTransition().size();
  double averageLoad = clusterMetrics.getAverageLoad();
  long aggregateRequestPerSecond = clusterMetrics.getLiveServerMetrics().entrySet().stream()
    .mapToLong(e -> e.getValue().getRequestCountPerSecond()).sum();

  summary = new Summary(currentTime, version, clusterId, liveServers + deadServers,
    liveServers, deadServers, regionCount, ritCount, averageLoad, aggregateRequestPerSecond);
}
 
Example #11
Source File: StochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public synchronized void setClusterMetrics(ClusterMetrics st) {
  super.setClusterMetrics(st);
  updateRegionLoad();
  for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
    cost.setClusterMetrics(st);
  }

  // update metrics size
  try {
    // by-table or ensemble mode
    int tablesCount = isByTable ? services.getTableDescriptors().getAll().size() : 1;
    int functionsCount = getCostFunctionNames().length;

    updateMetricsSize(tablesCount * (functionsCount + 1)); // +1 for overall
  } catch (Exception e) {
    LOG.error("failed to get the size of all tables", e);
  }
}
 
Example #12
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 5 votes vote down vote up
private CompletableFuture<List<ServerName>> getRegionServerList(List<String> serverNamesList) {
  CompletableFuture<List<ServerName>> future = new CompletableFuture<>();
  if (serverNamesList.isEmpty()) {
    CompletableFuture<ClusterMetrics> clusterMetricsCompletableFuture =
      getClusterMetrics(EnumSet.of(Option.SERVERS_NAME));
    addListener(clusterMetricsCompletableFuture, (clusterMetrics, err) -> {
      if (err != null) {
        future.completeExceptionally(err);
      } else {
        future.complete(clusterMetrics.getServersName());
      }
    });
    return future;
  } else {
    List<ServerName> serverList = new ArrayList<>();
    for (String regionServerName : serverNamesList) {
      ServerName serverName = null;
      try {
        serverName = ServerName.valueOf(regionServerName);
      } catch (Exception e) {
        future.completeExceptionally(
          new IllegalArgumentException(String.format("ServerName format: %s", regionServerName)));
      }
      if (serverName == null) {
        future.completeExceptionally(
          new IllegalArgumentException(String.format("Null ServerName: %s", regionServerName)));
      } else {
        serverList.add(serverName);
      }
    }
    future.complete(serverList);
  }
  return future;
}
 
Example #13
Source File: TestReplicationStatusSink.java    From hbase with Apache License 2.0 5 votes vote down vote up
private ReplicationLoadSink getLatestSinkMetric(Admin admin, ServerName server)
    throws IOException {
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
  ServerMetrics sm = metrics.getLiveServerMetrics().get(server);
  return sm.getReplicationLoadSink();
}
 
Example #14
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithoutStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.unset("hbase.regions.recovery.store.file.ref.count");
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that by default the feature is turned off so no regions
  // should be reopened
  Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());

  // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
  Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(0))
    .getRegionInfo(Mockito.any());
}
 
Example #15
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithLessThreshold() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of only 1 table
  Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen only 1 region with refCount > 400
  Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(1))
    .getRegionInfo(Mockito.any());
}
 
Example #16
Source File: TestRegionsRecoveryChore.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRegionReopensWithStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of 2 tables
  Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen total 3 regions that have refCount > 300
  Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(3))
    .getRegionInfo(Mockito.any());
}
 
Example #17
Source File: TestMasterFailoverBalancerPersistence.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test that if the master fails, the load balancer maintains its
 * state (running or not) when the next master takes over
 *
 * @throws Exception
 */
@Test
public void testMasterFailoverBalancerPersistence() throws Exception {
  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(3).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

  assertTrue(cluster.waitForActiveAndReadyMaster());
  HMaster active = cluster.getMaster();
  // check that the balancer is on by default for the active master
  ClusterMetrics clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is still running on new master
  clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  // turn off the load balancer
  active.balanceSwitch(false);

  // once more, kill active master and wait for new active master to show up
  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is not running on the new master
  clusterStatus = active.getClusterMetrics();
  assertFalse(clusterStatus.getBalancerOn());

  // Stop the cluster
  TEST_UTIL.shutdownMiniCluster();
}
 
Example #18
Source File: UserModeStrategy.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  List<Record> records = clientModeStrategy.createRecords(clusterMetrics);
  return clientModeStrategy.aggregateRecordsAndAddDistinct(
      ModeStrategyUtils.applyFilterAndGet(records, pushDownFilters), Field.USER, Field.CLIENT,
      Field.CLIENT_COUNT);
}
 
Example #19
Source File: RSGroupBasedLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void setClusterMetrics(ClusterMetrics sm) {
  this.clusterStatus = sm;
  if (internalBalancer != null) {
    internalBalancer.setClusterMetrics(sm);
  }
}
 
Example #20
Source File: RegionModeStrategy.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  List<Record> ret = new ArrayList<>();
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    long lastReportTimestamp = sm.getLastReportTimestamp();
    for (RegionMetrics rm : sm.getRegionMetrics().values()) {
      ret.add(createRecord(sm, rm, lastReportTimestamp));
    }
  }
  return ret;
}
 
Example #21
Source File: RawAsyncHBaseAdmin.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public CompletableFuture<ClusterMetrics> getClusterMetrics(EnumSet<Option> options) {
  return this
      .<ClusterMetrics> newMasterCaller()
      .action(
        (controller, stub) -> this
            .<GetClusterStatusRequest, GetClusterStatusResponse, ClusterMetrics> call(controller,
              stub, RequestConverter.buildGetClusterStatusRequest(options),
              (s, c, req, done) -> s.getClusterStatus(c, req, done),
              resp -> ClusterMetricsBuilder.toClusterMetrics(resp.getClusterStatus()))).call();
}
 
Example #22
Source File: TestRSGroupsBase.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
  throws IOException {
  Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
  Admin admin = TEST_UTIL.getAdmin();
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
  for (ServerName serverName : metrics.getServersName()) {
    for (RegionInfo region : admin.getRegions(serverName)) {
      TableName tableName = region.getTable();
      map.computeIfAbsent(tableName, k -> new TreeMap<>())
        .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
    }
  }
  return map;
}
 
Example #23
Source File: TestRSGroupsBase.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected int getNumServers() throws IOException {
  ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
  ServerName masterName = status.getMasterName();
  int count = 0;
  for (ServerName sn : status.getLiveServerMetrics().keySet()) {
    if (!sn.equals(masterName)) {
      count++;
    }
  }
  return count;
}
 
Example #24
Source File: IntegrationTestMTTR.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected boolean doAction() throws Exception {
  Admin admin = null;
  try {
    admin = util.getAdmin();
    ClusterMetrics status = admin.getClusterMetrics();
    return status != null;
  } finally {
    if (admin != null) {
      admin.close();
    }
  }
}
 
Example #25
Source File: RegionLocationFinder.java    From hbase with Apache License 2.0 5 votes vote down vote up
public void setClusterMetrics(ClusterMetrics status) {
  long currentTime = EnvironmentEdgeManager.currentTime();
  this.status = status;
  if (currentTime > lastFullRefresh + (CACHE_TIME / 2)) {
    // Only count the refresh if it includes user tables ( eg more than meta and namespace ).
    lastFullRefresh = scheduleFullRefresh()?currentTime:lastFullRefresh;
  }

}
 
Example #26
Source File: Action.java    From hbase with Apache License 2.0 5 votes vote down vote up
protected void unbalanceRegions(ClusterMetrics clusterStatus,
    List<ServerName> fromServers, List<ServerName> toServers,
    double fractionOfRegions) throws Exception {
  List<byte[]> victimRegions = new LinkedList<>();
  for (Map.Entry<ServerName, ServerMetrics> entry
    : clusterStatus.getLiveServerMetrics().entrySet()) {
    ServerName sn = entry.getKey();
    ServerMetrics serverLoad = entry.getValue();
    // Ugh.
    List<byte[]> regions = new LinkedList<>(serverLoad.getRegionMetrics().keySet());
    int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
    getLogger().debug("Removing {} regions from {}", victimRegionCount, sn);
    for (int i = 0; i < victimRegionCount; ++i) {
      int victimIx = RandomUtils.nextInt(0, regions.size());
      String regionId = RegionInfo.encodeRegionName(regions.remove(victimIx));
      victimRegions.add(Bytes.toBytes(regionId));
    }
  }

  getLogger().info("Moving {} regions from {} servers to {} different servers",
    victimRegions.size(), fromServers.size(), toServers.size());
  Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
  for (byte[] victimRegion : victimRegions) {
    // Don't keep moving regions if we're
    // trying to stop the monkey.
    if (context.isStopping()) {
      break;
    }
    int targetIx = RandomUtils.nextInt(0, toServers.size());
    admin.move(victimRegion, toServers.get(targetIx));
  }
}
 
Example #27
Source File: UnbalanceRegionsAction.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void perform() throws Exception {
  getLogger().info("Unbalancing regions");
  ClusterMetrics status = this.cluster.getClusterMetrics();
  List<ServerName> victimServers = new LinkedList<>(status.getLiveServerMetrics().keySet());
  int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size());
  List<ServerName> targetServers = new ArrayList<>(targetServerCount);
  for (int i = 0; i < targetServerCount; ++i) {
    int victimIx = RandomUtils.nextInt(0, victimServers.size());
    targetServers.add(victimServers.remove(victimIx));
  }
  unbalanceRegions(status, victimServers, targetServers, fractionOfRegions);
}
 
Example #28
Source File: ClusterStatusListener.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Acts upon the reception of a new cluster status.
 *
 * @param ncs the cluster status
 */
public void receive(ClusterMetrics ncs) {
  if (ncs.getDeadServerNames() != null) {
    for (ServerName sn : ncs.getDeadServerNames()) {
      if (!isDeadServer(sn)) {
        LOG.info("There is a new dead server: " + sn);
        deadServers.add(sn);
        if (deadServerHandler != null) {
          deadServerHandler.newDead(sn);
        }
      }
    }
  }
}
 
Example #29
Source File: ClusterStatusListener.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void channelRead0(ChannelHandlerContext ctx, DatagramPacket dp) throws Exception {
  ByteBufInputStream bis = new ByteBufInputStream(dp.content());
  try {
    ClusterStatusProtos.ClusterStatus csp = ClusterStatusProtos.ClusterStatus.parseFrom(bis);
    ClusterMetrics ncs = ClusterMetricsBuilder.toClusterMetrics(csp);
    receive(ncs);
  } finally {
    bis.close();
  }
}
 
Example #30
Source File: ClientModeStrategy.java    From hbase with Apache License 2.0 5 votes vote down vote up
List<Record> createRecords(ClusterMetrics clusterMetrics) {
  List<Record> ret = new ArrayList<>();
  for (ServerMetrics serverMetrics : clusterMetrics.getLiveServerMetrics().values()) {
    long lastReportTimestamp = serverMetrics.getLastReportTimestamp();
    serverMetrics.getUserMetrics().values().forEach(um -> um.getClientMetrics().values().forEach(
      clientMetrics -> ret.add(
            createRecord(um.getNameAsString(), clientMetrics, lastReportTimestamp,
                serverMetrics.getServerName().getServerName()))));
  }
  return ret;
}