org.apache.hadoop.hbase.ClusterMetrics Java Examples

@Override
public void perform() throws Exception {
  getLogger().info("Performing action: Restart region server holding META");
  ServerName server = cluster.getServerHoldingMeta();
  if (server == null) {
    getLogger().warn("No server is holding hbase:meta right now.");
    return;
  }
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  if (server.equals(clusterStatus.getMasterName())) {
    // Master holds the meta, so restart the master.
    restartMaster(server, sleepTime);
  } else {
    restartRs(server, sleepTime);
  }
}
 

/** Returns current region servers - active master */
protected ServerName[] getCurrentServers() throws IOException {
  ClusterMetrics clusterStatus = cluster.getClusterMetrics();
  Collection<ServerName> regionServers = clusterStatus.getLiveServerMetrics().keySet();
  int count = regionServers.size();
  if (count <= 0) {
    return new ServerName [] {};
  }
  ServerName master = clusterStatus.getMasterName();
  Set<ServerName> masters = new HashSet<>();
  masters.add(master);
  masters.addAll(clusterStatus.getBackupMasterNames());
  ArrayList<ServerName> tmp = new ArrayList<>(count);
  tmp.addAll(regionServers);
  tmp.removeAll(masters);

  if(skipMetaRS){
    ServerName metaServer = cluster.getServerHoldingMeta();
    tmp.remove(metaServer);
  }

  return tmp.toArray(new ServerName[0]);
}
 

/**
 * Get region info from local cluster.
 */
Map<ServerName, List<String>> getDeployedHRIs(final Admin admin) throws IOException {
  ClusterMetrics status = admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  Collection<ServerName> regionServers = status.getLiveServerMetrics().keySet();
  Map<ServerName, List<String>> mm = new HashMap<>();
  for (ServerName hsi : regionServers) {
    // list all online regions from this region server
    List<RegionInfo> regions = admin.getRegions(hsi);
    List<String> regionNames = new ArrayList<>(regions.size());
    for (RegionInfo hri : regions) {
      regionNames.add(hri.getRegionNameAsString());
    }
    mm.put(hsi, regionNames);
  }
  return mm;
}
 

@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);
  // Aggregation by LONG_REGION_SERVER field
  Map<String, Record> retMap =
      ModeStrategyUtils.aggregateRecords(records, Field.LONG_REGION_SERVER).stream()
          .collect(Collectors.toMap(r -> r.get(Field.LONG_REGION_SERVER).asString(), r -> r));

  // Add USED_HEAP_SIZE field and MAX_HEAP_SIZE field
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    Record record = retMap.get(sm.getServerName().getServerName());
    if (record == null) {
      continue;
    }

    Record newRecord = Record.builder().putAll(record)
      .put(Field.USED_HEAP_SIZE, sm.getUsedHeapSize())
      .put(Field.MAX_HEAP_SIZE, sm.getMaxHeapSize()).build();

    retMap.put(sm.getServerName().getServerName(), newRecord);
  }

  return new ArrayList<>(retMap.values());
}
 

@Test
public void testReplicationStatusSourceStartedTargetStoppedNoOps() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  Thread.sleep(10000);
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertFalse(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertEquals(0, loadSource.getReplicationLag());
  assertFalse(loadSource.isRecovered());
}
 

@Test
public void testReplicationStatusSourceStartedTargetStoppedNewOp() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  Admin hbaseAdmin = UTIL1.getAdmin();
  // add some values to source cluster
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  Thread.sleep(10000);
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertEquals(0, loadSource.getTimestampOfLastShippedOp());
  assertTrue(loadSource.getReplicationLag() > 0);
  assertFalse(loadSource.isRecovered());
}
 

@Test
public void testReplicationStatusAfterLagging() throws Exception {
  UTIL2.shutdownMiniHBaseCluster();
  restartSourceCluster(1);
  // add some values to cluster 1
  for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
    Put p = new Put(Bytes.toBytes("row" + i));
    p.addColumn(famName, Bytes.toBytes("col1"), Bytes.toBytes("val" + i));
    htable1.put(p);
  }
  UTIL2.startMiniHBaseCluster();
  Thread.sleep(10000);
  Admin hbaseAdmin = UTIL1.getAdmin();
  ServerName serverName = UTIL1.getHBaseCluster().getRegionServer(0).getServerName();
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> loadSources =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  assertEquals(1, loadSources.size());
  ReplicationLoadSource loadSource = loadSources.get(0);
  assertTrue(loadSource.hasEditsSinceRestart());
  assertTrue(loadSource.getTimestampOfLastShippedOp() > 0);
  assertEquals(0, loadSource.getReplicationLag());
}
 

@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  // Get records from RegionModeStrategy and add REGION_COUNT field
  List<Record> records = regionModeStrategy.selectModeFieldsAndAddCountField(fieldInfos,
      regionModeStrategy.getRecords(clusterMetrics, pushDownFilters), Field.REGION_COUNT);

  // Aggregation by NAMESPACE field and TABLE field
  return records.stream()
    .collect(Collectors.groupingBy(r -> {
      String namespace = r.get(Field.NAMESPACE).asString();
      String table = r.get(Field.TABLE).asString();
      return TableName.valueOf(namespace, table);
    }))
    .entrySet().stream()
    .flatMap(
      e -> e.getValue().stream()
        .reduce(Record::combine)
        .map(Stream::of)
        .orElse(Stream.empty()))
    .collect(Collectors.toList());
}
 

/**
 * Check for HBCK support.
 * Expects created connection.
 * @param supportedVersions list of zero or more supported versions.
 */
void checkHBCKSupport(ClusterConnection connection, String cmd, String ... supportedVersions)
    throws IOException {
  if (skipCheck) {
    LOG.info("Skipped {} command version check; 'skip' set", cmd);
    return;
  }
  try (Admin admin = connection.getAdmin()) {
    String serverVersion = admin.
        getClusterMetrics(EnumSet.of(ClusterMetrics.Option.HBASE_VERSION)).getHBaseVersion();
    String [] thresholdVersions = supportedVersions == null || supportedVersions.length == 0?
        MINIMUM_HBCK2_VERSION: supportedVersions;
    boolean supported = Version.check(serverVersion, thresholdVersions);
    if (!supported) {
      throw new UnsupportedOperationException(cmd + " not supported on server version=" +
          serverVersion + "; needs at least a server that matches or exceeds " +
          Arrays.toString(thresholdVersions));
    }
  }
}
 

private void refreshSummary(ClusterMetrics clusterMetrics) {
  String currentTime = ISO_8601_EXTENDED_TIME_FORMAT
    .format(System.currentTimeMillis());
  String version = clusterMetrics.getHBaseVersion();
  String clusterId = clusterMetrics.getClusterId();
  int liveServers = clusterMetrics.getLiveServerMetrics().size();
  int deadServers = clusterMetrics.getDeadServerNames().size();
  int regionCount = clusterMetrics.getRegionCount();
  int ritCount = clusterMetrics.getRegionStatesInTransition().size();
  double averageLoad = clusterMetrics.getAverageLoad();
  long aggregateRequestPerSecond = clusterMetrics.getLiveServerMetrics().entrySet().stream()
    .mapToLong(e -> e.getValue().getRequestCountPerSecond()).sum();

  summary = new Summary(currentTime, version, clusterId, liveServers + deadServers,
    liveServers, deadServers, regionCount, ritCount, averageLoad, aggregateRequestPerSecond);
}
 

@Override
public synchronized void setClusterMetrics(ClusterMetrics st) {
  super.setClusterMetrics(st);
  updateRegionLoad();
  for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
    cost.setClusterMetrics(st);
  }

  // update metrics size
  try {
    // by-table or ensemble mode
    int tablesCount = isByTable ? services.getTableDescriptors().getAll().size() : 1;
    int functionsCount = getCostFunctionNames().length;

    updateMetricsSize(tablesCount * (functionsCount + 1)); // +1 for overall
  } catch (Exception e) {
    LOG.error("failed to get the size of all tables", e);
  }
}
 

private CompletableFuture<List<ServerName>> getRegionServerList(List<String> serverNamesList) {
  CompletableFuture<List<ServerName>> future = new CompletableFuture<>();
  if (serverNamesList.isEmpty()) {
    CompletableFuture<ClusterMetrics> clusterMetricsCompletableFuture =
      getClusterMetrics(EnumSet.of(Option.SERVERS_NAME));
    addListener(clusterMetricsCompletableFuture, (clusterMetrics, err) -> {
      if (err != null) {
        future.completeExceptionally(err);
      } else {
        future.complete(clusterMetrics.getServersName());
      }
    });
    return future;
  } else {
    List<ServerName> serverList = new ArrayList<>();
    for (String regionServerName : serverNamesList) {
      ServerName serverName = null;
      try {
        serverName = ServerName.valueOf(regionServerName);
      } catch (Exception e) {
        future.completeExceptionally(
          new IllegalArgumentException(String.format("ServerName format: %s", regionServerName)));
      }
      if (serverName == null) {
        future.completeExceptionally(
          new IllegalArgumentException(String.format("Null ServerName: %s", regionServerName)));
      } else {
        serverList.add(serverName);
      }
    }
    future.complete(serverList);
  }
  return future;
}
 

private ReplicationLoadSink getLatestSinkMetric(Admin admin, ServerName server)
    throws IOException {
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
  ServerMetrics sm = metrics.getLiveServerMetrics().get(server);
  return sm.getReplicationLoadSink();
}
 

@Test
public void testRegionReopensWithoutStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(10);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.unset("hbase.regions.recovery.store.file.ref.count");
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that by default the feature is turned off so no regions
  // should be reopened
  Mockito.verify(hMaster, Mockito.times(0)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());

  // default maxCompactedStoreFileRefCount is -1 (no regions to be reopened using AM)
  Mockito.verify(hMaster, Mockito.times(0)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(0))
    .getRegionInfo(Mockito.any());
}
 

@Test
public void testRegionReopensWithLessThreshold() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 400);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of only 1 table
  Mockito.verify(hMaster, Mockito.times(1)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen only 1 region with refCount > 400
  Mockito.verify(hMaster, Mockito.times(1)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(1))
    .getRegionInfo(Mockito.any());
}
 

@Test
public void testRegionReopensWithStoreRefConfig() throws Exception {
  regionNo = 0;
  ClusterMetrics clusterMetrics = TestRegionsRecoveryChore.getClusterMetrics(4);
  final Map<ServerName, ServerMetrics> serverMetricsMap =
    clusterMetrics.getLiveServerMetrics();
  LOG.debug("All Region Names with refCount....");
  for (ServerMetrics serverMetrics : serverMetricsMap.values()) {
    Map<byte[], RegionMetrics> regionMetricsMap = serverMetrics.getRegionMetrics();
    for (RegionMetrics regionMetrics : regionMetricsMap.values()) {
      LOG.debug("name: " + new String(regionMetrics.getRegionName()) + " refCount: " +
        regionMetrics.getStoreRefCount());
    }
  }
  Mockito.when(hMaster.getClusterMetrics()).thenReturn(clusterMetrics);
  Mockito.when(hMaster.getAssignmentManager()).thenReturn(assignmentManager);
  for (byte[] regionName : REGION_NAME_LIST) {
    Mockito.when(assignmentManager.getRegionInfo(regionName))
      .thenReturn(TestRegionsRecoveryChore.getRegionInfo(regionName));
  }
  Stoppable stoppable = new StoppableImplementation();
  Configuration configuration = getCustomConf();
  configuration.setInt("hbase.regions.recovery.store.file.ref.count", 300);
  regionsRecoveryChore = new RegionsRecoveryChore(stoppable, configuration, hMaster);
  regionsRecoveryChore.chore();

  // Verify that we need to reopen regions of 2 tables
  Mockito.verify(hMaster, Mockito.times(2)).reopenRegions(Mockito.any(), Mockito.anyList(),
    Mockito.anyLong(), Mockito.anyLong());
  Mockito.verify(hMaster, Mockito.times(1)).getClusterMetrics();

  // Verify that we need to reopen total 3 regions that have refCount > 300
  Mockito.verify(hMaster, Mockito.times(3)).getAssignmentManager();
  Mockito.verify(assignmentManager, Mockito.times(3))
    .getRegionInfo(Mockito.any());
}
 

/**
 * Test that if the master fails, the load balancer maintains its
 * state (running or not) when the next master takes over
 *
 * @throws Exception
 */
@Test
public void testMasterFailoverBalancerPersistence() throws Exception {
  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(3).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

  assertTrue(cluster.waitForActiveAndReadyMaster());
  HMaster active = cluster.getMaster();
  // check that the balancer is on by default for the active master
  ClusterMetrics clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is still running on new master
  clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  // turn off the load balancer
  active.balanceSwitch(false);

  // once more, kill active master and wait for new active master to show up
  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is not running on the new master
  clusterStatus = active.getClusterMetrics();
  assertFalse(clusterStatus.getBalancerOn());

  // Stop the cluster
  TEST_UTIL.shutdownMiniCluster();
}
 

@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  List<Record> records = clientModeStrategy.createRecords(clusterMetrics);
  return clientModeStrategy.aggregateRecordsAndAddDistinct(
      ModeStrategyUtils.applyFilterAndGet(records, pushDownFilters), Field.USER, Field.CLIENT,
      Field.CLIENT_COUNT);
}
 

@Override
public void setClusterMetrics(ClusterMetrics sm) {
  this.clusterStatus = sm;
  if (internalBalancer != null) {
    internalBalancer.setClusterMetrics(sm);
  }
}
 

@Override public List<Record> getRecords(ClusterMetrics clusterMetrics,
    List<RecordFilter> pushDownFilters) {
  List<Record> ret = new ArrayList<>();
  for (ServerMetrics sm : clusterMetrics.getLiveServerMetrics().values()) {
    long lastReportTimestamp = sm.getLastReportTimestamp();
    for (RegionMetrics rm : sm.getRegionMetrics().values()) {
      ret.add(createRecord(sm, rm, lastReportTimestamp));
    }
  }
  return ret;
}
 

@Override
public CompletableFuture<ClusterMetrics> getClusterMetrics(EnumSet<Option> options) {
  return this
      .<ClusterMetrics> newMasterCaller()
      .action(
        (controller, stub) -> this
            .<GetClusterStatusRequest, GetClusterStatusResponse, ClusterMetrics> call(controller,
              stub, RequestConverter.buildGetClusterStatusRequest(options),
              (s, c, req, done) -> s.getClusterStatus(c, req, done),
              resp -> ClusterMetricsBuilder.toClusterMetrics(resp.getClusterStatus()))).call();
}
 

protected Map<TableName, Map<ServerName, List<String>>> getTableServerRegionMap()
  throws IOException {
  Map<TableName, Map<ServerName, List<String>>> map = Maps.newTreeMap();
  Admin admin = TEST_UTIL.getAdmin();
  ClusterMetrics metrics =
    admin.getClusterMetrics(EnumSet.of(ClusterMetrics.Option.SERVERS_NAME));
  for (ServerName serverName : metrics.getServersName()) {
    for (RegionInfo region : admin.getRegions(serverName)) {
      TableName tableName = region.getTable();
      map.computeIfAbsent(tableName, k -> new TreeMap<>())
        .computeIfAbsent(serverName, k -> new ArrayList<>()).add(region.getRegionNameAsString());
    }
  }
  return map;
}
 

protected int getNumServers() throws IOException {
  ClusterMetrics status = ADMIN.getClusterMetrics(EnumSet.of(Option.MASTER, Option.LIVE_SERVERS));
  ServerName masterName = status.getMasterName();
  int count = 0;
  for (ServerName sn : status.getLiveServerMetrics().keySet()) {
    if (!sn.equals(masterName)) {
      count++;
    }
  }
  return count;
}
 

@Override
protected boolean doAction() throws Exception {
  Admin admin = null;
  try {
    admin = util.getAdmin();
    ClusterMetrics status = admin.getClusterMetrics();
    return status != null;
  } finally {
    if (admin != null) {
      admin.close();
    }
  }
}
 

public void setClusterMetrics(ClusterMetrics status) {
  long currentTime = EnvironmentEdgeManager.currentTime();
  this.status = status;
  if (currentTime > lastFullRefresh + (CACHE_TIME / 2)) {
    // Only count the refresh if it includes user tables ( eg more than meta and namespace ).
    lastFullRefresh = scheduleFullRefresh()?currentTime:lastFullRefresh;
  }

}
 

protected void unbalanceRegions(ClusterMetrics clusterStatus,
    List<ServerName> fromServers, List<ServerName> toServers,
    double fractionOfRegions) throws Exception {
  List<byte[]> victimRegions = new LinkedList<>();
  for (Map.Entry<ServerName, ServerMetrics> entry
    : clusterStatus.getLiveServerMetrics().entrySet()) {
    ServerName sn = entry.getKey();
    ServerMetrics serverLoad = entry.getValue();
    // Ugh.
    List<byte[]> regions = new LinkedList<>(serverLoad.getRegionMetrics().keySet());
    int victimRegionCount = (int)Math.ceil(fractionOfRegions * regions.size());
    getLogger().debug("Removing {} regions from {}", victimRegionCount, sn);
    for (int i = 0; i < victimRegionCount; ++i) {
      int victimIx = RandomUtils.nextInt(0, regions.size());
      String regionId = RegionInfo.encodeRegionName(regions.remove(victimIx));
      victimRegions.add(Bytes.toBytes(regionId));
    }
  }

  getLogger().info("Moving {} regions from {} servers to {} different servers",
    victimRegions.size(), fromServers.size(), toServers.size());
  Admin admin = this.context.getHBaseIntegrationTestingUtility().getAdmin();
  for (byte[] victimRegion : victimRegions) {
    // Don't keep moving regions if we're
    // trying to stop the monkey.
    if (context.isStopping()) {
      break;
    }
    int targetIx = RandomUtils.nextInt(0, toServers.size());
    admin.move(victimRegion, toServers.get(targetIx));
  }
}
 

@Override
public void perform() throws Exception {
  getLogger().info("Unbalancing regions");
  ClusterMetrics status = this.cluster.getClusterMetrics();
  List<ServerName> victimServers = new LinkedList<>(status.getLiveServerMetrics().keySet());
  int targetServerCount = (int)Math.ceil(fractionOfServers * victimServers.size());
  List<ServerName> targetServers = new ArrayList<>(targetServerCount);
  for (int i = 0; i < targetServerCount; ++i) {
    int victimIx = RandomUtils.nextInt(0, victimServers.size());
    targetServers.add(victimServers.remove(victimIx));
  }
  unbalanceRegions(status, victimServers, targetServers, fractionOfRegions);
}
 

/**
 * Acts upon the reception of a new cluster status.
 *
 * @param ncs the cluster status
 */
public void receive(ClusterMetrics ncs) {
  if (ncs.getDeadServerNames() != null) {
    for (ServerName sn : ncs.getDeadServerNames()) {
      if (!isDeadServer(sn)) {
        LOG.info("There is a new dead server: " + sn);
        deadServers.add(sn);
        if (deadServerHandler != null) {
          deadServerHandler.newDead(sn);
        }
      }
    }
  }
}
 

@Override
protected void channelRead0(ChannelHandlerContext ctx, DatagramPacket dp) throws Exception {
  ByteBufInputStream bis = new ByteBufInputStream(dp.content());
  try {
    ClusterStatusProtos.ClusterStatus csp = ClusterStatusProtos.ClusterStatus.parseFrom(bis);
    ClusterMetrics ncs = ClusterMetricsBuilder.toClusterMetrics(csp);
    receive(ncs);
  } finally {
    bis.close();
  }
}
 

List<Record> createRecords(ClusterMetrics clusterMetrics) {
  List<Record> ret = new ArrayList<>();
  for (ServerMetrics serverMetrics : clusterMetrics.getLiveServerMetrics().values()) {
    long lastReportTimestamp = serverMetrics.getLastReportTimestamp();
    serverMetrics.getUserMetrics().values().forEach(um -> um.getClientMetrics().values().forEach(
      clientMetrics -> ret.add(
            createRecord(um.getNameAsString(), clientMetrics, lastReportTimestamp,
                serverMetrics.getServerName().getServerName()))));
  }
  return ret;
}