org.apache.hadoop.hbase.master.RegionPlan Java Examples

The following examples show how to use org.apache.hadoop.hbase.master.RegionPlan. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
    // snapshot current region assignment
    Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);

    // update with new plan
    for (RegionPlan regionPlan : newRegionPlan) {
        regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
    }

    Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
    for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
        clusterState.get(entry.getValue()).add(entry.getKey());

    StochasticLoadBalancer balancer = new StochasticLoadBalancer();
    Configuration conf = admin.getConfiguration();
    conf.setFloat("hbase.regions.slop", 0.2f);
    balancer.setConf(conf);
    return balancer.balanceCluster(clusterState);
}
 
Example #2
Source File: Balance.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SimplifiableIfStatement")
private boolean preview(List<RegionPlan> regionPlanList, boolean asynchronous) throws IOException, InterruptedException {
    final boolean proceed;
    if (args.isForceProceed()) {
        proceed = true;
    } else {
        balance(args, regionPlanList, Phase.PREVIEW, asynchronous);
        if (regionPlanList.size() > 0) {
            System.out.println(regionPlanList.size() + " of " + getRegionAssignmentMap(admin, tableNameSet).size() + " region(s) will be moved.");
            warnBalanceAgain(regionPlanList);
            proceed = Util.askProceed();
        } else {
            System.out.println("There is no region to move.");
            proceed = false;
        }
    }

    return proceed;
}
 
Example #3
Source File: BaseLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public List<RegionPlan>
    balanceCluster(Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfAllTable) {
  if (isByTable) {
    List<RegionPlan> result = new ArrayList<>();
    loadOfAllTable.forEach((tableName, loadOfOneTable) -> {
      LOG.info("Start Generate Balance plan for table: " + tableName);
      List<RegionPlan> partialPlans = balanceTable(tableName, loadOfOneTable);
      if (partialPlans != null) {
        result.addAll(partialPlans);
      }
    });
    return result;
  } else {
    LOG.info("Start Generate Balance plan for cluster.");
    return balanceTable(HConstants.ENSEMBLE_TABLE_NAME, toEnsumbleTableLoad(loadOfAllTable));
  }
}
 
Example #4
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
    // snapshot current region assignment
    Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);

    // update with new plan
    for (RegionPlan regionPlan : newRegionPlan) {
        regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
    }

    Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
    for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
        clusterState.get(entry.getValue()).add(entry.getKey());

    StochasticLoadBalancer balancer = new StochasticLoadBalancer();
    Configuration conf = admin.getConfiguration();
    conf.setFloat("hbase.regions.slop", 0.2f);
    balancer.setConf(conf);
    return balancer.balanceCluster(clusterState);
}
 
Example #5
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
    // snapshot current region assignment
    Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);

    // update with new plan
    for (RegionPlan regionPlan : newRegionPlan) {
        regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
    }

    Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
    for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
        clusterState.get(entry.getValue()).add(entry.getKey());

    StochasticLoadBalancer balancer = new StochasticLoadBalancer();
    Configuration conf = admin.getConfiguration();
    conf.setFloat("hbase.regions.slop", 0.2f);
    balancer.setConf(conf);
    return balancer.balanceCluster(clusterState);
}
 
Example #6
Source File: StochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Create all of the RegionPlan's needed to move from the initial cluster state to the desired
 * state.
 *
 * @param cluster The state of the cluster
 * @return List of RegionPlan's that represent the moves needed to get to desired final state.
 */
private List<RegionPlan> createRegionPlans(Cluster cluster) {
  List<RegionPlan> plans = new LinkedList<>();
  for (int regionIndex = 0;
       regionIndex < cluster.regionIndexToServerIndex.length; regionIndex++) {
    int initialServerIndex = cluster.initialRegionIndexToServerIndex[regionIndex];
    int newServerIndex = cluster.regionIndexToServerIndex[regionIndex];

    if (initialServerIndex != newServerIndex) {
      RegionInfo region = cluster.regions[regionIndex];
      ServerName initialServer = cluster.servers[initialServerIndex];
      ServerName newServer = cluster.servers[newServerIndex];

      if (LOG.isTraceEnabled()) {
        LOG.trace("Moving Region " + region.getEncodedName() + " from server "
            + initialServer.getHostname() + " to " + newServer.getHostname());
      }
      RegionPlan rp = new RegionPlan(region, initialServer, newServer);
      plans.add(rp);
    }
  }
  return plans;
}
 
Example #7
Source File: Balance.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SimplifiableIfStatement")
private boolean preview(List<RegionPlan> regionPlanList, boolean asynchronous) throws IOException, InterruptedException {
    final boolean proceed;
    if (args.isForceProceed()) {
        proceed = true;
    } else {
        balance(args, regionPlanList, Phase.PREVIEW, asynchronous);
        if (regionPlanList.size() > 0) {
            System.out.println(regionPlanList.size() + " of " + getRegionAssignmentMap(admin, tableNameSet).size() + " region(s) will be moved.");
            warnBalanceAgain(regionPlanList);
            proceed = Util.askProceed();
        } else {
            System.out.println("There is no region to move.");
            proceed = false;
        }
    }

    return proceed;
}
 
Example #8
Source File: RangerAuthorizationCoprocessor.java    From ranger with Apache License 2.0 6 votes vote down vote up
@Override
public void postBalance(ObserverContext<MasterCoprocessorEnvironment> ctx, List<RegionPlan> plans) throws IOException {
	if(LOG.isDebugEnabled()) {
		LOG.debug("==> RangerAuthorizationCoprocessor.postBalance()");
	}

	try {
		activatePluginClassLoader();
		implMasterObserver.postBalance(ctx, plans);
	} finally {
		deactivatePluginClassLoader();
	}

	if(LOG.isDebugEnabled()) {
		LOG.debug("<== RangerAuthorizationCoprocessor.postBalance()");
	}
}
 
Example #9
Source File: Balance.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SimplifiableIfStatement")
private boolean preview(List<RegionPlan> regionPlanList, boolean asynchronous) throws IOException, InterruptedException {
    final boolean proceed;
    if (args.isForceProceed()) {
        proceed = true;
    } else {
        balance(args, regionPlanList, Phase.PREVIEW, asynchronous);
        if (regionPlanList.size() > 0) {
            System.out.println(regionPlanList.size() + " of " + getRegionAssignmentMap(admin, tableNameSet).size() + " region(s) will be moved.");
            warnBalanceAgain(regionPlanList);
            proceed = Util.askProceed();
        } else {
            System.out.println("There is no region to move.");
            proceed = false;
        }
    }

    return proceed;
}
 
Example #10
Source File: IndexMasterObserver.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Override
public void postMove(ObserverContext<MasterCoprocessorEnvironment> ctx, HRegionInfo region,
        ServerName srcServer, ServerName destServer) throws IOException {
    if (balancer != null && balancer.isTableColocated(region.getTable())) {
        AssignmentManager am = ctx.getEnvironment().getMasterServices().getAssignmentManager();
        RegionStates regionStates = am.getRegionStates();
        String tableName = region.getTable().getNameAsString();
        String correspondingTable =
                region.getTable().getNameAsString()
                        .startsWith(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX) ? MetaDataUtil
                        .getUserTableName(tableName) : MetaDataUtil
                        .getLocalIndexTableName(tableName);
        List<HRegionInfo> regions =
                regionStates.getRegionsOfTable(TableName.valueOf(correspondingTable));
        for (HRegionInfo hri : regions) {
            if (Bytes.compareTo(region.getStartKey(), hri.getStartKey()) == 0
                    && destServer != null) {
                balancer.regionOnline(hri, destServer);
                am.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, destServer));
                am.unassign(hri);
            }
        }
    }
    super.postMove(ctx, region, srcServer, destServer);
}
 
Example #11
Source File: CommandAdapter.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
public static List<RegionPlan> makePlan(HBaseAdmin admin, List<RegionPlan> newRegionPlan) throws IOException {
    // snapshot current region assignment
    Map<HRegionInfo, ServerName> regionAssignmentMap = createRegionAssignmentMap(admin);

    // update with new plan
    for (RegionPlan regionPlan : newRegionPlan) {
        regionAssignmentMap.put(regionPlan.getRegionInfo(), regionPlan.getDestination());
    }

    Map<ServerName, List<HRegionInfo>> clusterState = initializeRegionMap(admin);
    for (Map.Entry<HRegionInfo, ServerName> entry : regionAssignmentMap.entrySet())
        clusterState.get(entry.getValue()).add(entry.getKey());

    StochasticLoadBalancer balancer = new StochasticLoadBalancer();
    Configuration conf = admin.getConfiguration();
    conf.setFloat("hbase.regions.slop", 0.2f);
    balancer.setConf(conf);
    return balancer.balanceCluster(clusterState);
}
 
Example #12
Source File: TestRSGroupBasedLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Test the load balancing algorithm.
 *
 * Invariant is that all servers of the group should be hosting either floor(average) or
 * ceiling(average)
 */
@Test
public void testBalanceCluster() throws Exception {
  // Test with/without per table balancer.
  boolean[] perTableBalancerConfigs = { true, false };
  for (boolean isByTable : perTableBalancerConfigs) {
    Configuration conf = loadBalancer.getConf();
    conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
    loadBalancer.setConf(conf);
    Map<ServerName, List<RegionInfo>> servers = mockClusterServers();
    ArrayListMultimap<String, ServerAndLoad> list = convertToGroupBasedMap(servers);
    LOG.info("Mock Cluster :  " + printStats(list));
    Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
        (Map) mockClusterServersWithTables(servers);
    List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
    ArrayListMultimap<String, ServerAndLoad> balancedCluster = reconcile(list, plans);
    LOG.info("Mock Balance : " + printStats(balancedCluster));
    assertClusterAsBalanced(balancedCluster);
  }
}
 
Example #13
Source File: RSGroupableBalancerTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
protected ArrayListMultimap<String, ServerAndLoad> reconcile(
    ArrayListMultimap<String, ServerAndLoad> previousLoad,
    List<RegionPlan> plans) {
  ArrayListMultimap<String, ServerAndLoad> result = ArrayListMultimap
      .create();
  result.putAll(previousLoad);
  if (plans != null) {
    for (RegionPlan plan : plans) {
      ServerName source = plan.getSource();
      updateLoad(result, source, -1);
      ServerName destination = plan.getDestination();
      updateLoad(result, destination, +1);
    }
  }
  return result;
}
 
Example #14
Source File: TestStochasticLoadBalancer.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testNeedBalance() {
  float minCost = conf.getFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 0.05f);
  conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", 1.0f);
  try {
    // Test with/without per table balancer.
    boolean[] perTableBalancerConfigs = {true, false};
    for (boolean isByTable : perTableBalancerConfigs) {
      conf.setBoolean(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE, isByTable);
      loadBalancer.setConf(conf);
      for (int[] mockCluster : clusterStateMocks) {
        Map<ServerName, List<RegionInfo>> servers = mockClusterServers(mockCluster);
        Map<TableName, Map<ServerName, List<RegionInfo>>> LoadOfAllTable =
            (Map) mockClusterServersWithTables(servers);
        List<RegionPlan> plans = loadBalancer.balanceCluster(LoadOfAllTable);
        boolean emptyPlans = plans == null || plans.isEmpty();
        assertTrue(emptyPlans || needsBalanceIdleRegion(mockCluster));
      }
    }
  } finally {
    // reset config
    conf.unset(HConstants.HBASE_MASTER_LOADBALANCE_BYTABLE);
    conf.setFloat("hbase.master.balancer.stochastic.minCostNeedBalance", minCost);
    loadBalancer.setConf(conf);
  }
}
 
Example #15
Source File: Balance.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SimplifiableIfStatement")
private boolean preview(List<RegionPlan> regionPlanList, boolean asynchronous) throws IOException, InterruptedException {
    final boolean proceed;
    if (args.isForceProceed()) {
        proceed = true;
    } else {
        balance(args, regionPlanList, Phase.PREVIEW, asynchronous);
        if (regionPlanList.size() > 0) {
            System.out.println(regionPlanList.size() + " of " + getRegionAssignmentMap(admin, tableNameSet).size() + " region(s) will be moved.");
            warnBalanceAgain(regionPlanList);
            proceed = Util.askProceed();
        } else {
            System.out.println("There is no region to move.");
            proceed = false;
        }
    }

    return proceed;
}
 
Example #16
Source File: BalancerTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * This assumes the RegionPlan HSI instances are the same ones in the map, so
 * actually no need to even pass in the map, but I think it's clearer.
 *
 * @param list
 * @param plans
 * @return a list of all added {@link ServerAndLoad} values.
 */
protected List<ServerAndLoad> reconcile(List<ServerAndLoad> list,
                                        List<RegionPlan> plans,
                                        Map<ServerName, List<RegionInfo>> servers) {
  List<ServerAndLoad> result = new ArrayList<>(list.size());

  Map<ServerName, ServerAndLoad> map = new HashMap<>(list.size());
  for (ServerAndLoad sl : list) {
    map.put(sl.getServerName(), sl);
  }
  if (plans != null) {
    for (RegionPlan plan : plans) {
      ServerName source = plan.getSource();

      updateLoad(map, source, -1);
      ServerName destination = plan.getDestination();
      updateLoad(map, destination, +1);

      servers.get(source).remove(plan.getRegionInfo());
      servers.get(destination).add(plan.getRegionInfo());
    }
  }
  result.clear();
  result.addAll(map.values());
  return result;
}
 
Example #17
Source File: TestStochasticLoadBalancerHeterogeneousCost.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Test
public void testOverloaded() throws IOException {
  final List<String> rules = Collections.singletonList("rs[0-1] 50");

  final int numNodes = 2;
  final int numRegions = 120;
  final int numRegionsPerServer = 60;

  TestStochasticLoadBalancerHeterogeneousCostRules.createRulesFile(RULES_FILE);
  final Map<ServerName, List<RegionInfo>> serverMap =
      this.createServerMap(numNodes, numRegions, numRegionsPerServer, 1, 1);
  final List<RegionPlan> plans =
      BalancerTestBase.loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
  // As we disabled all the other cost functions, balancing only according to
  // the heterogeneous cost function should return nothing.
  assertNull(plans);
}
 
Example #18
Source File: Balance.java    From hbase-tools with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings("SimplifiableIfStatement")
private boolean preview(List<RegionPlan> regionPlanList, boolean asynchronous) throws IOException, InterruptedException {
    final boolean proceed;
    if (args.isForceProceed()) {
        proceed = true;
    } else {
        balance(args, regionPlanList, Phase.PREVIEW, asynchronous);
        if (regionPlanList.size() > 0) {
            System.out.println(regionPlanList.size() + " of " + getRegionAssignmentMap(admin, tableNameSet).size() + " region(s) will be moved.");
            warnBalanceAgain(regionPlanList);
            proceed = Util.askProceed();
        } else {
            System.out.println("There is no region to move.");
            proceed = false;
        }
    }

    return proceed;
}
 
Example #19
Source File: TestMultiParallel.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
public void postBalance(final ObserverContext<MasterCoprocessorEnvironment> ctx,
    List<RegionPlan> plans) throws IOException {
  if (!plans.isEmpty()) {
    postBalanceCount.incrementAndGet();
  }
}
 
Example #20
Source File: TestReportRegionStateTransitionRetry.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testRetryOnClose() throws Exception {
  RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
  ProcedureExecutor<MasterProcedureEnv> procExec =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
  AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
  RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region);

  CountDownLatch latch = new CountDownLatch(1);
  RESUME_AND_FAIL.set(latch);
  Future<byte[]> future =
    am.moveAsync(new RegionPlan(region, rsn.getRegionLocation(), rsn.getRegionLocation()));
  TransitRegionStateProcedure proc =
    procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure)
      .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get();

  // wait until we schedule the OpenRegionProcedure
  UTIL.waitFor(10000,
    () -> proc.getCurrentStateId() == REGION_STATE_TRANSITION_CONFIRM_OPENED_VALUE);
  // Fail the reportRegionStateTransition for closing
  latch.countDown();
  future.get();

  // confirm that the region can still be write
  try (Table table = UTIL.getConnection().getTableBuilder(NAME, null).setWriteRpcTimeout(1000)
    .setOperationTimeout(2000).build()) {
    table.put(
      new Put(Bytes.toBytes("key")).addColumn(CF, Bytes.toBytes("cq"), Bytes.toBytes("val")));
  }
}
 
Example #21
Source File: TestRegionAssignedToMultipleRegionServers.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void test() throws Exception {
  RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
  AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
  RegionStateNode rsn = am.getRegionStates().getRegionStateNode(region);

  ServerName sn = rsn.getRegionLocation();
  ARRIVE = new CountDownLatch(1);
  HALT = true;
  am.moveAsync(new RegionPlan(region, sn, sn));
  ARRIVE.await();

  // let's restart the master
  EXCLUDE_SERVERS.add(rsn.getRegionLocation());
  KILL = true;
  HMaster activeMaster = UTIL.getMiniHBaseCluster().getMaster();
  activeMaster.abort("For testing");
  activeMaster.join();
  KILL = false;

  // sleep a while to reproduce the problem, as after the fix in HBASE-21472 the execution logic
  // is changed so the old code to reproduce the problem can not compile...
  Thread.sleep(10000);
  HALT = false;
  Thread.sleep(5000);

  HRegionServer rs = UTIL.getMiniHBaseCluster().getRegionServer(sn);
  assertNotNull(rs.getRegion(region.getEncodedName()));
  assertNull(UTIL.getOtherRegionServer(rs).getRegion(region.getEncodedName()));
}
 
Example #22
Source File: IndexLoadBalancer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Prepare region plans for cluster state
 * @param clusterState if balancing is table wise then cluster state contains only indexed or
 *            index table regions, otherwise it contains all index tables regions.
 * @param regionPlans
 * @return
 */
private List<RegionPlan> prepareRegionPlansForClusterState(
        Map<ServerName, List<HRegionInfo>> clusterState, List<RegionPlan> regionPlans) {
    if (regionPlans == null) regionPlans = new ArrayList<RegionPlan>();
    ImmutableBytesWritable startKey = new ImmutableBytesWritable();
    for (Entry<ServerName, List<HRegionInfo>> serverVsRegionList : clusterState.entrySet()) {
        List<HRegionInfo> regionInfos = serverVsRegionList.getValue();
        ServerName server = serverVsRegionList.getKey();
        for (HRegionInfo regionInfo : regionInfos) {
            if (!isTableColocated(regionInfo.getTable())) continue;
            TableName mappedTableName = getMappedTableToColocate(regionInfo.getTable());
            startKey.set(regionInfo.getStartKey());
            ServerName sn = this.colocationInfo.get(mappedTableName).get(startKey);
            if (sn.equals(server)) {
                continue;
            } else {
                RegionPlan rp = new RegionPlan(regionInfo, server, sn);
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Selected server " + rp.getDestination()
                            + " as destination for region "
                            + regionInfo.getRegionNameAsString() + " from colocation info.");
                }
                regionOnline(regionInfo, rp.getDestination());
                regionPlans.add(rp);
            }
        }
    }
    return regionPlans;
}
 
Example #23
Source File: MoveRegionProcedure.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Override
protected void deserializeStateData(ProcedureStateSerializer serializer)
    throws IOException {
  super.deserializeStateData(serializer);

  final MoveRegionStateData state = serializer.deserialize(MoveRegionStateData.class);
  final RegionInfo regionInfo = getRegion(); // Get it from super class deserialization.
  final ServerName sourceServer = ProtobufUtil.toServerName(state.getSourceServer());
  final ServerName destinationServer = state.hasDestinationServer() ?
      ProtobufUtil.toServerName(state.getDestinationServer()) : null;
  this.plan = new RegionPlan(regionInfo, sourceServer, destinationServer);
}
 
Example #24
Source File: TestStochasticLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testLosingRs() throws Exception {
  int numNodes = 3;
  int numRegions = 20;
  int numRegionsPerServer = 3; //all servers except one
  int replication = 1;
  int numTables = 2;

  Map<ServerName, List<RegionInfo>> serverMap =
      createServerMap(numNodes, numRegions, numRegionsPerServer, replication, numTables);
  List<ServerAndLoad> list = convertToList(serverMap);


  List<RegionPlan> plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
  assertNotNull(plans);

  // Apply the plan to the mock cluster.
  List<ServerAndLoad> balancedCluster = reconcile(list, plans, serverMap);

  assertClusterAsBalanced(balancedCluster);

  ServerName sn = serverMap.keySet().toArray(new ServerName[serverMap.size()])[0];

  ServerName deadSn = ServerName.valueOf(sn.getHostname(), sn.getPort(), sn.getStartcode() - 100);

  serverMap.put(deadSn, new ArrayList<>(0));

  plans = loadBalancer.balanceTable(HConstants.ENSEMBLE_TABLE_NAME, serverMap);
  assertNull(plans);
}
 
Example #25
Source File: TestSimpleLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
private void testImpactOfBalanceClusterOverall(boolean useLoadOfAllTable) throws Exception {
  Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad = new TreeMap<>();
  Map<ServerName, List<RegionInfo>> clusterServers =
      mockUniformClusterServers(mockUniformCluster);
  List<ServerAndLoad> clusterList = convertToList(clusterServers);
  clusterLoad.put(TableName.valueOf(name.getMethodName()), clusterServers);
  // use overall can achieve both table and cluster level balance
  HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> LoadOfAllTable =
      mockClusterServersWithTables(clusterServers);
  if (useLoadOfAllTable) {
    loadBalancer.setClusterLoad((Map) LoadOfAllTable);
  } else {
    loadBalancer.setClusterLoad(clusterLoad);
  }
  List<RegionPlan> clusterplans1 = new ArrayList<RegionPlan>();
  List<Pair<TableName, Integer>> regionAmountList = new ArrayList<Pair<TableName, Integer>>();
  for (Map.Entry<TableName, TreeMap<ServerName, List<RegionInfo>>> mapEntry : LoadOfAllTable
      .entrySet()) {
    TableName tableName = mapEntry.getKey();
    TreeMap<ServerName, List<RegionInfo>> servers = mapEntry.getValue();
    List<ServerAndLoad> list = convertToList(servers);
    LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
    List<RegionPlan> partialplans = loadBalancer.balanceTable(tableName, servers);
    if (partialplans != null) clusterplans1.addAll(partialplans);
    List<ServerAndLoad> balancedClusterPerTable = reconcile(list, partialplans, servers);
    LOG.info("Mock Balance : " + printMock(balancedClusterPerTable));
    assertClusterAsBalanced(balancedClusterPerTable);
    for (Map.Entry<ServerName, List<RegionInfo>> entry : servers.entrySet()) {
      returnRegions(entry.getValue());
      returnServer(entry.getKey());
    }
  }
  List<ServerAndLoad> balancedCluster1 = reconcile(clusterList, clusterplans1, clusterServers);
  assertTrue(assertClusterOverallAsBalanced(balancedCluster1, LoadOfAllTable.keySet().size()));
}
 
Example #26
Source File: RSGroupBasedLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * can achieve table balanced rather than overall balanced
 */
@Override
public List<RegionPlan> balanceTable(TableName tableName,
    Map<ServerName, List<RegionInfo>> loadOfOneTable) {
  if (!isOnline()) {
    LOG.error(RSGroupInfoManager.class.getSimpleName()
        + " is not online, unable to perform balanceTable");
    return null;
  }
  Map<TableName, Map<ServerName, List<RegionInfo>>> loadOfThisTable = new HashMap<>();
  loadOfThisTable.put(tableName, loadOfOneTable);
  Pair<Map<TableName, Map<ServerName, List<RegionInfo>>>, List<RegionPlan>>
    correctedStateAndRegionPlans;
  // Calculate correct assignments and a list of RegionPlan for mis-placed regions
  try {
    correctedStateAndRegionPlans = correctAssignments(loadOfThisTable);
  } catch (IOException e) {
    LOG.error("get correct assignments and mis-placed regions error ", e);
    return null;
  }
  Map<TableName, Map<ServerName, List<RegionInfo>>> correctedLoadOfThisTable =
      correctedStateAndRegionPlans.getFirst();
  List<RegionPlan> regionPlans = correctedStateAndRegionPlans.getSecond();
  List<RegionPlan> tablePlans =
      this.internalBalancer.balanceTable(tableName, correctedLoadOfThisTable.get(tableName));

  if (tablePlans != null) {
    regionPlans.addAll(tablePlans);
  }
  return regionPlans;
}
 
Example #27
Source File: SimpleLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Add a region from the head or tail to the List of regions to return.
 */
private void addRegionPlan(final MinMaxPriorityQueue<RegionPlan> regionsToMove,
    final boolean fetchFromTail, final ServerName sn, List<RegionPlan> regionsToReturn) {
  RegionPlan rp = null;
  if (!fetchFromTail) rp = regionsToMove.remove();
  else rp = regionsToMove.removeLast();
  rp.setDestination(sn);
  regionsToReturn.add(rp);
}
 
Example #28
Source File: TestRaceBetweenSCPAndTRSP.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void test() throws Exception {
  RegionInfo region = UTIL.getMiniHBaseCluster().getRegions(NAME).get(0).getRegionInfo();
  AssignmentManager am = UTIL.getMiniHBaseCluster().getMaster().getAssignmentManager();
  ServerName sn = am.getRegionStates().getRegionState(region).getServerName();

  // Assign the CountDownLatches that get nulled in background threads else we NPE checking
  // the static.
  ARRIVE_REGION_OPENING = new CountDownLatch(1);
  CountDownLatch arriveRegionOpening = ARRIVE_REGION_OPENING;
  RESUME_REGION_OPENING = new CountDownLatch(1);
  ARRIVE_GET_REGIONS_ON_SERVER = new CountDownLatch(1);
  CountDownLatch arriveGetRegionsOnServer = ARRIVE_GET_REGIONS_ON_SERVER;
  RESUME_GET_REGIONS_ON_SERVER = new CountDownLatch(1);

  Future<byte[]> moveFuture = am.moveAsync(new RegionPlan(region, sn, sn));
  arriveRegionOpening.await();

  UTIL.getMiniHBaseCluster().killRegionServer(sn);
  arriveGetRegionsOnServer.await();
  RESUME_REGION_OPENING.countDown();

  moveFuture.get();
  ProcedureExecutor<?> procExec =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
  long scpProcId =
    procExec.getProcedures().stream().filter(p -> p instanceof ServerCrashProcedure)
      .map(p -> (ServerCrashProcedure) p).findAny().get().getProcId();
  RESUME_GET_REGIONS_ON_SERVER.countDown();
  UTIL.waitFor(60000, () -> procExec.isFinished(scpProcId));
}
 
Example #29
Source File: IndexLoadBalancer.java    From phoenix with Apache License 2.0 5 votes vote down vote up
private void updateRegionPlans(List<RegionPlan> regionPlans) {
    for (RegionPlan regionPlan : regionPlans) {
        HRegionInfo hri = regionPlan.getRegionInfo();
        if (!isTableColocated(hri.getTable())) continue;
        if (LOG.isDebugEnabled()) {
            LOG.debug("Saving region plan of region " + hri.getRegionNameAsString() + '.');
        }
        regionOnline(hri, regionPlan.getDestination());
    }
}
 
Example #30
Source File: TestSimpleLoadBalancer.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Test the load balancing algorithm.
 *
 * Invariant is that all servers should be hosting either floor(average) or
 * ceiling(average) at both table level and cluster level
 */
@Test
public void testBalanceClusterOverall() throws Exception {
  Map<TableName, Map<ServerName, List<RegionInfo>>> clusterLoad = new TreeMap<>();
  for (int[] mockCluster : clusterStateMocks) {
    Map<ServerName, List<RegionInfo>> clusterServers = mockClusterServers(mockCluster, 30);
    List<ServerAndLoad> clusterList = convertToList(clusterServers);
    clusterLoad.put(TableName.valueOf(name.getMethodName()), clusterServers);
    HashMap<TableName, TreeMap<ServerName, List<RegionInfo>>> result =
        mockClusterServersWithTables(clusterServers);
    loadBalancer.setClusterLoad(clusterLoad);
    List<RegionPlan> clusterplans = new ArrayList<>();
    List<Pair<TableName, Integer>> regionAmountList = new ArrayList<>();
    for (Map.Entry<TableName, TreeMap<ServerName, List<RegionInfo>>> mapEntry : result
        .entrySet()) {
      TableName tableName = mapEntry.getKey();
      TreeMap<ServerName, List<RegionInfo>> servers = mapEntry.getValue();
      List<ServerAndLoad> list = convertToList(servers);
      LOG.info("Mock Cluster : " + printMock(list) + " " + printStats(list));
      List<RegionPlan> partialplans = loadBalancer.balanceTable(tableName, servers);
      if(partialplans != null) clusterplans.addAll(partialplans);
      List<ServerAndLoad> balancedClusterPerTable = reconcile(list, partialplans, servers);
      LOG.info("Mock Balance : " + printMock(balancedClusterPerTable));
      assertClusterAsBalanced(balancedClusterPerTable);
      for (Map.Entry<ServerName, List<RegionInfo>> entry : servers.entrySet()) {
        returnRegions(entry.getValue());
        returnServer(entry.getKey());
      }
    }
    List<ServerAndLoad> balancedCluster = reconcile(clusterList, clusterplans, clusterServers);
    assertTrue(assertClusterOverallAsBalanced(balancedCluster, result.keySet().size()));
  }
}