Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility#getHBaseCluster()

@BeforeClass
public static void setUp() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
  TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1");
  StartMiniClusterOption option =
    StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build();
  TEST_UTIL.startMiniCluster(option);
  cluster = TEST_UTIL.getHBaseCluster();
  master = ((MiniHBaseCluster) cluster).getMaster();
  master.balanceSwitch(false);
  hbaseAdmin = TEST_UTIL.getAdmin();
  // wait till the balancer is in online mode
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return master.isInitialized() &&
        ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() &&
        master.getServerManager().getOnlineServersList().size() >= 3;
    }
  });
}
 

@Before
@Override
public void setUp() throws Exception {
  utility = new HBaseTestingUtility();
  Configuration conf = utility.getConfiguration();
  RSGroupUtil.enableRSGroup(conf);
  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE);
  conf.setInt("hbase.hfile.compaction.discharger.interval", 10);
  utility.startMiniCluster(NUM_SLAVES_BASE);
  MiniHBaseCluster cluster = utility.getHBaseCluster();
  final HMaster master = cluster.getMaster();

  //wait for balancer to come online
  utility.waitFor(60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() {
      return master.isInitialized() &&
          ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
    }
  });
  admin = utility.getAdmin();
}
 

@BeforeClass
public static void startCluster() throws Exception {
  metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  TEST_UTIL = new HBaseTestingUtility();
  conf = TEST_UTIL.getConfiguration();
  conf.getLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);

  TEST_UTIL.startMiniCluster(2);
  cluster = TEST_UTIL.getHBaseCluster();

  cluster.waitForActiveAndReadyMaster();

  while (cluster.getLiveRegionServerThreads().size() < 2) {
    Threads.sleep(100);
  }
}
 

/**
 * Ensures each region of SYSTEM.CATALOG is on a different region server
 */
private static void moveRegion(HRegionInfo regionInfo, ServerName srcServerName, ServerName dstServerName) throws Exception  {
    Admin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
    HBaseTestingUtility util = getUtility();
    MiniHBaseCluster cluster = util.getHBaseCluster();
    HMaster master = cluster.getMaster();
    AssignmentManager am = master.getAssignmentManager();
   
    HRegionServer dstServer = util.getHBaseCluster().getRegionServer(dstServerName);
    HRegionServer srcServer = util.getHBaseCluster().getRegionServer(srcServerName);
    byte[] encodedRegionNameInBytes = regionInfo.getEncodedNameAsBytes();
    admin.move(encodedRegionNameInBytes, Bytes.toBytes(dstServer.getServerName().getServerName()));
    while (dstServer.getOnlineRegion(regionInfo.getRegionName()) == null
            || dstServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
            || srcServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)) {
        // wait for the move to be finished
        Thread.sleep(100);
    }
}
 

private static void shutdown(HBaseTestingUtility util) throws Exception {
  if (util.getHBaseCluster() == null) {
    return;
  }
  Admin admin = util.getAdmin();
  if (!admin.listReplicationPeers(Pattern.compile(PEER_ID)).isEmpty()) {
    if (admin
      .getReplicationPeerSyncReplicationState(PEER_ID) != SyncReplicationState.DOWNGRADE_ACTIVE) {
      admin.transitReplicationPeerSyncReplicationState(PEER_ID,
        SyncReplicationState.DOWNGRADE_ACTIVE);
    }
    admin.removeReplicationPeer(PEER_ID);
  }
  util.shutdownMiniCluster();
}
 

@BeforeClass
public static void startCluster() throws Exception {
  LOG.info("Starting cluster");
  TEST_UTIL = new HBaseTestingUtility();
  // Set master class and use default values for other options.
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .masterClass(TestMasterMetrics.MyMaster.class).build();
  TEST_UTIL.startMiniCluster(option);
  cluster = TEST_UTIL.getHBaseCluster();
  LOG.info("Waiting for active/ready master");
  cluster.waitForActiveAndReadyMaster();
  master = cluster.getMaster();
}
 

/**
 * Test that if the master fails, the load balancer maintains its
 * state (running or not) when the next master takes over
 *
 * @throws Exception
 */
@Test
public void testMasterFailoverBalancerPersistence() throws Exception {
  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();

  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(3).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();

  assertTrue(cluster.waitForActiveAndReadyMaster());
  HMaster active = cluster.getMaster();
  // check that the balancer is on by default for the active master
  ClusterMetrics clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is still running on new master
  clusterStatus = active.getClusterMetrics();
  assertTrue(clusterStatus.getBalancerOn());

  // turn off the load balancer
  active.balanceSwitch(false);

  // once more, kill active master and wait for new active master to show up
  active = killActiveAndWaitForNewActive(cluster);

  // ensure the load balancer is not running on the new master
  clusterStatus = active.getClusterMetrics();
  assertFalse(clusterStatus.getBalancerOn());

  // Stop the cluster
  TEST_UTIL.shutdownMiniCluster();
}
 

@BeforeClass
public static void startCluster() throws Exception {
  metricsHelper = CompatibilityFactory.getInstance(MetricsAssertHelper.class);
  TEST_UTIL = new HBaseTestingUtility();
  TABLES_ON_MASTER = LoadBalancer.isTablesOnMaster(TEST_UTIL.getConfiguration());
  conf = TEST_UTIL.getConfiguration();
  conf.getLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  // testMobMetrics creates few hfiles and manages compaction manually.
  conf.setInt("hbase.hstore.compactionThreshold", 100);
  conf.setInt("hbase.hstore.compaction.max", 100);
  conf.setInt("hbase.regionserver.periodicmemstoreflusher.rangeofdelayseconds", 4*60);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);

  TEST_UTIL.startMiniCluster();
  cluster = TEST_UTIL.getHBaseCluster();
  cluster.waitForActiveAndReadyMaster();
  admin = TEST_UTIL.getAdmin();
  connection = TEST_UTIL.getConnection();

  while (cluster.getLiveRegionServerThreads().isEmpty() &&
      cluster.getRegionServer(0) == null &&
      rs.getMetrics() == null) {
    Threads.sleep(100);
  }
  rs = cluster.getRegionServer(0);
  metricsRegionServer = rs.getMetrics();
  serverSource = metricsRegionServer.getMetricsSource();
}
 

@Test
public void testForCheckingIfEnableAndDisableWorksFineAfterSwitch()
    throws Exception {
  final int NUM_MASTERS = 2;
  final int NUM_REGIONS_TO_CREATE = 4;

  // Start the cluster
  log("Starting cluster");
  Configuration conf = HBaseConfiguration.create();
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(NUM_MASTERS).build();
  TEST_UTIL.startMiniCluster(option);
  MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
  log("Waiting for active/ready master");
  cluster.waitForActiveAndReadyMaster();

  // Create a table with regions
  final TableName tableName = TableName.valueOf(name.getMethodName());
  byte[] family = Bytes.toBytes("family");
  log("Creating table with " + NUM_REGIONS_TO_CREATE + " regions");
  Table ht = TEST_UTIL.createMultiRegionTable(tableName, family, NUM_REGIONS_TO_CREATE);
  int numRegions = -1;
  try (RegionLocator r = TEST_UTIL.getConnection().getRegionLocator(tableName)) {
    numRegions = r.getStartKeys().length;
  }
  numRegions += 1; // catalogs
  log("Waiting for no more RIT\n");
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);
  log("Disabling table\n");
  TEST_UTIL.getAdmin().disableTable(tableName);

  NavigableSet<String> regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
  assertEquals("The number of regions for the table tableRestart should be 0 and only" +
    "the catalog table should be present.", 1, regions.size());

  List<MasterThread> masterThreads = cluster.getMasterThreads();
  MasterThread activeMaster = null;
  if (masterThreads.get(0).getMaster().isActiveMaster()) {
    activeMaster = masterThreads.get(0);
  } else {
    activeMaster = masterThreads.get(1);
  }
  activeMaster.getMaster().stop(
      "stopping the active master so that the backup can become active");
  cluster.hbaseCluster.waitOnMaster(activeMaster);
  cluster.waitForActiveAndReadyMaster();

  assertTrue("The table should not be in enabled state",
      cluster.getMaster().getTableStateManager().isTableState(
      TableName.valueOf(name.getMethodName()), TableState.State.DISABLED,
      TableState.State.DISABLING));
  log("Enabling table\n");
  // Need a new Admin, the previous one is on the old master
  Admin admin = TEST_UTIL.getAdmin();
  admin.enableTable(tableName);
  admin.close();
  log("Waiting for no more RIT\n");
  TEST_UTIL.waitUntilNoRegionsInTransition(60000);
  log("Verifying there are " + numRegions + " assigned on cluster\n");
  regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
  assertEquals("The assigned regions were not onlined after master" +
    " switch except for the catalog table.", 5, regions.size());
  assertTrue("The table should be in enabled state", cluster.getMaster().getTableStateManager()
    .isTableState(TableName.valueOf(name.getMethodName()), TableState.State.ENABLED));
  ht.close();
  TEST_UTIL.shutdownMiniCluster();
}
 

/**
 * Test meta in transition when master failover.
 * This test used to manipulate region state up in zk. That is not allowed any more in hbase2
 * so I removed that messing. That makes this test anemic.
 */
@Test
public void testMetaInTransitionWhenMasterFailover() throws Exception {
  // Start the cluster
  HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.startMiniCluster();
  try {
    MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
    LOG.info("Cluster started");

    HMaster activeMaster = cluster.getMaster();
    ServerName metaServerName = cluster.getServerHoldingMeta();
    HRegionServer hrs = cluster.getRegionServer(metaServerName);

    // Now kill master, meta should remain on rs, where we placed it before.
    LOG.info("Aborting master");
    activeMaster.abort("test-kill");
    cluster.waitForMasterToStop(activeMaster.getServerName(), 30000);
    LOG.info("Master has aborted");

    // meta should remain where it was
    RegionState metaState = MetaTableLocator.getMetaRegionState(hrs.getZooKeeper());
    assertEquals("hbase:meta should be online on RS",
        metaState.getServerName(), metaServerName);
    assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState());

    // Start up a new master
    LOG.info("Starting up a new master");
    activeMaster = cluster.startMaster().getMaster();
    LOG.info("Waiting for master to be ready");
    cluster.waitForActiveAndReadyMaster();
    LOG.info("Master is ready");

    // ensure meta is still deployed on RS
    metaState = MetaTableLocator.getMetaRegionState(activeMaster.getZooKeeper());
    assertEquals("hbase:meta should be online on RS",
        metaState.getServerName(), metaServerName);
    assertEquals("hbase:meta should be online on RS", State.OPEN, metaState.getState());

    // Done, shutdown the cluster
  } finally {
    TEST_UTIL.shutdownMiniCluster();
  }
}
 

@Test
public void testReducerNumEstimation() throws Exception{
    // Skip the test for Tez. Tez use a different mechanism.
    // Equivalent test is in TestTezAutoParallelism
    Assume.assumeTrue("Skip this test for TEZ",
            Util.isMapredExecType(cluster.getExecType()));
    // use the estimation
    Configuration conf = HBaseConfiguration.create(new Configuration());
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    int clientPort = util.startMiniZKCluster().getClientPort();
    util.startMiniHBaseCluster(1, 1);

    String query = "a = load '/passwd';" +
            "b = group a by $0;" +
            "store b into 'output';";
    PigServer ps = new PigServer(cluster.getExecType(), cluster.getProperties());
    PhysicalPlan pp = Util.buildPp(ps, query);
    MROperPlan mrPlan = Util.buildMRPlan(pp, pc);

    pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
    pc.getConf().setProperty("pig.exec.reducers.max", "10");
    pc.getConf().setProperty(HConstants.ZOOKEEPER_CLIENT_PORT, Integer.toString(clientPort));
    ConfigurationValidator.validatePigProperties(pc.getProperties());
    conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    JobControlCompiler jcc = new JobControlCompiler(pc, conf);
    JobControl jc=jcc.compile(mrPlan, "Test");
    Job job = jc.getWaitingJobs().get(0);
    long reducer=Math.min((long)Math.ceil(new File("test/org/apache/pig/test/data/passwd").length()/100.0), 10);

    Util.assertParallelValues(-1, -1, reducer, reducer, job.getJobConf());

    // use the PARALLEL key word, it will override the estimated reducer number
    query = "a = load '/passwd';" +
            "b = group a by $0 PARALLEL 2;" +
            "store b into 'output';";
    pp = Util.buildPp(ps, query);
    mrPlan = Util.buildMRPlan(pp, pc);

    pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
    pc.getConf().setProperty("pig.exec.reducers.max", "10");
    ConfigurationValidator.validatePigProperties(pc.getProperties());
    conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    jcc = new JobControlCompiler(pc, conf);
    jc=jcc.compile(mrPlan, "Test");
    job = jc.getWaitingJobs().get(0);

    Util.assertParallelValues(-1, 2, -1, 2, job.getJobConf());

    final byte[] COLUMNFAMILY = Bytes.toBytes("pig");
    util.createTable(Bytes.toBytesBinary("test_table"), COLUMNFAMILY);

    // the estimation won't take effect when it apply to non-dfs or the files doesn't exist, such as hbase
    query = "a = load 'hbase://test_table' using org.apache.pig.backend.hadoop.hbase.HBaseStorage('c:f1 c:f2');" +
            "b = group a by $0 ;" +
            "store b into 'output';";
    pp = Util.buildPp(ps, query);
    mrPlan = Util.buildMRPlan(pp, pc);

    pc.getConf().setProperty("pig.exec.reducers.bytes.per.reducer", "100");
    pc.getConf().setProperty("pig.exec.reducers.max", "10");

    ConfigurationValidator.validatePigProperties(pc.getProperties());
    conf = ConfigurationUtil.toConfiguration(pc.getProperties());
    jcc = new JobControlCompiler(pc, conf);
    jc=jcc.compile(mrPlan, "Test");
    job = jc.getWaitingJobs().get(0);

    Util.assertParallelValues(-1, -1, -1, 1, job.getJobConf());

    util.deleteTable(Bytes.toBytesBinary("test_table"));
    // In HBase 0.90.1 and above we can use util.shutdownMiniHBaseCluster()
    // here instead.
    MiniHBaseCluster hbc = util.getHBaseCluster();
    if (hbc != null) {
        hbc.shutdown();
        hbc.join();
    }
    util.shutdownMiniZKCluster();
}
 

/**
 * Verifies that the given tables each have a single region and are on
 * different region servers. If they are on the same server moves tableName2
 * to the other region server.
 */
private void ensureTablesOnDifferentRegionServers(String tableName1, String tableName2) throws Exception  {
	byte[] table1 = Bytes.toBytes(tableName1);
	byte[] table2 = Bytes.toBytes(tableName2);
	Admin admin = driver.getConnectionQueryServices(getUrl(), TEST_PROPERTIES).getAdmin();
	HBaseTestingUtility util = getUtility();
	MiniHBaseCluster cluster = util.getHBaseCluster();
	HMaster master = cluster.getMaster();
	AssignmentManager am = master.getAssignmentManager();
  
	// verify there is only a single region for data table
	List<RegionInfo> tableRegions = admin.getRegions(TableName.valueOf(table1));
	assertEquals("Expected single region for " + table1, tableRegions.size(), 1);
	RegionInfo hri1 = tableRegions.get(0);
  
	// verify there is only a single region for index table
	tableRegions = admin.getRegions(TableName.valueOf(table2));
	RegionInfo hri2 = tableRegions.get(0);
	assertEquals("Expected single region for " + table2, tableRegions.size(), 1);
  
	ServerName serverName1 = am.getRegionStates().getRegionServerOfRegion(hri1);
	ServerName serverName2 = am.getRegionStates().getRegionServerOfRegion(hri2);
  
	// if data table and index table are on same region server, move the index table to the other region server
	if (serverName1.equals(serverName2)) {
	    HRegionServer server1 = util.getHBaseCluster().getRegionServer(0);
	    HRegionServer server2 = util.getHBaseCluster().getRegionServer(1);
	    HRegionServer dstServer = null;
	    HRegionServer srcServer = null;
	    if (server1.getServerName().equals(serverName2)) {
	        dstServer = server2;
	        srcServer = server1;
	    } else {
	        dstServer = server1;
	        srcServer = server2;
	    }
	    byte[] encodedRegionNameInBytes = hri2.getEncodedNameAsBytes();
	    admin.move(encodedRegionNameInBytes, Bytes.toBytes(dstServer.getServerName().getServerName()));
	    while (dstServer.getOnlineRegion(hri2.getRegionName()) == null
	            || dstServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
	            || srcServer.getRegionsInTransitionInRS().containsKey(encodedRegionNameInBytes)
	            || master.getAssignmentManager().getRegionStates().isRegionInTransition(hri2)) {
	        // wait for the move to be finished
	        Thread.sleep(1);
	    }
	}
  
	hri1 = admin.getRegions(TableName.valueOf(table1)).get(0);
	serverName1 = am.getRegionStates().getRegionServerOfRegion(hri1);
	hri2 = admin.getRegions(TableName.valueOf(table2)).get(0);
	serverName2 = am.getRegionStates().getRegionServerOfRegion(hri2);

	// verify index and data tables are on different servers
	assertNotEquals("Tables " + tableName1 + " and " + tableName2 + " should be on different region servers", serverName1, serverName2);
}