Java Code Examples for org.apache.hadoop.hbase.HBaseTestingUtility#startMiniCluster()

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf1.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}
 

@Before
  public void setUp() throws Exception {
      final Configuration hbaseConf = HBaseConfiguration.create();
hbaseConf.setInt("replication.stats.thread.period.seconds", 5);
      hbaseConf.setLong("replication.sleep.before.failover", 2000);
      hbaseConf.setInt("replication.source.maxretriesmultiplier", 10);
      hbaseConf.setBoolean(HConstants.REPLICATION_ENABLE_KEY, true);

      // add kafka properties. we prefix each property with kafka
addKafkaProperties(hbaseConf);

      utility = new HBaseTestingUtility(hbaseConf);
      utility.startMiniCluster();
      numRegionServers = utility.getHBaseCluster().getRegionServerThreads().size();

      // setup kafka
      kafkaServer = new KafkaServer(utility.getZkCluster().getClientPort(), 9092);

  }
 

@Before
public void setUpBefore() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.getConfiguration().setInt("dfs.datanode.max.xceivers", 9192);
  TEST_UTIL.startMiniCluster(3);
  conf = TEST_UTIL.getConfiguration();
  this.connection = ConnectionFactory.createConnection(conf);
  assertEquals(0, TEST_UTIL.getAdmin().listTableDescriptors().size());

  // setup the table
  table = TableName.valueOf(TABLE_BASE + "-" + tableIdx);
  tableIdx++;
  htbl = setupTable(table);
  populateTable(htbl);
  assertEquals(5, scanMeta());
  LOG.info("Table " + table + " has " + tableRowCount(conf, table)
      + " entries.");
  assertEquals(16, tableRowCount(conf, table));
  TEST_UTIL.getAdmin().disableTable(table);
  assertEquals(1, TEST_UTIL.getAdmin().listTableDescriptors().size());
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Set true to filter replication edits for dropped table
  conf1.setBoolean(REPLICATION_DROP_ON_DELETED_COLUMN_FAMILY_KEY, true);
  conf1.set(ZOOKEEPER_ZNODE_PARENT, "/1");
  conf1.setInt("replication.source.nb.capacity", 1);
  utility1 = new HBaseTestingUtility(conf1);
  utility1.startMiniZKCluster();
  MiniZooKeeperCluster miniZK = utility1.getZkCluster();
  conf1 = utility1.getConfiguration();

  conf2 = HBaseConfiguration.create(conf1);
  conf2.set(ZOOKEEPER_ZNODE_PARENT, "/2");
  utility2 = new HBaseTestingUtility(conf2);
  utility2.setZkCluster(miniZK);

  utility1.startMiniCluster(1);
  utility2.startMiniCluster(1);

  admin1 = utility1.getAdmin();
  admin2 = utility2.getAdmin();
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  Configuration conf = TEST_UTIL.getConfiguration();
  // Up the handlers; this test needs more than usual.
  conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
  enableSecurity(conf);
  verifyConfiguration(conf);

  // We expect 0.98 scanning semantics
  conf.setBoolean(AccessControlConstants.CF_ATTRIBUTE_EARLY_OUT, false);

  TEST_UTIL.startMiniCluster();
  TEST_UTIL.waitTableEnabled(PermissionStorage.ACL_TABLE_NAME.getName(), 50000);

  READER = User.createUserForTesting(conf, "reader", new String[0]);
  LIMITED = User.createUserForTesting(conf, "limited", new String[0]);
  DENIED = User.createUserForTesting(conf, "denied", new String[0]);
}
 

@Before
@Override
public void setUp() throws Exception {
  utility = new HBaseTestingUtility();
  utility.getConfiguration().setInt("hbase.hfile.compaction.discharger.interval", 10);
  utility.startMiniCluster();
  admin = utility.getAdmin();
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  testUtil = new HBaseTestingUtility();
  conf = testUtil.getConfiguration();

  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
  dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();

  conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
  conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());

  conf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5);
  
  // Tune down the connection thread pool size
  conf.setInt("hbase.hconnection.threads.core", 5);
  conf.setInt("hbase.hconnection.threads.max", 10);
  // Tunn down handler threads in regionserver
  conf.setInt("hbase.regionserver.handler.count", 10);

  // Set to random port
  conf.setInt("hbase.master.port", 0);
  conf.setInt("hbase.master.info.port", 0);
  conf.setInt("hbase.regionserver.port", 0);
  conf.setInt("hbase.regionserver.info.port", 0);

  testUtil.startMiniCluster();
  hBaseAdmin = testUtil.getHBaseAdmin();
  conn = testUtil.getConnection();
  txStateStorage = new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf), new TxMetricsCollector());
  txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
  txManager.startAndWait();
}
 

@BeforeAll
public static void startHBase() throws Exception {
  Configuration config = HBaseConfiguration.create();
  config.set("hbase.master.hostname", "localhost");
  config.set("hbase.regionserver.hostname", "localhost");
  util = new HBaseTestingUtility(config);
  util.startMiniCluster();
  admin = util.getHBaseAdmin();
  // create the table
  table = util.createTable(Bytes.toBytes(tableName), cf);
  util.waitTableEnabled(table.getName());
  // setup the client
  client = new HBaseClient((c,t) -> table, table.getConfiguration(), tableName);
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  TEST_UTIL.getConfiguration().set(
      RpcServerFactory.CUSTOM_RPC_SERVER_IMPL_CONF_KEY,
      NettyRpcServer.class.getName());
  TEST_UTIL.startMiniCluster();
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  testUtil = new HBaseTestingUtility();
  conf = testUtil.getConfiguration();

  conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, tmpFolder.newFolder().getAbsolutePath());
  dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();

  conf.unset(TxConstants.Manager.CFG_TX_HDFS_USER);
  conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());

  conf.setLong(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5);

  // Tune down the connection thread pool size
  conf.setInt("hbase.hconnection.threads.core", 5);
  conf.setInt("hbase.hconnection.threads.max", 10);
  // Tunn down handler threads in regionserver
  conf.setInt("hbase.regionserver.handler.count", 10);

  // Set to random port
  conf.setInt("hbase.master.port", 0);
  conf.setInt("hbase.master.info.port", 0);
  conf.setInt("hbase.regionserver.port", 0);
  conf.setInt("hbase.regionserver.info.port", 0);

  testUtil.startMiniCluster();
  hBaseAdmin = testUtil.getHBaseAdmin();
  txStateStorage =
      new HDFSTransactionStateStorage(conf, new SnapshotCodecProvider(conf),
          new TxMetricsCollector());
  txManager = new TransactionManager(conf, txStateStorage, new TxMetricsCollector());
  txManager.startAndWait();
}
 

@BeforeClass
public static synchronized void setUp() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    hbaseTestUtil = new HBaseTestingUtility(conf);
    setUpConfigForMiniCluster(conf);
    conf.set(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB, QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
    hbaseTestUtil.startMiniCluster();
    // establish url and quorum. Need to use PhoenixDriver and not PhoenixTestDriver
    zkQuorum = "localhost:" + hbaseTestUtil.getZkCluster().getClientPort();
    url = PhoenixRuntime.JDBC_PROTOCOL + PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR + zkQuorum;
    DriverManager.registerDriver(PhoenixDriver.INSTANCE);
}
 

@Before
public void setupTest() throws Exception {
  final TableName tableName = TableName.valueOf("tableName");
  TEST_UTIL = new HBaseTestingUtility();
  CONF = TEST_UTIL.getConfiguration();
  THRESHOLD = CONF.getInt(RSRpcServices.BATCH_ROWS_THRESHOLD_NAME,
    RSRpcServices.BATCH_ROWS_THRESHOLD_DEFAULT);
  CONF.setBoolean("hbase.rpc.rows.size.threshold.reject", rejectLargeBatchOp);
  TEST_UTIL.startMiniCluster();
  TEST_UTIL.createTable(tableName, TEST_FAM);
  RS = TEST_UTIL.getRSForFirstRegionInTable(tableName);
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Stack up three coprocessors just so I can check bypass skips subsequent calls.
  Configuration conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      new String [] {TestCoprocessor.class.getName(),
        TestCoprocessor2.class.getName(),
        TestCoprocessor3.class.getName()});
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();
}
 

@BeforeClass
public static void beforeClass() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  // Set the scanner lease to 20min, so the scanner can't be closed by RegionServer
  TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 1200000);
  TEST_UTIL.getConfiguration()
      .setInt(CompactionConfiguration.HBASE_HSTORE_COMPACTION_MIN_KEY, 100);
  TEST_UTIL.getConfiguration().set("dfs.blocksize", "64000");
  TEST_UTIL.getConfiguration().set("dfs.namenode.fs-limits.min-block-size", "1024");
  TEST_UTIL.getConfiguration().set(TimeToLiveHFileCleaner.TTL_CONF_KEY, "0");
  TEST_UTIL.startMiniCluster(RS_NUMBER);
  admin = TEST_UTIL.getAdmin();
}
 

@BeforeClass
public static synchronized void setUp() throws Exception {
    hbaseTestUtil = new HBaseTestingUtility();
    conf = hbaseTestUtil.getConfiguration();
    setUpConfigForMiniCluster(conf);
    conf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, "/hbase-test");
    hbaseTestUtil.startMiniCluster();
    Class.forName(PhoenixDriver.class.getName());
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  UTIL = new HBaseTestingUtility();
  UTIL.startMiniCluster();
  HRavenTestUtil.createSchema(UTIL);

  hbaseConnection =
      ConnectionFactory.createConnection(UTIL.getConfiguration());
  ht = hbaseConnection
      .getTable(TableName.valueOf(HdfsConstants.HDFS_USAGE_TABLE));

  ownerList.add(0, "user1");
  ownerList.add(1, "user2");
  ownerList.add(2, "user3");
  ownerList.add(3, "user1");
  ownerList.add(4, "user2");
  ownerList.add(5, "user6");

  pathList.add(0, "dir2");
  pathList.add(1, "dir3");
  pathList.add(2, "dir4");
  pathList.add(3, "abc");
  pathList.add(4, "abcde2");
  pathList.add(5, "dir5");

  fc.add(0, 10L);
  fc.add(1, 120L);
  fc.add(2, 12345L);
  fc.add(3, 8000L);
  fc.add(4, 98L);
  fc.add(5, 99999L);

  sc.add(0, 1024567L);
  sc.add(1, 9000120L);
  sc.add(2, 992112345L);
  sc.add(3, 128000L);
  sc.add(4, 98L);
  sc.add(5, 811111199999L);

  ac.add(0, 210L);
  ac.add(1, 2120L);
  ac.add(2, 1235L);
  ac.add(3, 18000L);
  ac.add(4, 22298L);
  ac.add(5, 9L);

  dc.add(0, 3L);
  dc.add(1, 10L);
  dc.add(2, 45L);
  dc.add(3, 100L);
  dc.add(4, 18L);
  dc.add(5, 1109L);
}
 

/**
 * Test that we correctly abort a RegionServer when we run tests with an unsupported HBase
 * version. The 'completeness' of this test requires that we run the test with both a version of
 * HBase that wouldn't be supported with WAL Compression. Currently, this is the default version
 * (0.94.4) so just running 'mvn test' will run the full test. However, this test will not fail
 * when running against a version of HBase with WALCompression enabled. Therefore, to fully test
 * this functionality, we need to run the test against both a supported and an unsupported version
 * of HBase (as long as we want to support an version of HBase that doesn't support custom WAL
 * Codecs).
 * @throws Exception on failure
 */
@Test(timeout = 300000 /* 5 mins */)
public void testDoesNotStartRegionServerForUnsupportedCompressionAndVersion() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    IndexTestingUtils.setupConfig(conf);
    // enable WAL Compression
    conf.setBoolean(HConstants.ENABLE_WAL_COMPRESSION, true);

    // check the version to see if it isn't supported
    String version = VersionInfo.getVersion();
    boolean supported = false;
    if (Indexer.validateVersion(version, conf) == null) {
        supported = true;
    }

    // start the minicluster
    HBaseTestingUtility util = new HBaseTestingUtility(conf);
    // set replication required parameter
    ConfigUtil.setReplicationConfigIfAbsent(conf);
    try {
        util.startMiniCluster();

        // setup the primary table
        @SuppressWarnings("deprecation")
        HTableDescriptor desc = new HTableDescriptor(
                "testDoesNotStartRegionServerForUnsupportedCompressionAndVersion");
        byte[] family = Bytes.toBytes("f");
        desc.addFamily(new HColumnDescriptor(family));

        // enable indexing to a non-existant index table
        String indexTableName = "INDEX_TABLE";
        ColumnGroup fam1 = new ColumnGroup(indexTableName);
        fam1.add(new CoveredColumn(family, CoveredColumn.ALL_QUALIFIERS));
        CoveredColumnIndexSpecifierBuilder builder = new CoveredColumnIndexSpecifierBuilder();
        builder.addIndexGroup(fam1);
        builder.build(desc);

        // get a reference to the regionserver, so we can ensure it aborts
        HRegionServer server = util.getMiniHBaseCluster().getRegionServer(0);

        // create the primary table
        HBaseAdmin admin = util.getHBaseAdmin();
        if (supported) {
            admin.createTable(desc);
            assertFalse("Hosting regeion server failed, even the HBase version (" + version
                    + ") supports WAL Compression.", server.isAborted());
        } else {
            admin.createTableAsync(desc, null);

            // wait for the regionserver to abort - if this doesn't occur in the timeout, assume its
            // broken.
            while (!server.isAborted()) {
                LOG.debug("Waiting on regionserver to abort..");
            }
        }
    } finally {
        // cleanup
        util.shutdownMiniCluster();
    }
}
 

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  util = new HBaseTestingUtility();
  util.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
  util.startMiniCluster();
}
 

@BeforeClass
public static void setup() throws Exception {
  testUtil = new HBaseTestingUtility();
  Configuration conf = testUtil.getConfiguration();
  conf.setBoolean(TxConstants.Manager.CFG_DO_PERSIST, false);
  conf.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, tmpFolder.newFolder().getAbsolutePath());

  // Tune down the connection thread pool size
  conf.setInt("hbase.hconnection.threads.core", 5);
  conf.setInt("hbase.hconnection.threads.max", 10);
  // Tunn down handler threads in regionserver
  conf.setInt("hbase.regionserver.handler.count", 10);

  // Set to random port
  conf.setInt("hbase.master.port", 0);
  conf.setInt("hbase.master.info.port", 0);
  conf.setInt("hbase.regionserver.port", 0);
  conf.setInt("hbase.regionserver.info.port", 0);

  testUtil.startMiniCluster();

  String zkClusterKey = testUtil.getClusterKey(); // hostname:clientPort:parentZnode
  String zkQuorum = zkClusterKey.substring(0, zkClusterKey.lastIndexOf(':'));
  LOG.info("Zookeeper Quorum is running at {}", zkQuorum);
  conf.set(TxConstants.Service.CFG_DATA_TX_ZOOKEEPER_QUORUM, zkQuorum);

  Injector injector = Guice.createInjector(
      new ConfigModule(conf),
      new ZKModule(),
      new DiscoveryModules().getDistributedModules(),
      Modules.override(new TransactionModules().getDistributedModules())
          .with(new AbstractModule() {
            @Override
            protected void configure() {
              bind(TransactionStateStorage.class).to(InMemoryTransactionStateStorage.class).in(Scopes.SINGLETON);
            }
          }),
      new TransactionClientModule()
  );

  zkClientService = injector.getInstance(ZKClientService.class);
  zkClientService.startAndWait();

  // start a tx server
  txService = injector.getInstance(TransactionService.class);
  try {
    LOG.info("Starting transaction service");
    txService.startAndWait();
  } catch (Exception e) {
    LOG.error("Failed to start service: ", e);
    throw e;
  }

  Tests.waitForTxReady(injector.getInstance(TransactionSystemClient.class));
}
 

@BeforeClass
public static void setupBeforeClass() throws Exception {
  UTIL = new HBaseTestingUtility();
  UTIL.startMiniCluster();
}