org.apache.hadoop.hbase.coprocessor.CoprocessorHost Java Examples

The following examples show how to use org.apache.hadoop.hbase.coprocessor.CoprocessorHost. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HMaster.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to
 * automatically remove quotas for a table when that table is deleted.
 */
@VisibleForTesting
public void updateConfigurationForQuotasObserver(Configuration conf) {
  // We're configured to not delete quotas on table deletion, so we don't need to add the obs.
  if (!conf.getBoolean(
        MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
        MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
    return;
  }
  String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
  final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
  String[] updatedCoprocs = new String[length + 1];
  if (length > 0) {
    System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
  }
  updatedCoprocs[length] = MasterQuotasObserver.class.getName();
  conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
}
 
Example #2
Source File: TestNamespaceAuditor.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void before() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
  conf.setStrings(
    CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
    MasterSyncObserver.class.getName(), CPMasterObserver.class.getName());
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
  conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
  conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
    RegionServerObserver.class);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numMasters(2).build();
  UTIL.startMiniCluster(option);
  waitForQuotaInitialize(UTIL);
  ADMIN = UTIL.getAdmin();
}
 
Example #3
Source File: TestAsyncTableUseMetaReplicas.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setInt(HConstants.META_REPLICAS_NUM, 3);
  conf.setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    FailPrimaryMetaScanCp.class.getName());
  UTIL.startMiniCluster(3);
  try (ConnectionRegistry registry = ConnectionRegistryFactory.getRegistry(conf)) {
    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry);
  }
  try (Table table = UTIL.createTable(TABLE_NAME, FAMILY)) {
    table.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE));
  }
  UTIL.flush(TableName.META_TABLE_NAME);
  // wait for the store file refresh so we can read the region location from secondary meta
  // replicas
  Thread.sleep(2000);
}
 
Example #4
Source File: TestMasterReplication.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  baseConfiguration = HBaseConfiguration.create();
  // smaller block size and capacity to trigger more operations
  // and test them
  baseConfiguration.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
  baseConfiguration.setInt("replication.source.size.capacity", 1024);
  baseConfiguration.setLong("replication.source.sleepforretries", 100);
  baseConfiguration.setInt("hbase.regionserver.maxlogs", 10);
  baseConfiguration.setLong("hbase.master.logcleaner.ttl", 10);
  baseConfiguration.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
  baseConfiguration.set("hbase.replication.source.fs.conf.provider",
    TestSourceFSConfigurationProvider.class.getCanonicalName());
  baseConfiguration.set(HConstants.REPLICATION_CLUSTER_ID, "12345");
  baseConfiguration.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  baseConfiguration.setStrings(
      CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      CoprocessorCounter.class.getName());
  table = TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName)
          .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName1)
          .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();
}
 
Example #5
Source File: SnapshotWithAclTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  // Enable security
  enableSecurity(conf);
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName());
  // Verify enableSecurity sets up what we require
  verifyConfiguration(conf);
  // Enable EXEC permission checking
  conf.setBoolean(AccessControlConstants.EXEC_PERMISSION_CHECKS_KEY, true);
  TEST_UTIL.startMiniCluster();
  TEST_UTIL.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME);
  MasterCoprocessorHost cpHost =
    TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterCoprocessorHost();
  cpHost.load(AccessController.class, Coprocessor.PRIORITY_HIGHEST, conf);

  USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
  USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
  USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
  USER_NONE = User.createUserForTesting(conf, "usernone", new String[0]);
}
 
Example #6
Source File: TestRefreshHFilesBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static void setUp(String regionImpl) {
  try {
    CONF.set(HConstants.REGION_IMPL, regionImpl);
    CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);

    CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
            RefreshHFilesEndpoint.class.getName());
    cluster = HTU.startMiniCluster(NUM_RS);

    // Create table
    table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);

    // this will create 2 regions spread across slaves
    HTU.loadNumericRows(table, FAMILY, 1, 20);
    HTU.flush(TABLE_NAME);
  } catch (Exception ex) {
    LOG.error("Couldn't finish setup", ex);
  }
}
 
Example #7
Source File: TestMultiParallel.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  // Uncomment the following lines if more verbosity is needed for
  // debugging (see HBASE-12285 for details).
  //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
  //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
  //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
  UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
      KeyValueCodec.class.getCanonicalName());
  // Disable table on master for now as the feature is broken
  //UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
  // We used to ask for system tables on Master exclusively but not needed by test and doesn't
  // work anyways -- so commented out.
  // UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
  UTIL.getConfiguration()
      .set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyMasterObserver.class.getName());
  UTIL.startMiniCluster(slaves);
  Table t = UTIL.createMultiRegionTable(TEST_TABLE, Bytes.toBytes(FAMILY));
  UTIL.waitTableEnabled(TEST_TABLE);
  t.close();
  CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration());
  assertTrue(MyMasterObserver.start.get());
}
 
Example #8
Source File: TestAvoidCellReferencesIntoShippedBlocks.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * @throws java.lang.Exception
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  ROWS[0] = ROW;
  ROWS[1] = ROW1;
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    MultiRowMutationEndpoint.class.getName());
  conf.setInt("hbase.regionserver.handler.count", 20);
  conf.setInt("hbase.bucketcache.size", 400);
  conf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
  conf.setInt("hbase.hstore.compactionThreshold", 7);
  conf.setFloat("hfile.block.cache.size", 0.2f);
  conf.setFloat("hbase.regionserver.global.memstore.size", 0.1f);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);// do not retry
  conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 500000);
  FAMILIES_1[0] = FAMILY;
  TEST_UTIL.startMiniCluster(SLAVES);
  compactReadLatch = new CountDownLatch(1);
}
 
Example #9
Source File: TestClientClusterMetrics.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers metrics");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 
Example #10
Source File: SecureTestUtil.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static void verifyConfiguration(Configuration conf) {
  String coprocs = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
  boolean accessControllerLoaded = false;
  for (String coproc : coprocs.split(",")) {
    try {
      accessControllerLoaded = AccessController.class.isAssignableFrom(Class.forName(coproc));
      if (accessControllerLoaded) break;
    } catch (ClassNotFoundException cnfe) {
    }
  }
  if (!(conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY).contains(
      AccessController.class.getName())
      && accessControllerLoaded && conf.get(
      CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY).contains(
      AccessController.class.getName()))) {
    throw new RuntimeException("AccessController is missing from a system coprocessor list");
  }
  if (conf.getInt(HFile.FORMAT_VERSION_KEY, 2) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
    throw new RuntimeException("Post 0.96 security features require HFile version >= 3");
  }

  if (!conf.getBoolean(User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY, false)) {
    throw new RuntimeException("Post 2.0.0 security features require set "
        + User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY + " to true");
  }
}
 
Example #11
Source File: RegionServerCoprocessorHost.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Override
public RegionServerCoprocessor checkAndGetInstance(Class<?> implClass)
    throws InstantiationException, IllegalAccessException {
  try {
    if (RegionServerCoprocessor.class.isAssignableFrom(implClass)) {
      return implClass.asSubclass(RegionServerCoprocessor.class).getDeclaredConstructor()
          .newInstance();
    } else {
      LOG.error("{} is not of type RegionServerCoprocessor. Check the configuration of {}",
          implClass.getName(), CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
      return null;
    }
  } catch (NoSuchMethodException | InvocationTargetException e) {
    throw (InstantiationException) new InstantiationException(implClass.getName()).initCause(e);
  }
}
 
Example #12
Source File: TestClientOperationInterrupt.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessor.class.getName());
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  Admin admin = util.getAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  Table ht = util.createTable(tableName, new byte[][]{dummy, test});

  Put p = new Put(row1);
  p.addColumn(dummy, dummy, dummy);
  ht.put(p);
}
 
Example #13
Source File: TestProcedurePriority.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  UTIL.getConfiguration().setLong(ProcedureExecutor.WORKER_KEEP_ALIVE_TIME_CONF_KEY, 5000);
  UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 4);
  UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MyCP.class.getName());
  UTIL.startMiniCluster(3);
  CORE_POOL_SIZE =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getCorePoolSize();
  TABLE_COUNT = 50 * CORE_POOL_SIZE;
  List<Future<?>> futures = new ArrayList<>();
  AsyncAdmin admin = UTIL.getAsyncConnection().getAdmin();
  Semaphore concurrency = new Semaphore(10);
  for (int i = 0; i < TABLE_COUNT; i++) {
    concurrency.acquire();
    futures.add(admin
      .createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE_NAME_PREFIX + i))
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build())
      .whenComplete((r, e) -> concurrency.release()));
  }
  for (Future<?> future : futures) {
    future.get(3, TimeUnit.MINUTES);
  }
  UTIL.getAdmin().balance(true);
  UTIL.waitUntilNoRegionsInTransition();
}
 
Example #14
Source File: VisibilityController.java    From hbase with Apache License 2.0 6 votes vote down vote up
/****************************** Region related hooks ******************************/

  @Override
  public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
    // Read the entire labels table and populate the zk
    if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
      this.labelsRegion = true;
      synchronized (this) {
        this.accessControllerAvailable = CoprocessorHost.getLoadedCoprocessors()
          .contains(AccessController.class.getName());
      }
      initVisibilityLabelService(e.getEnvironment());
    } else {
      checkAuths = e.getEnvironment().getConfiguration()
          .getBoolean(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION, false);
      initVisibilityLabelService(e.getEnvironment());
    }
  }
 
Example #15
Source File: TestRSGroupsBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
public static void setUpTestBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
  if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
    RSGroupUtil.enableRSGroup(conf);
  }
  if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
        CPMasterObserver.class.getName());
  } else {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
  }

  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
    NUM_SLAVES_BASE - 1);
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setInt("hbase.rpc.timeout", 100000);

  TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
  initialize();
}
 
Example #16
Source File: TestClientClusterStatus.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers status");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 
Example #17
Source File: TestAsyncAggregationClient.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    AggregateImplementation.class.getName());
  UTIL.startMiniCluster(3);
  byte[][] splitKeys = new byte[8][];
  for (int i = 111; i < 999; i += 111) {
    splitKeys[i / 111 - 1] = Bytes.toBytes(String.format("%03d", i));
  }
  UTIL.createTable(TABLE_NAME, CF, splitKeys);
  CONN = ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get();
  TABLE = CONN.getTable(TABLE_NAME);
  TABLE.putAll(LongStream.range(0, COUNT)
      .mapToObj(l -> new Put(Bytes.toBytes(String.format("%03d", l)))
          .addColumn(CF, CQ, Bytes.toBytes(l)).addColumn(CF, CQ2, Bytes.toBytes(l * l)))
      .collect(Collectors.toList())).get();
}
 
Example #18
Source File: AbstractTestFSWAL.java    From hbase with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Make block sizes small.
  TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
  // quicker heartbeat interval for faster DN death notification
  TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
  TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
  TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);

  // faster failover with cluster.shutdown();fs.close() idiom
  TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connect.max.retries", 1);
  TEST_UTIL.getConfiguration().setInt("dfs.client.block.recovery.retries", 1);
  TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connection.maxidletime", 500);
  TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
    SampleRegionWALCoprocessor.class.getName());
  TEST_UTIL.startMiniDFSCluster(3);

  CONF = TEST_UTIL.getConfiguration();
  FS = TEST_UTIL.getDFSCluster().getFileSystem();
}
 
Example #19
Source File: BackupManager.java    From hbase with Apache License 2.0 6 votes vote down vote up
/**
 * This method modifies the Region Server configuration in order to inject backup-related features
 * TESTs only.
 * @param conf configuration
 */
@VisibleForTesting
public static void decorateRegionServerConfiguration(Configuration conf) {
  if (!isBackupEnabled(conf)) {
    return;
  }

  String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY);
  String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
  if (classes == null) {
    conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass);
  } else if (!classes.contains(regionProcedureClass)) {
    conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY,
      classes + "," + regionProcedureClass);
  }
  String coproc = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
  String regionObserverClass = BackupObserver.class.getName();
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    (coproc == null ? "" : coproc + ",") + regionObserverClass);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Added region procedure manager: {}. Added region observer: {}",
      regionProcedureClass, regionObserverClass);
  }
}
 
Example #20
Source File: TestRegionServerAbort.java    From hbase with Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  testUtil = new HBaseTestingUtility();
  conf = testUtil.getConfiguration();
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      StopBlockingRegionObserver.class.getName());
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      StopBlockingRegionObserver.class.getName());
  // make sure we have multiple blocks so that the client does not prefetch all block locations
  conf.set("dfs.blocksize", Long.toString(100 * 1024));
  // prefetch the first block
  conf.set(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, Long.toString(100 * 1024));
  conf.set(HConstants.REGION_IMPL, ErrorThrowingHRegion.class.getName());

  testUtil.startMiniZKCluster();
  dfsCluster = testUtil.startMiniDFSCluster(2);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numRegionServers(2).build();
  cluster = testUtil.startMiniHBaseCluster(option);
}
 
Example #21
Source File: IndexLoadBalancerIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupCluster() throws Exception {
    final int NUM_RS = 4;
    Configuration conf = UTIL.getConfiguration();
    conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true);
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
        LoadBalancer.class);
    IndexTestingUtils.setupConfig(conf);
    // disable version checking, so we can test against whatever version of HBase happens to be
    // installed (right now, its generally going to be SNAPSHOT versions).
    conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
    // set replication required parameter
    ConfigUtil.setReplicationConfigIfAbsent(conf);
    UTIL.startMiniCluster(NUM_RS);
    admin = UTIL.getHBaseAdmin();
}
 
Example #22
Source File: MutableIndexFailureIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Before
public void doSetup() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    conf.setInt("hbase.client.retries.number", 2);
    conf.setInt("hbase.client.pause", 5000);
    conf.setInt("hbase.balancer.period", Integer.MAX_VALUE);
    conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0);
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
        LoadBalancer.class);
    util = new HBaseTestingUtility(conf);
    util.startMiniCluster(NUM_SLAVES);
    String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
    url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
            + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
    driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
}
 
Example #23
Source File: CustomSaslAuthenticationProviderTestBase.java    From hbase with Apache License 2.0 6 votes vote down vote up
private static void createBaseCluster(HBaseTestingUtility util, File keytabFile, MiniKdc kdc)
  throws Exception {
  String servicePrincipal = "hbase/localhost";
  String spnegoPrincipal = "HTTP/localhost";
  kdc.createPrincipal(keytabFile, servicePrincipal);
  util.startMiniZKCluster();

  HBaseKerberosUtils.setSecuredConfiguration(util.getConfiguration(),
    servicePrincipal + "@" + kdc.getRealm(), spnegoPrincipal + "@" + kdc.getRealm());
  HBaseKerberosUtils.setSSLConfiguration(util, SecureTestCluster.class);

  util.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    TokenProvider.class.getName());
  util.startMiniDFSCluster(1);
  Path rootdir = util.getDataTestDirOnTestFS("TestCustomSaslAuthenticationProvider");
  CommonFSUtils.setRootDir(util.getConfiguration(), rootdir);
}
 
Example #24
Source File: TestMasterHandlerFullWhenTransitRegion.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      DelayOpenCP.class.getName());
  //set handler number to 1.
  UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 1);
  UTIL.startMiniCluster(2);
  UTIL.createTable(TableName.valueOf(TABLENAME), "fa");
}
 
Example #25
Source File: TestMigrateRSGroupInfo.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  TEST_UTIL.getConfiguration().setClass(HConstants.MASTER_IMPL, HMasterForTest.class,
    HMaster.class);
  // confirm that we could enable rs group by setting the old CP.
  TEST_UTIL.getConfiguration().setBoolean(RSGroupUtil.RS_GROUP_ENABLED, false);
  TEST_UTIL.getConfiguration().set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
    RSGroupAdminEndpoint.class.getName());
  setUpTestBeforeClass();
  RS_GROUP_ADMIN_CLIENT = new RSGroupAdminClient(TEST_UTIL.getConnection());
  for (int i = 0; i < NUM_TABLES; i++) {
    TEST_UTIL.createTable(TableName.valueOf(TABLE_NAME_PREFIX + i), FAMILY);
  }
}
 
Example #26
Source File: TestMasterAbortAndRSGotKilled.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    DelayCloseCP.class.getName());
  UTIL.startMiniCluster(3);
  UTIL.getAdmin().balancerSwitch(false, true);
  UTIL.createTable(TABLE_NAME, CF);
  UTIL.waitTableAvailable(TABLE_NAME);
}
 
Example #27
Source File: TestStochasticBalancerJmxMetrics.java    From hbase with Apache License 2.0 5 votes vote down vote up
/**
 * Setup the environment for the test.
 */
@BeforeClass
public static void setupBeforeClass() throws Exception {

  conf = UTIL.getConfiguration();

  conf.setClass("hbase.util.ip.to.rack.determiner", MockMapping.class, DNSToSwitchMapping.class);
  conf.setFloat("hbase.master.balancer.stochastic.maxMovePercent", 0.75f);
  conf.setFloat("hbase.regions.slop", 0.0f);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
  Random rand = new Random();
  for (int i = 0; i < 10; i++) {
    do {
      int sign = i % 2 == 0 ? 1 : -1;
      connectorPort += sign * rand.nextInt(100);
    } while (!HBaseTestingUtility.available(connectorPort));
    try {
      conf.setInt("regionserver.rmi.registry.port", connectorPort);

      UTIL.startMiniCluster();
      break;
    } catch (Exception e) {
      LOG.debug("Encountered exception when starting cluster. Trying port " + connectorPort, e);
      try {
        // this is to avoid "IllegalStateException: A mini-cluster is already running"
        UTIL.shutdownMiniCluster();
      } catch (Exception ex) {
        LOG.debug("Encountered exception shutting down cluster", ex);
      }
    }
  }
}
 
Example #28
Source File: TestWALFactory.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  CommonFSUtils.setWALRootDir(TEST_UTIL.getConfiguration(), new Path("file:///tmp/wal"));
  // Make block sizes small.
  TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
  // needed for testAppendClose()
  // quicker heartbeat interval for faster DN death notification
  TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
  TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
  TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);

  // faster failover with cluster.shutdown();fs.close() idiom
  TEST_UTIL.getConfiguration()
      .setInt("hbase.ipc.client.connect.max.retries", 1);
  TEST_UTIL.getConfiguration().setInt(
      "dfs.client.block.recovery.retries", 1);
  TEST_UTIL.getConfiguration().setInt(
    "hbase.ipc.client.connection.maxidletime", 500);
  TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.timeout", 10000);
  TEST_UTIL.getConfiguration().setInt("hbase.lease.recovery.dfs.timeout", 1000);
  TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
      SampleRegionWALCoprocessor.class.getName());
  TEST_UTIL.startMiniDFSCluster(3);

  conf = TEST_UTIL.getConfiguration();
  cluster = TEST_UTIL.getDFSCluster();

  hbaseDir = TEST_UTIL.createRootDir();
  hbaseWALDir = TEST_UTIL.createWALRootDir();
}
 
Example #29
Source File: TestJMXListener.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  // Default RMI timeouts are too long. Make them short for test.
  System.setProperty("sun.rmi.transport.connectionTimeout", Integer.toString(5000));
  System.setProperty("sun.rmi.transport.tcp.handshakeTimeout", Integer.toString(5000));
  System.setProperty("sun.rmi.transport.tcp.responseTimeout", Integer.toString(5000));
  System.setProperty("sun.rmi.transport.tcp.readTimeout", Integer.toString(5000));
  Configuration conf = UTIL.getConfiguration();
  // To test what happens when the jmx listener can't put up its port, uncomment the below.
  BoundSocketMaker bsm = null; // new BoundSocketMaker(HBaseTestingUtility::randomFreePort);
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY, JMXListener.class.getName());
  CONNECTOR_PORT = bsm == null? HBaseTestingUtility.randomFreePort(): bsm.getPort();
  // Make sure the JMX listener is up before we proceed. If it is not up, retry. It may not
  // come up if there is a port clash/bind exception except its called something else in rmi.
  for (int i = 0; i < 10; i++) {
    conf.setInt("regionserver.rmi.registry.port", CONNECTOR_PORT);
    UTIL.startMiniCluster();
    // Make sure we can get make a JMX connection before proceeding. It may have failed setup
    // because of port clash/bind-exception. Deal with it here.
    JMXConnector connector = null;
    try {
      connector = JMXConnectorFactory.
        connect(JMXListener.buildJMXServiceURL(CONNECTOR_PORT, CONNECTOR_PORT));
      break;
    } catch (IOException ioe) {
      UTIL.shutdownMiniCluster();
      CONNECTOR_PORT = HBaseTestingUtility.randomFreePort();
    } finally {
      if (connector != null) {
        connector.close();
      }
    }
  }
  if (bsm != null) {
    bsm.close();
  }
}
 
Example #30
Source File: TestBulkLoadHFiles.java    From hbase with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  util.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, "");
  util.getConfiguration().setInt(BulkLoadHFiles.MAX_FILES_PER_REGION_PER_FAMILY,
    MAX_FILES_PER_REGION_PER_FAMILY);
  // change default behavior so that tag values are returned with normal rpcs
  util.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
    KeyValueCodecWithTags.class.getCanonicalName());
  util.startMiniCluster();

  setupNamespace();
}