Java Code Examples for org.apache.hadoop.hbase.coprocessor.CoprocessorHost

The following examples show how to use org.apache.hadoop.hbase.coprocessor.CoprocessorHost. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: phoenix   Source File: MutableIndexFailureIT.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void doSetup() throws Exception {
    Configuration conf = HBaseConfiguration.create();
    setUpConfigForMiniCluster(conf);
    conf.setInt("hbase.client.retries.number", 2);
    conf.setInt("hbase.client.pause", 5000);
    conf.setInt("hbase.balancer.period", Integer.MAX_VALUE);
    conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0);
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
        LoadBalancer.class);
    util = new HBaseTestingUtility(conf);
    util.startMiniCluster(NUM_SLAVES);
    String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
    url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
            + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
    driver = initAndRegisterDriver(url, ReadOnlyProps.EMPTY_PROPS);
}
 
Example 2
Source Project: phoenix   Source File: IndexLoadBalancerIT.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupCluster() throws Exception {
    final int NUM_RS = 4;
    Configuration conf = UTIL.getConfiguration();
    conf.setBoolean(HConstants.REGIONSERVER_INFO_PORT_AUTO, true);
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, IndexMasterObserver.class.getName());
    conf.setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS, IndexLoadBalancer.class,
        LoadBalancer.class);
    IndexTestingUtils.setupConfig(conf);
    // disable version checking, so we can test against whatever version of HBase happens to be
    // installed (right now, its generally going to be SNAPSHOT versions).
    conf.setBoolean(Indexer.CHECK_VERSION_CONF_KEY, false);
    // set replication required parameter
    ConfigUtil.setReplicationConfigIfAbsent(conf);
    UTIL.startMiniCluster(NUM_RS);
    admin = UTIL.getHBaseAdmin();
}
 
Example 3
Source Project: hbase   Source File: BackupManager.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * This method modifies the Region Server configuration in order to inject backup-related features
 * TESTs only.
 * @param conf configuration
 */
@VisibleForTesting
public static void decorateRegionServerConfiguration(Configuration conf) {
  if (!isBackupEnabled(conf)) {
    return;
  }

  String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY);
  String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
  if (classes == null) {
    conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass);
  } else if (!classes.contains(regionProcedureClass)) {
    conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY,
      classes + "," + regionProcedureClass);
  }
  String coproc = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
  String regionObserverClass = BackupObserver.class.getName();
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    (coproc == null ? "" : coproc + ",") + regionObserverClass);
  if (LOG.isDebugEnabled()) {
    LOG.debug("Added region procedure manager: {}. Added region observer: {}",
      regionProcedureClass, regionObserverClass);
  }
}
 
Example 4
Source Project: hbase   Source File: TestAsyncAggregationClient.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    AggregateImplementation.class.getName());
  UTIL.startMiniCluster(3);
  byte[][] splitKeys = new byte[8][];
  for (int i = 111; i < 999; i += 111) {
    splitKeys[i / 111 - 1] = Bytes.toBytes(String.format("%03d", i));
  }
  UTIL.createTable(TABLE_NAME, CF, splitKeys);
  CONN = ConnectionFactory.createAsyncConnection(UTIL.getConfiguration()).get();
  TABLE = CONN.getTable(TABLE_NAME);
  TABLE.putAll(LongStream.range(0, COUNT)
      .mapToObj(l -> new Put(Bytes.toBytes(String.format("%03d", l)))
          .addColumn(CF, CQ, Bytes.toBytes(l)).addColumn(CF, CQ2, Bytes.toBytes(l * l)))
      .collect(Collectors.toList())).get();
}
 
Example 5
Source Project: hbase   Source File: VisibilityController.java    License: Apache License 2.0 6 votes vote down vote up
/****************************** Region related hooks ******************************/

  @Override
  public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
    // Read the entire labels table and populate the zk
    if (e.getEnvironment().getRegion().getRegionInfo().getTable().equals(LABELS_TABLE_NAME)) {
      this.labelsRegion = true;
      synchronized (this) {
        this.accessControllerAvailable = CoprocessorHost.getLoadedCoprocessors()
          .contains(AccessController.class.getName());
      }
      initVisibilityLabelService(e.getEnvironment());
    } else {
      checkAuths = e.getEnvironment().getConfiguration()
          .getBoolean(VisibilityConstants.CHECK_AUTHS_FOR_MUTATION, false);
      initVisibilityLabelService(e.getEnvironment());
    }
  }
 
Example 6
Source Project: hbase   Source File: HMaster.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Adds the {@code MasterQuotasObserver} to the list of configured Master observers to
 * automatically remove quotas for a table when that table is deleted.
 */
@VisibleForTesting
public void updateConfigurationForQuotasObserver(Configuration conf) {
  // We're configured to not delete quotas on table deletion, so we don't need to add the obs.
  if (!conf.getBoolean(
        MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE,
        MasterQuotasObserver.REMOVE_QUOTA_ON_TABLE_DELETE_DEFAULT)) {
    return;
  }
  String[] masterCoprocs = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
  final int length = null == masterCoprocs ? 0 : masterCoprocs.length;
  String[] updatedCoprocs = new String[length + 1];
  if (length > 0) {
    System.arraycopy(masterCoprocs, 0, updatedCoprocs, 0, masterCoprocs.length);
  }
  updatedCoprocs[length] = MasterQuotasObserver.class.getName();
  conf.setStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, updatedCoprocs);
}
 
Example 7
Source Project: hbase   Source File: RegionServerCoprocessorHost.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public RegionServerCoprocessor checkAndGetInstance(Class<?> implClass)
    throws InstantiationException, IllegalAccessException {
  try {
    if (RegionServerCoprocessor.class.isAssignableFrom(implClass)) {
      return implClass.asSubclass(RegionServerCoprocessor.class).getDeclaredConstructor()
          .newInstance();
    } else {
      LOG.error("{} is not of type RegionServerCoprocessor. Check the configuration of {}",
          implClass.getName(), CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY);
      return null;
    }
  } catch (NoSuchMethodException | InvocationTargetException e) {
    throw (InstantiationException) new InstantiationException(implClass.getName()).initCause(e);
  }
}
 
Example 8
Source Project: hbase   Source File: TestMasterReplication.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setUp() throws Exception {
  baseConfiguration = HBaseConfiguration.create();
  // smaller block size and capacity to trigger more operations
  // and test them
  baseConfiguration.setInt("hbase.regionserver.hlog.blocksize", 1024 * 20);
  baseConfiguration.setInt("replication.source.size.capacity", 1024);
  baseConfiguration.setLong("replication.source.sleepforretries", 100);
  baseConfiguration.setInt("hbase.regionserver.maxlogs", 10);
  baseConfiguration.setLong("hbase.master.logcleaner.ttl", 10);
  baseConfiguration.setBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY, true);
  baseConfiguration.set("hbase.replication.source.fs.conf.provider",
    TestSourceFSConfigurationProvider.class.getCanonicalName());
  baseConfiguration.set(HConstants.REPLICATION_CLUSTER_ID, "12345");
  baseConfiguration.setLong(HConstants.THREAD_WAKE_FREQUENCY, 100);
  baseConfiguration.setStrings(
      CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      CoprocessorCounter.class.getName());
  table = TableDescriptorBuilder.newBuilder(tableName)
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName)
          .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(famName1)
          .setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build())
      .setColumnFamily(ColumnFamilyDescriptorBuilder.of(noRepfamName)).build();
}
 
Example 9
Source Project: hbase   Source File: TestRefreshHFilesBase.java    License: Apache License 2.0 6 votes vote down vote up
public static void setUp(String regionImpl) {
  try {
    CONF.set(HConstants.REGION_IMPL, regionImpl);
    CONF.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);

    CONF.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
            RefreshHFilesEndpoint.class.getName());
    cluster = HTU.startMiniCluster(NUM_RS);

    // Create table
    table = HTU.createTable(TABLE_NAME, FAMILY, SPLIT_KEY);

    // this will create 2 regions spread across slaves
    HTU.loadNumericRows(table, FAMILY, 1, 20);
    HTU.flush(TABLE_NAME);
  } catch (Exception ex) {
    LOG.error("Couldn't finish setup", ex);
  }
}
 
Example 10
Source Project: hbase   Source File: SecureTestUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void verifyConfiguration(Configuration conf) {
  String coprocs = conf.get(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY);
  boolean accessControllerLoaded = false;
  for (String coproc : coprocs.split(",")) {
    try {
      accessControllerLoaded = AccessController.class.isAssignableFrom(Class.forName(coproc));
      if (accessControllerLoaded) break;
    } catch (ClassNotFoundException cnfe) {
    }
  }
  if (!(conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY).contains(
      AccessController.class.getName())
      && accessControllerLoaded && conf.get(
      CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY).contains(
      AccessController.class.getName()))) {
    throw new RuntimeException("AccessController is missing from a system coprocessor list");
  }
  if (conf.getInt(HFile.FORMAT_VERSION_KEY, 2) < HFile.MIN_FORMAT_VERSION_WITH_TAGS) {
    throw new RuntimeException("Post 0.96 security features require HFile version >= 3");
  }

  if (!conf.getBoolean(User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY, false)) {
    throw new RuntimeException("Post 2.0.0 security features require set "
        + User.HBASE_SECURITY_AUTHORIZATION_CONF_KEY + " to true");
  }
}
 
Example 11
private static void createBaseCluster(HBaseTestingUtility util, File keytabFile, MiniKdc kdc)
  throws Exception {
  String servicePrincipal = "hbase/localhost";
  String spnegoPrincipal = "HTTP/localhost";
  kdc.createPrincipal(keytabFile, servicePrincipal);
  util.startMiniZKCluster();

  HBaseKerberosUtils.setSecuredConfiguration(util.getConfiguration(),
    servicePrincipal + "@" + kdc.getRealm(), spnegoPrincipal + "@" + kdc.getRealm());
  HBaseKerberosUtils.setSSLConfiguration(util, SecureTestCluster.class);

  util.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    TokenProvider.class.getName());
  util.startMiniDFSCluster(1);
  Path rootdir = util.getDataTestDirOnTestFS("TestCustomSaslAuthenticationProvider");
  CommonFSUtils.setRootDir(util.getConfiguration(), rootdir);
}
 
Example 12
Source Project: hbase   Source File: TestClientClusterStatus.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers status");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 
Example 13
Source Project: hbase   Source File: TestClientClusterMetrics.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  Configuration conf = HBaseConfiguration.create();
  conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyObserver.class.getName());
  UTIL = new HBaseTestingUtility(conf);
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(MASTERS).numRegionServers(SLAVES).numDataNodes(SLAVES).build();
  UTIL.startMiniCluster(option);
  CLUSTER = UTIL.getHBaseCluster();
  CLUSTER.waitForActiveAndReadyMaster();
  ADMIN = UTIL.getAdmin();
  // Kill one region server
  List<RegionServerThread> rsts = CLUSTER.getLiveRegionServerThreads();
  RegionServerThread rst = rsts.get(rsts.size() - 1);
  DEAD = rst.getRegionServer();
  DEAD.stop("Test dead servers metrics");
  while (rst.isAlive()) {
    Thread.sleep(500);
  }
}
 
Example 14
Source Project: hbase   Source File: TestMultiParallel.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  // Uncomment the following lines if more verbosity is needed for
  // debugging (see HBASE-12285 for details).
  //((Log4JLogger)RpcServer.LOG).getLogger().setLevel(Level.ALL);
  //((Log4JLogger)RpcClient.LOG).getLogger().setLevel(Level.ALL);
  //((Log4JLogger)ScannerCallable.LOG).getLogger().setLevel(Level.ALL);
  UTIL.getConfiguration().set(HConstants.RPC_CODEC_CONF_KEY,
      KeyValueCodec.class.getCanonicalName());
  // Disable table on master for now as the feature is broken
  //UTIL.getConfiguration().setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
  // We used to ask for system tables on Master exclusively but not needed by test and doesn't
  // work anyways -- so commented out.
  // UTIL.getConfiguration().setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
  UTIL.getConfiguration()
      .set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, MyMasterObserver.class.getName());
  UTIL.startMiniCluster(slaves);
  Table t = UTIL.createMultiRegionTable(TEST_TABLE, Bytes.toBytes(FAMILY));
  UTIL.waitTableEnabled(TEST_TABLE);
  t.close();
  CONNECTION = ConnectionFactory.createConnection(UTIL.getConfiguration());
  assertTrue(MyMasterObserver.start.get());
}
 
Example 15
/**
 * @throws java.lang.Exception
 */
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  ROWS[0] = ROW;
  ROWS[1] = ROW1;
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    MultiRowMutationEndpoint.class.getName());
  conf.setInt("hbase.regionserver.handler.count", 20);
  conf.setInt("hbase.bucketcache.size", 400);
  conf.setStrings(HConstants.BUCKET_CACHE_IOENGINE_KEY, "offheap");
  conf.setInt("hbase.hstore.compactionThreshold", 7);
  conf.setFloat("hfile.block.cache.size", 0.2f);
  conf.setFloat("hbase.regionserver.global.memstore.size", 0.1f);
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0);// do not retry
  conf.setInt(HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD, 500000);
  FAMILIES_1[0] = FAMILY;
  TEST_UTIL.startMiniCluster(SLAVES);
  compactReadLatch = new CountDownLatch(1);
}
 
Example 16
Source Project: hbase   Source File: TestClientOperationInterrupt.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = HBaseConfiguration.create();
  conf.setStrings(CoprocessorHost.USER_REGION_COPROCESSOR_CONF_KEY,
      TestCoprocessor.class.getName());
  util = new HBaseTestingUtility(conf);
  util.startMiniCluster();

  Admin admin = util.getAdmin();
  if (admin.tableExists(tableName)) {
    if (admin.isTableEnabled(tableName)) {
      admin.disableTable(tableName);
    }
    admin.deleteTable(tableName);
  }
  Table ht = util.createTable(tableName, new byte[][]{dummy, test});

  Put p = new Put(row1);
  p.addColumn(dummy, dummy, dummy);
  ht.put(p);
}
 
Example 17
Source Project: hbase   Source File: TestNamespaceAuditor.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void before() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, CustomObserver.class.getName());
  conf.setStrings(
    CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
    MasterSyncObserver.class.getName(), CPMasterObserver.class.getName());
  conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
  conf.setBoolean(QuotaUtil.QUOTA_CONF_KEY, true);
  conf.setClass("hbase.coprocessor.regionserver.classes", CPRegionServerObserver.class,
    RegionServerObserver.class);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numMasters(2).build();
  UTIL.startMiniCluster(option);
  waitForQuotaInitialize(UTIL);
  ADMIN = UTIL.getAdmin();
}
 
Example 18
Source Project: hbase   Source File: TestAsyncTableUseMetaReplicas.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  Configuration conf = UTIL.getConfiguration();
  conf.setInt(HConstants.META_REPLICAS_NUM, 3);
  conf.setInt(StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
  conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    FailPrimaryMetaScanCp.class.getName());
  UTIL.startMiniCluster(3);
  try (ConnectionRegistry registry = ConnectionRegistryFactory.getRegistry(conf)) {
    RegionReplicaTestHelper.waitUntilAllMetaReplicasAreReady(UTIL, registry);
  }
  try (Table table = UTIL.createTable(TABLE_NAME, FAMILY)) {
    table.put(new Put(ROW).addColumn(FAMILY, QUALIFIER, VALUE));
  }
  UTIL.flush(TableName.META_TABLE_NAME);
  // wait for the store file refresh so we can read the region location from secondary meta
  // replicas
  Thread.sleep(2000);
}
 
Example 19
Source Project: hbase   Source File: SnapshotWithAclTestBase.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  // Enable security
  enableSecurity(conf);
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AccessController.class.getName());
  // Verify enableSecurity sets up what we require
  verifyConfiguration(conf);
  // Enable EXEC permission checking
  conf.setBoolean(AccessControlConstants.EXEC_PERMISSION_CHECKS_KEY, true);
  TEST_UTIL.startMiniCluster();
  TEST_UTIL.waitUntilAllRegionsAssigned(PermissionStorage.ACL_TABLE_NAME);
  MasterCoprocessorHost cpHost =
    TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterCoprocessorHost();
  cpHost.load(AccessController.class, Coprocessor.PRIORITY_HIGHEST, conf);

  USER_OWNER = User.createUserForTesting(conf, "owner", new String[0]);
  USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
  USER_RO = User.createUserForTesting(conf, "rouser", new String[0]);
  USER_NONE = User.createUserForTesting(conf, "usernone", new String[0]);
}
 
Example 20
Source Project: hbase   Source File: TestProcedurePriority.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  UTIL.getConfiguration().setLong(ProcedureExecutor.WORKER_KEEP_ALIVE_TIME_CONF_KEY, 5000);
  UTIL.getConfiguration().setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 4);
  UTIL.getConfiguration().set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, MyCP.class.getName());
  UTIL.startMiniCluster(3);
  CORE_POOL_SIZE =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getCorePoolSize();
  TABLE_COUNT = 50 * CORE_POOL_SIZE;
  List<Future<?>> futures = new ArrayList<>();
  AsyncAdmin admin = UTIL.getAsyncConnection().getAdmin();
  Semaphore concurrency = new Semaphore(10);
  for (int i = 0; i < TABLE_COUNT; i++) {
    concurrency.acquire();
    futures.add(admin
      .createTable(TableDescriptorBuilder.newBuilder(TableName.valueOf(TABLE_NAME_PREFIX + i))
        .setColumnFamily(ColumnFamilyDescriptorBuilder.of(CF)).build())
      .whenComplete((r, e) -> concurrency.release()));
  }
  for (Future<?> future : futures) {
    future.get(3, TimeUnit.MINUTES);
  }
  UTIL.getAdmin().balance(true);
  UTIL.waitUntilNoRegionsInTransition();
}
 
Example 21
Source Project: hbase   Source File: TestRSGroupsBase.java    License: Apache License 2.0 6 votes vote down vote up
public static void setUpTestBeforeClass() throws Exception {
  Configuration conf = TEST_UTIL.getConfiguration();
  conf.setFloat("hbase.master.balancer.stochastic.tableSkewCost", 6000);
  if (conf.get(RSGroupUtil.RS_GROUP_ENABLED) == null) {
    RSGroupUtil.enableRSGroup(conf);
  }
  if (conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) != null) {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
      conf.get(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY) + "," +
        CPMasterObserver.class.getName());
  } else {
    conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY, CPMasterObserver.class.getName());
  }

  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART,
    NUM_SLAVES_BASE - 1);
  conf.setBoolean(SnapshotManager.HBASE_SNAPSHOT_ENABLED, true);
  conf.setInt("hbase.rpc.timeout", 100000);

  TEST_UTIL.startMiniCluster(NUM_SLAVES_BASE - 1);
  initialize();
}
 
Example 22
Source Project: hbase   Source File: AbstractTestFSWAL.java    License: Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void setUpBeforeClass() throws Exception {
  // Make block sizes small.
  TEST_UTIL.getConfiguration().setInt("dfs.blocksize", 1024 * 1024);
  // quicker heartbeat interval for faster DN death notification
  TEST_UTIL.getConfiguration().setInt("dfs.namenode.heartbeat.recheck-interval", 5000);
  TEST_UTIL.getConfiguration().setInt("dfs.heartbeat.interval", 1);
  TEST_UTIL.getConfiguration().setInt("dfs.client.socket-timeout", 5000);

  // faster failover with cluster.shutdown();fs.close() idiom
  TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connect.max.retries", 1);
  TEST_UTIL.getConfiguration().setInt("dfs.client.block.recovery.retries", 1);
  TEST_UTIL.getConfiguration().setInt("hbase.ipc.client.connection.maxidletime", 500);
  TEST_UTIL.getConfiguration().set(CoprocessorHost.WAL_COPROCESSOR_CONF_KEY,
    SampleRegionWALCoprocessor.class.getName());
  TEST_UTIL.startMiniDFSCluster(3);

  CONF = TEST_UTIL.getConfiguration();
  FS = TEST_UTIL.getDFSCluster().getFileSystem();
}
 
Example 23
Source Project: hbase   Source File: TestRegionServerAbort.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void setup() throws Exception {
  testUtil = new HBaseTestingUtility();
  conf = testUtil.getConfiguration();
  conf.set(CoprocessorHost.REGIONSERVER_COPROCESSOR_CONF_KEY,
      StopBlockingRegionObserver.class.getName());
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
      StopBlockingRegionObserver.class.getName());
  // make sure we have multiple blocks so that the client does not prefetch all block locations
  conf.set("dfs.blocksize", Long.toString(100 * 1024));
  // prefetch the first block
  conf.set(DFSConfigKeys.DFS_CLIENT_READ_PREFETCH_SIZE_KEY, Long.toString(100 * 1024));
  conf.set(HConstants.REGION_IMPL, ErrorThrowingHRegion.class.getName());

  testUtil.startMiniZKCluster();
  dfsCluster = testUtil.startMiniDFSCluster(2);
  StartMiniClusterOption option = StartMiniClusterOption.builder().numRegionServers(2).build();
  cluster = testUtil.startMiniHBaseCluster(option);
}
 
Example 24
Source Project: kylin-on-parquet-v2   Source File: CubeVisitServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
    Configuration conf = util.getConfiguration();
    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
            TestRowProcessorEndpoint.RowProcessorEndpoint.class.getName());
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
    conf.setInt(HConstants.MASTER_PORT, 17000);
    conf.setInt(HConstants.MASTER_INFO_PORT, 17010);
    conf.setInt(HConstants.REGIONSERVER_PORT, 17020);
    conf.setLong("hbase.hregion.row.processor.timeout", 1000L);
    util.startMiniCluster();
    staticCreateTestMetadata();

    prepareTestData();
}
 
Example 25
Source Project: eagle   Source File: TestWithHBaseCoprocessor.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUpHBase() throws IOException {
    System.setProperty("config.resource", "/application-co.conf");
    Configuration conf = HBaseConfiguration.create();
    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, AggregateProtocolEndPoint.class.getName());
    conf.set("zookeeper.znode.parent", getZkZnodeParent());
    conf.setInt("hbase.master.info.port", -1);//avoid port clobbering
    conf.setInt("hbase.regionserver.info.port", -1);//avoid port clobbering

    int attempts = 0;
    hbase = new HBaseTestingUtility(conf);
    boolean successToStart = false;
    while (attempts < 3) {
        try {
            attempts ++;
            hbase.startMiniCluster();
            successToStart = true;
        } catch (Exception e) {
            LOG.error("Error to start mini cluster (tried {} times): {}", attempts, e.getMessage(), e);
            try {
                hbase.shutdownMiniCluster();
            } catch (Exception e1) {
                LOG.warn(e.getMessage(), e);
            }
        }
    }

    Assert.assertTrue("Failed to start mini cluster in " + attempts + " attempts", successToStart);

    HTable table = hbase.createTable(String.valueOf("unittest"),"f");
    HTableDescriptor descriptor = new HTableDescriptor(table.getTableDescriptor());
    descriptor.addCoprocessor(AggregateProtocolEndPoint.class.getName());
    hbase.getHBaseAdmin().modifyTable("unittest",descriptor);

    System.setProperty("storage.hbase.autoCreateTable","false");
    System.setProperty("storage.hbase.coprocessorEnabled", String.valueOf(true));
    System.setProperty("storage.hbase.zookeeperZnodeParent", getZkZnodeParent());
    System.setProperty("storage.hbase.zookeeperPropertyClientPort", String.valueOf(hbase.getZkCluster().getClientPort()));
}
 
Example 26
Source Project: kylin   Source File: CubeVisitServiceTest.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setupBeforeClass() throws Exception {
    Configuration conf = util.getConfiguration();
    conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
            TestRowProcessorEndpoint.RowProcessorEndpoint.class.getName());
    conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
    conf.setInt(HConstants.MASTER_PORT, 17000);
    conf.setInt(HConstants.MASTER_INFO_PORT, 17010);
    conf.setInt(HConstants.REGIONSERVER_PORT, 17020);
    conf.setLong("hbase.hregion.row.processor.timeout", 1000L);
    util.startMiniCluster();
    staticCreateTestMetadata();

    prepareTestData();
}
 
Example 27
Source Project: hbase   Source File: TestBackupDeleteWithFailures.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * @throws Exception if starting the mini cluster or setting up the tables fails
 */
@Override
@Before
public void setUp() throws Exception {
  conf1.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
    MasterSnapshotObserver.class.getName());
  conf1.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 1);
  super.setUp();
}
 
Example 28
Source Project: hbase   Source File: TestRpcControllerFactory.java    License: Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void setUp() throws Exception {
  // load an endpoint so we have an endpoint to test - it doesn't matter which one, but
  // this is already in tests, so we can just use it.
  Configuration conf = UTIL.getConfiguration();
  conf.set(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
    ProtobufCoprocessorService.class.getName());

  UTIL.startMiniCluster();
}
 
Example 29
Source Project: hbase   Source File: SnapshotScannerHDFSAclHelper.java    License: Apache License 2.0 5 votes vote down vote up
public static boolean isAclSyncToHdfsEnabled(Configuration conf) {
  String[] masterCoprocessors = conf.getStrings(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
  Set<String> masterCoprocessorSet = new HashSet<>();
  if (masterCoprocessors != null) {
    Collections.addAll(masterCoprocessorSet, masterCoprocessors);
  }
  return conf.getBoolean(SnapshotScannerHDFSAclHelper.ACL_SYNC_TO_HDFS_ENABLE, false)
      && masterCoprocessorSet.contains(SnapshotScannerHDFSAclController.class.getName())
      && masterCoprocessorSet.contains(AccessController.class.getName());
}
 
Example 30
Source Project: hbase   Source File: MasterCoprocessorHost.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public MasterCoprocessor checkAndGetInstance(Class<?> implClass)
    throws InstantiationException, IllegalAccessException {
  try {
    if (MasterCoprocessor.class.isAssignableFrom(implClass)) {
      return implClass.asSubclass(MasterCoprocessor.class).getDeclaredConstructor().newInstance();
    } else {
      LOG.error("{} is not of type MasterCoprocessor. Check the configuration of {}",
          implClass.getName(), CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY);
      return null;
    }
  } catch (NoSuchMethodException | InvocationTargetException e) {
    throw (InstantiationException) new InstantiationException(implClass.getName()).initCause(e);
  }
}