org.apache.hadoop.hbase.ipc.RpcControllerFactory Java Examples

The following examples show how to use org.apache.hadoop.hbase.ipc.RpcControllerFactory. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: IndexHandlerIT.java    From phoenix with Apache License 2.0 6 votes vote down vote up
@Test
public void testClientWritesWithPriority() throws Exception {
    Configuration conf = new Configuration(UTIL.getConfiguration());
    // add the keys for our rpc factory
    conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
        CountingIndexClientRpcFactory.class.getName());
    // and set the index table as the current table
    conf.setStrings(IndexQosRpcControllerFactory.INDEX_TABLE_NAMES_KEY,
        TestTable.getTableNameString());
    HTable table = new HTable(conf, TestTable.getTableName());

    // do a write to the table
    Put p = new Put(row);
    p.add(family, qual, new byte[] { 1, 0, 1, 0 });
    table.put(p);
    table.flushCommits();

    // check the counts on the rpc controller
    assertEquals("Didn't get the expected number of index priority writes!", 1,
        (int) CountingIndexClientRpcController.priorityCounts
                .get(QueryServicesOptions.DEFAULT_INDEX_MIN_PRIORITY));

    table.close();
}
 
Example #2
Source File: MasterRegistry.java    From hbase with Apache License 2.0 6 votes vote down vote up
MasterRegistry(Configuration conf) throws IOException {
  this.hedgedReadFanOut = Math.max(1, conf.getInt(MASTER_REGISTRY_HEDGED_REQS_FANOUT_KEY,
    MASTER_REGISTRY_HEDGED_REQS_FANOUT_DEFAULT));
  int rpcTimeoutMs = (int) Math.min(Integer.MAX_VALUE,
    conf.getLong(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT));
  // XXX: we pass cluster id as null here since we do not have a cluster id yet, we have to fetch
  // this through the master registry...
  // This is a problem as we will use the cluster id to determine the authentication method
  rpcClient = RpcClientFactory.createClient(conf, null);
  rpcControllerFactory = RpcControllerFactory.instantiate(conf);
  Set<ServerName> masterAddrs = parseMasterAddrs(conf);
  ImmutableMap.Builder<ServerName, ClientMetaService.Interface> builder =
    ImmutableMap.builderWithExpectedSize(masterAddrs.size());
  User user = User.getCurrent();
  for (ServerName masterAddr : masterAddrs) {
    builder.put(masterAddr,
      ClientMetaService.newStub(rpcClient.createRpcChannel(masterAddr, user, rpcTimeoutMs)));
  }
  masterAddr2Stub = builder.build();
}
 
Example #3
Source File: ServerUtil.java    From phoenix with Apache License 2.0 6 votes vote down vote up
public static Configuration getIndexWriterConnection(Configuration conf) {
    Configuration clonedConfig = PropertiesUtil.cloneConfig(conf);
    /*
     * Set the rpc controller factory so that the HTables used by IndexWriter would
     * set the correct priorities on the remote RPC calls.
     */
    clonedConfig.setClass(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
            InterRegionServerIndexRpcControllerFactory.class, RpcControllerFactory.class);
    // lower the number of rpc retries.  We inherit config from HConnectionManager#setServerSideHConnectionRetries,
    // which by default uses a multiplier of 10.  That is too many retries for our synchronous index writes
    clonedConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
        conf.getInt(INDEX_WRITER_RPC_RETRIES_NUMBER,
            DEFAULT_INDEX_WRITER_RPC_RETRIES_NUMBER));
    clonedConfig.setInt(HConstants.HBASE_CLIENT_PAUSE, conf
        .getInt(INDEX_WRITER_RPC_PAUSE, DEFAULT_INDEX_WRITER_RPC_PAUSE));
    return clonedConfig;
}
 
Example #4
Source File: IndexQosCompat.java    From phoenix with Apache License 2.0 5 votes vote down vote up
/**
 * Set the index rpc controller, if the rpc controller exists. No-op if there the RpcController
 * is not on the classpath.
 * @param conf to update
 */
public static void setPhoenixIndexRpcController(Configuration conf) {
    if (rpcControllerExists()) {
        // then we can load the class just fine
        conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
            PhoenixIndexRpcSchedulerFactory.class.getName());
    }
}
 
Example #5
Source File: TestRpcControllerFactory.java    From hbase with Apache License 2.0 5 votes vote down vote up
@Test
public void testFallbackToDefaultRpcControllerFactory() {
  Configuration conf = new Configuration(UTIL.getConfiguration());
  conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, "foo.bar.Baz");

  // Should not fail
  RpcControllerFactory factory = RpcControllerFactory.instantiate(conf);
  assertNotNull(factory);
  assertEquals(factory.getClass(), RpcControllerFactory.class);
}
 
Example #6
Source File: UngroupedAggregateRegionObserver.java    From phoenix with Apache License 2.0 5 votes vote down vote up
@Override
public void start(CoprocessorEnvironment e) throws IOException {
    // Can't use ClientKeyValueBuilder on server-side because the memstore expects to
    // be able to get a single backing buffer for a KeyValue.
    this.kvBuilder = GenericKeyValueBuilder.INSTANCE;
    /*
     * We need to create a copy of region's configuration since we don't want any side effect of
     * setting the RpcControllerFactory.
     */
    upsertSelectConfig = PropertiesUtil.cloneConfig(e.getConfiguration());
    /*
     * Till PHOENIX-3995 is fixed, we need to use the
     * InterRegionServerIndexRpcControllerFactory. Although this would cause remote RPCs to use
     * index handlers on the destination region servers, it is better than using the regular
     * priority handlers which could result in a deadlock.
     */
    upsertSelectConfig.setClass(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY,
        InterRegionServerIndexRpcControllerFactory.class, RpcControllerFactory.class);

    compactionConfig = ServerUtil.getCompactionConfig(e.getConfiguration());

    // For retries of index write failures, use the same # of retries as the rebuilder
    indexWriteConfig = PropertiesUtil.cloneConfig(e.getConfiguration());
    indexWriteConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
        e.getConfiguration().getInt(QueryServices.INDEX_REBUILD_RPC_RETRIES_COUNTER,
            QueryServicesOptions.DEFAULT_INDEX_REBUILD_RPC_RETRIES_COUNTER));
    indexWriteProps = new ReadOnlyProps(indexWriteConfig.iterator());
}
 
Example #7
Source File: TestClientMetaServiceRPCs.java    From hbase with Apache License 2.0 4 votes vote down vote up
private static HBaseRpcController getRpcController() {
  return RpcControllerFactory.instantiate(conf).newController();
}
 
Example #8
Source File: AsyncConnectionImpl.java    From hbase with Apache License 2.0 4 votes vote down vote up
public AsyncConnectionImpl(Configuration conf, ConnectionRegistry registry, String clusterId,
    SocketAddress localAddress, User user) {
  this.conf = conf;
  this.user = user;
  if (user.isLoginFromKeytab()) {
    spawnRenewalChore(user.getUGI());
  }
  this.connConf = new AsyncConnectionConfiguration(conf);
  this.registry = registry;
  if (conf.getBoolean(CLIENT_SIDE_METRICS_ENABLED_KEY, false)) {
    this.metrics = Optional.of(new MetricsConnection(this.toString(), () -> null, () -> null));
  } else {
    this.metrics = Optional.empty();
  }
  this.rpcClient = RpcClientFactory.createClient(
      conf, clusterId, localAddress, metrics.orElse(null));
  this.rpcControllerFactory = RpcControllerFactory.instantiate(conf);
  this.hostnameCanChange = conf.getBoolean(RESOLVE_HOSTNAME_ON_FAIL_KEY, true);
  this.rpcTimeout =
    (int) Math.min(Integer.MAX_VALUE, TimeUnit.NANOSECONDS.toMillis(connConf.getRpcTimeoutNs()));
  this.locator = new AsyncRegionLocator(this, RETRY_TIMER);
  this.callerFactory = new AsyncRpcRetryingCallerFactory(this, RETRY_TIMER);
  if (conf.getBoolean(CLIENT_NONCES_ENABLED_KEY, true)) {
    nonceGenerator = PerClientRandomNonceGenerator.get();
  } else {
    nonceGenerator = NO_NONCE_GENERATOR;
  }
  this.stats = Optional.ofNullable(ServerStatisticTracker.create(conf));
  this.backoffPolicy = ClientBackoffPolicyFactory.create(conf);
  ClusterStatusListener listener = null;
  if (conf.getBoolean(STATUS_PUBLISHED, STATUS_PUBLISHED_DEFAULT)) {
    // TODO: this maybe a blocking operation, better to create it outside the constructor and pass
    // it in, just like clusterId. Not a big problem for now as the default value is false.
    Class<? extends ClusterStatusListener.Listener> listenerClass = conf.getClass(
      STATUS_LISTENER_CLASS, DEFAULT_STATUS_LISTENER_CLASS, ClusterStatusListener.Listener.class);
    if (listenerClass == null) {
      LOG.warn("{} is true, but {} is not set", STATUS_PUBLISHED, STATUS_LISTENER_CLASS);
    } else {
      try {
        listener = new ClusterStatusListener(
          new ClusterStatusListener.DeadServerHandler() {
            @Override
            public void newDead(ServerName sn) {
              locator.clearCache(sn);
              rpcClient.cancelConnections(sn);
            }
          }, conf, listenerClass);
      } catch (IOException e) {
        LOG.warn("Failed create of ClusterStatusListener, not a critical, ignoring...", e);
      }
    }
  }
  this.clusterStatusListener = listener;
}
 
Example #9
Source File: HBaseHbck.java    From hbase with Apache License 2.0 4 votes vote down vote up
HBaseHbck(BlockingInterface hbck, RpcControllerFactory rpcControllerFactory) {
  this.hbck = hbck;
  this.rpcControllerFactory = rpcControllerFactory;
}
 
Example #10
Source File: QueryServicesOptions.java    From phoenix with Apache License 2.0 4 votes vote down vote up
public static QueryServicesOptions withDefaults() {
    Configuration config = HBaseFactoryProvider.getConfigurationFactory().getConfiguration();
    QueryServicesOptions options = new QueryServicesOptions(config)
        .setIfUnset(STATS_USE_CURRENT_TIME_ATTRIB, DEFAULT_STATS_USE_CURRENT_TIME)
        .setIfUnset(RUN_UPDATE_STATS_ASYNC, DEFAULT_RUN_UPDATE_STATS_ASYNC)
        .setIfUnset(COMMIT_STATS_ASYNC, DEFAULT_COMMIT_STATS_ASYNC)
        .setIfUnset(KEEP_ALIVE_MS_ATTRIB, DEFAULT_KEEP_ALIVE_MS)
        .setIfUnset(THREAD_POOL_SIZE_ATTRIB, DEFAULT_THREAD_POOL_SIZE)
        .setIfUnset(QUEUE_SIZE_ATTRIB, DEFAULT_QUEUE_SIZE)
        .setIfUnset(THREAD_TIMEOUT_MS_ATTRIB, DEFAULT_THREAD_TIMEOUT_MS)
        .setIfUnset(CLIENT_SPOOL_THRESHOLD_BYTES_ATTRIB, DEFAULT_CLIENT_SPOOL_THRESHOLD_BYTES)
        .setIfUnset(SERVER_SPOOL_THRESHOLD_BYTES_ATTRIB, DEFAULT_SERVER_SPOOL_THRESHOLD_BYTES)
        .setIfUnset(SPOOL_DIRECTORY, DEFAULT_SPOOL_DIRECTORY)
        .setIfUnset(MAX_MEMORY_PERC_ATTRIB, DEFAULT_MAX_MEMORY_PERC)
        .setIfUnset(MAX_TENANT_MEMORY_PERC_ATTRIB, DEFAULT_MAX_TENANT_MEMORY_PERC)
        .setIfUnset(MAX_SERVER_CACHE_SIZE_ATTRIB, DEFAULT_MAX_SERVER_CACHE_SIZE)
        .setIfUnset(SCAN_CACHE_SIZE_ATTRIB, DEFAULT_SCAN_CACHE_SIZE)
        .setIfUnset(DATE_FORMAT_ATTRIB, DEFAULT_DATE_FORMAT)
        .setIfUnset(DATE_FORMAT_TIMEZONE_ATTRIB, DEFAULT_DATE_FORMAT_TIMEZONE)
        .setIfUnset(STATS_UPDATE_FREQ_MS_ATTRIB, DEFAULT_STATS_UPDATE_FREQ_MS)
        .setIfUnset(MIN_STATS_UPDATE_FREQ_MS_ATTRIB, DEFAULT_MIN_STATS_UPDATE_FREQ_MS)
        .setIfUnset(STATS_CACHE_THREAD_POOL_SIZE, DEFAULT_STATS_CACHE_THREAD_POOL_SIZE)
        .setIfUnset(CALL_QUEUE_ROUND_ROBIN_ATTRIB, DEFAULT_CALL_QUEUE_ROUND_ROBIN)
        .setIfUnset(MAX_MUTATION_SIZE_ATTRIB, DEFAULT_MAX_MUTATION_SIZE)
        .setIfUnset(ROW_KEY_ORDER_SALTED_TABLE_ATTRIB, DEFAULT_FORCE_ROW_KEY_ORDER)
        .setIfUnset(USE_INDEXES_ATTRIB, DEFAULT_USE_INDEXES)
        .setIfUnset(IMMUTABLE_ROWS_ATTRIB, DEFAULT_IMMUTABLE_ROWS)
        .setIfUnset(INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, DEFAULT_INDEX_MUTATE_BATCH_SIZE_THRESHOLD)
        .setIfUnset(MAX_SPOOL_TO_DISK_BYTES_ATTRIB, DEFAULT_MAX_SPOOL_TO_DISK_BYTES)
        .setIfUnset(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA)
        .setIfUnset(GROUPBY_SPILLABLE_ATTRIB, DEFAULT_GROUPBY_SPILLABLE)
        .setIfUnset(GROUPBY_MAX_CACHE_SIZE_ATTRIB, DEFAULT_GROUPBY_MAX_CACHE_MAX)
        .setIfUnset(GROUPBY_SPILL_FILES_ATTRIB, DEFAULT_GROUPBY_SPILL_FILES)
        .setIfUnset(SEQUENCE_CACHE_SIZE_ATTRIB, DEFAULT_SEQUENCE_CACHE_SIZE)
        .setIfUnset(SCAN_RESULT_CHUNK_SIZE, DEFAULT_SCAN_RESULT_CHUNK_SIZE)
        .setIfUnset(ALLOW_ONLINE_TABLE_SCHEMA_UPDATE, DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE)
        .setIfUnset(NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK, DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK)
        .setIfUnset(DELAY_FOR_SCHEMA_UPDATE_CHECK, DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK)
        .setIfUnset(GLOBAL_METRICS_ENABLED, DEFAULT_IS_GLOBAL_METRICS_ENABLED)
        .setIfUnset(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, DEFAULT_CLIENT_RPC_CONTROLLER_FACTORY)
        .setIfUnset(USE_BYTE_BASED_REGEX_ATTRIB, DEFAULT_USE_BYTE_BASED_REGEX)
        .setIfUnset(FORCE_ROW_KEY_ORDER_ATTRIB, DEFAULT_FORCE_ROW_KEY_ORDER)
        .setIfUnset(COLLECT_REQUEST_LEVEL_METRICS, DEFAULT_REQUEST_LEVEL_METRICS_ENABLED)
        .setIfUnset(ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE, DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE)
        .setIfUnset(ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE, DEFAULT_ALLOW_VIEWS_ADD_NEW_CF_BASE_TABLE)
        .setIfUnset(RENEW_LEASE_THRESHOLD_MILLISECONDS, DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS)
        .setIfUnset(RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS, DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS)
        .setIfUnset(RENEW_LEASE_THREAD_POOL_SIZE, DEFAULT_RENEW_LEASE_THREAD_POOL_SIZE)
        .setIfUnset(IS_NAMESPACE_MAPPING_ENABLED, DEFAULT_IS_NAMESPACE_MAPPING_ENABLED)
        .setIfUnset(IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, DEFAULT_IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE)
        .setIfUnset(LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB, DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)
        .setIfUnset(AUTO_UPGRADE_ENABLED, DEFAULT_AUTO_UPGRADE_ENABLED)
        .setIfUnset(UPLOAD_BINARY_DATA_TYPE_ENCODING, DEFAULT_UPLOAD_BINARY_DATA_TYPE_ENCODING)
        .setIfUnset(TRACING_ENABLED, DEFAULT_TRACING_ENABLED)
        .setIfUnset(TRACING_BATCH_SIZE, DEFAULT_TRACING_BATCH_SIZE)
        .setIfUnset(TRACING_THREAD_POOL_SIZE, DEFAULT_TRACING_THREAD_POOL_SIZE)
        .setIfUnset(STATS_COLLECTION_ENABLED, DEFAULT_STATS_COLLECTION_ENABLED)
        .setIfUnset(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION)
        .setIfUnset(USE_STATS_FOR_PARALLELIZATION, DEFAULT_USE_STATS_FOR_PARALLELIZATION)
        .setIfUnset(UPLOAD_BINARY_DATA_TYPE_ENCODING, DEFAULT_UPLOAD_BINARY_DATA_TYPE_ENCODING)
        .setIfUnset(COST_BASED_OPTIMIZER_ENABLED, DEFAULT_COST_BASED_OPTIMIZER_ENABLED)
        .setIfUnset(PHOENIX_ACLS_ENABLED,  DEFAULT_PHOENIX_ACLS_ENABLED)
        .setIfUnset(LOG_LEVEL,  DEFAULT_LOGGING_LEVEL)
        .setIfUnset(LOG_SAMPLE_RATE,  DEFAULT_LOG_SAMPLE_RATE)
        .setIfUnset(TxConstants.TX_PRE_014_CHANGESET_KEY, Boolean.FALSE.toString())
        .setIfUnset(CLIENT_METRICS_TAG, DEFAULT_CLIENT_METRICS_TAG)
        .setIfUnset(CLIENT_INDEX_ASYNC_THRESHOLD, DEFAULT_CLIENT_INDEX_ASYNC_THRESHOLD)
        ;
    // HBase sets this to 1, so we reset it to something more appropriate.
    // Hopefully HBase will change this, because we can't know if a user set
    // it to 1, so we'll change it.
    int scanCaching = config.getInt(SCAN_CACHE_SIZE_ATTRIB, 0);
    if (scanCaching == 1) {
        config.setInt(SCAN_CACHE_SIZE_ATTRIB, DEFAULT_SCAN_CACHE_SIZE);
    } else if (scanCaching <= 0) { // Provides the user with a way of setting it to 1
        config.setInt(SCAN_CACHE_SIZE_ATTRIB, 1);
    }
    return options;
}