Java Code Examples for org.apache.hadoop.hbase.util.Threads#getBoundedCachedThreadPool()

public boolean start() {
  if (running.getAndSet(true)) {
    LOG.warn("Already running");
    return false;
  }

  LOG.info("Instantiated, coreThreads={} (allowCoreThreadTimeOut=true), queueMaxSize={}, " +
      "operationDelay={}", this.corePoolSize, this.queueMaxSize, this.operationDelay);

  // Create the timeout executor
  timeoutExecutor = new TimeoutExecutorThread();
  timeoutExecutor.start();

  // Create the thread pool that will execute RPCs
  threadPool = Threads.getBoundedCachedThreadPool(corePoolSize, 60L, TimeUnit.SECONDS,
    Threads.newDaemonThreadFactory(this.getClass().getSimpleName(),
        getUncaughtExceptionHandler()));
  return true;
}
 

public HFileReplicator(Configuration sourceClusterConf,
    String sourceBaseNamespaceDirPath, String sourceHFileArchiveDirPath,
    Map<String, List<Pair<byte[], List<String>>>> tableQueueMap, Configuration conf,
    AsyncClusterConnection connection, List<String> sourceClusterIds) throws IOException {
  this.sourceClusterConf = sourceClusterConf;
  this.sourceBaseNamespaceDirPath = sourceBaseNamespaceDirPath;
  this.sourceHFileArchiveDirPath = sourceHFileArchiveDirPath;
  this.bulkLoadHFileMap = tableQueueMap;
  this.conf = conf;
  this.connection = connection;
  this.sourceClusterIds = sourceClusterIds;

  userProvider = UserProvider.instantiate(conf);
  fsDelegationToken = new FsDelegationToken(userProvider, "renewer");
  this.hbaseStagingDir =
    new Path(CommonFSUtils.getRootDir(conf), HConstants.BULKLOAD_STAGING_DIR_NAME);
  this.maxCopyThreads =
      this.conf.getInt(REPLICATION_BULKLOAD_COPY_MAXTHREADS_KEY,
        REPLICATION_BULKLOAD_COPY_MAXTHREADS_DEFAULT);
  this.exec = Threads.getBoundedCachedThreadPool(maxCopyThreads, 60, TimeUnit.SECONDS,
      new ThreadFactoryBuilder().setDaemon(true)
          .setNameFormat("HFileReplicationCopier-%1$d-" + this.sourceBaseNamespaceDirPath).
        build());
  this.copiesPerThread =
      conf.getInt(REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_KEY,
        REPLICATION_BULKLOAD_COPY_HFILES_PERTHREAD_DEFAULT);

  sinkFs = FileSystem.get(conf);
}
 

public OutputSink(WALSplitter.PipelineController controller, EntryBuffers entryBuffers,
    int numWriters) {
  this.numThreads = numWriters;
  this.controller = controller;
  this.entryBuffers = entryBuffers;
  this.closeThreadPool = Threads.getBoundedCachedThreadPool(numThreads, 30L, TimeUnit.SECONDS,
      Threads.newDaemonThreadFactory("split-log-closeStream-"));
  this.closeCompletionService = new ExecutorCompletionService<>(closeThreadPool);
}
 

private static synchronized ThreadPoolExecutor getArchiveExecutor(final Configuration conf) {
  if (archiveExecutor == null) {
    int maxThreads = conf.getInt("hbase.hfilearchiver.thread.pool.max", 8);
    archiveExecutor = Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
      getThreadFactory());

    // Shutdown this ThreadPool in a shutdown hook
    Runtime.getRuntime().addShutdownHook(new Thread(() -> archiveExecutor.shutdown()));
  }
  return archiveExecutor;
}
 

FlushTableSubprocedurePool(String name, Configuration conf, Abortable abortable) {
  this.abortable = abortable;
  // configure the executor service
  long keepAlive = conf.getLong(
    RegionServerFlushTableProcedureManager.FLUSH_TIMEOUT_MILLIS_KEY,
    RegionServerFlushTableProcedureManager.FLUSH_TIMEOUT_MILLIS_DEFAULT);
  int threads = conf.getInt(CONCURENT_FLUSH_TASKS_KEY, DEFAULT_CONCURRENT_FLUSH_TASKS);
  this.name = name;
  executor = Threads.getBoundedCachedThreadPool(threads, keepAlive, TimeUnit.MILLISECONDS,
      "rs(" + name + ")-flush-proc");
  taskPool = new ExecutorCompletionService<>(executor);
}
 

SnapshotSubprocedurePool(String name, Configuration conf, Abortable abortable) {
  this.abortable = abortable;
  // configure the executor service
  long keepAlive = conf.getLong(
    RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_KEY,
    RegionServerSnapshotManager.SNAPSHOT_TIMEOUT_MILLIS_DEFAULT);
  int threads = conf.getInt(CONCURENT_SNAPSHOT_TASKS_KEY, DEFAULT_CONCURRENT_SNAPSHOT_TASKS);
  this.name = name;
  executor = Threads.getBoundedCachedThreadPool(threads, keepAlive, TimeUnit.MILLISECONDS,
      "rs(" + name + ")-snapshot");
  taskPool = new ExecutorCompletionService<>(executor);
}
 

@BeforeClass
public static void setUp() throws Exception {
  threadPool = Threads.getBoundedCachedThreadPool(32, 60L, TimeUnit.SECONDS,
    Threads.newDaemonThreadFactory("ProcedureDispatcher",
      new UncaughtExceptionHandler() {
        @Override
        public void uncaughtException(Thread t, Throwable e) {
          LOG.warn("Failed thread " + t.getName(), e);
        }
      }));
  executorService = new ExecutorCompletionService(threadPool);
}
 

@Override
public void init(Context context) throws IOException {
  super.init(context);
  this.conf = HBaseConfiguration.create(ctx.getConfiguration());
  decorateConf();
  this.maxRetriesMultiplier = this.conf.getInt("replication.source.maxretriesmultiplier", 300);
  this.socketTimeoutMultiplier = this.conf.getInt("replication.source.socketTimeoutMultiplier",
      maxRetriesMultiplier);
  // A Replicator job is bound by the RPC timeout. We will wait this long for all Replicator
  // tasks to terminate when doStop() is called.
  long maxTerminationWaitMultiplier = this.conf.getLong(
      "replication.source.maxterminationmultiplier",
      DEFAULT_MAX_TERMINATION_WAIT_MULTIPLIER);
  this.maxTerminationWait = maxTerminationWaitMultiplier *
      this.conf.getLong(HConstants.HBASE_RPC_TIMEOUT_KEY, HConstants.DEFAULT_HBASE_RPC_TIMEOUT);
  // TODO: This connection is replication specific or we should make it particular to
  // replication and make replication specific settings such as compression or codec to use
  // passing Cells.
  this.conn = createConnection(this.conf);
  this.sleepForRetries =
      this.conf.getLong("replication.source.sleepforretries", 1000);
  this.metrics = context.getMetrics();
  // ReplicationQueueInfo parses the peerId out of the znode for us
  this.replicationSinkMgr = createReplicationSinkManager(conn);
  // per sink thread pool
  this.maxThreads = this.conf.getInt(HConstants.REPLICATION_SOURCE_MAXTHREADS_KEY,
    HConstants.REPLICATION_SOURCE_MAXTHREADS_DEFAULT);
  this.exec = Threads.getBoundedCachedThreadPool(maxThreads, 60, TimeUnit.SECONDS,
      new ThreadFactoryBuilder().setDaemon(true).setNameFormat("SinkThread-%d").build());
  this.abortable = ctx.getAbortable();
  // Set the size limit for replication RPCs to 95% of the max request size.
  // We could do with less slop if we have an accurate estimate of encoded size. Being
  // conservative for now.
  this.replicationRpcLimit = (int)(0.95 * conf.getLong(RpcServer.MAX_REQUEST_SIZE,
    RpcServer.DEFAULT_MAX_REQUEST_SIZE));
  this.dropOnDeletedTables =
      this.conf.getBoolean(REPLICATION_DROP_ON_DELETED_TABLE_KEY, false);
  this.dropOnDeletedColumnFamilies = this.conf
      .getBoolean(REPLICATION_DROP_ON_DELETED_COLUMN_FAMILY_KEY, false);

  this.replicationBulkLoadDataEnabled =
      conf.getBoolean(HConstants.REPLICATION_BULKLOAD_ENABLE_KEY,
        HConstants.REPLICATION_BULKLOAD_ENABLE_DEFAULT);
  if (this.replicationBulkLoadDataEnabled) {
    replicationClusterId = this.conf.get(HConstants.REPLICATION_CLUSTER_ID);
  }
  // Construct base namespace directory and hfile archive directory path
  Path rootDir = CommonFSUtils.getRootDir(conf);
  Path baseNSDir = new Path(HConstants.BASE_NAMESPACE_DIR);
  baseNamespaceDir = new Path(rootDir, baseNSDir);
  hfileArchiveDir = new Path(rootDir, new Path(HConstants.HFILE_ARCHIVE_DIRECTORY, baseNSDir));
  isSerial = context.getPeerConfig().isSerial();
}
 

private static ThreadPoolExecutor initializePool(int size) {
  return Threads.getBoundedCachedThreadPool(size, 1, TimeUnit.MINUTES, "dir-scan");
}
 

public static ThreadPoolExecutor createExecutor(final Configuration conf, final String name) {
  int maxThreads = conf.getInt("hbase.snapshot.thread.pool.max", 8);
  return Threads.getBoundedCachedThreadPool(maxThreads, 30L, TimeUnit.SECONDS,
            Threads.newDaemonThreadFactory(name));
}