org.apache.hadoop.hbase.util.Threads Java Examples

private boolean waitForGroupTableOnline() {
  while (isMasterRunning(masterServices)) {
    try {
      TableStateManager tsm = masterServices.getTableStateManager();
      if (!tsm.isTablePresent(RSGROUP_TABLE_NAME)) {
        createRSGroupTable();
      }
      // try reading from the table
      FutureUtils.get(conn.getTable(RSGROUP_TABLE_NAME).get(new Get(ROW_KEY)));
      LOG.info("RSGroup table={} is online, refreshing cached information", RSGROUP_TABLE_NAME);
      RSGroupInfoManagerImpl.this.refresh(true);
      online = true;
      // flush any inconsistencies between ZK and HTable
      RSGroupInfoManagerImpl.this.flushConfig();
      // migrate after we are online.
      migrate();
      return true;
    } catch (Exception e) {
      LOG.warn("Failed to perform check", e);
      // 100ms is short so let's just ignore the interrupt
      Threads.sleepWithoutInterrupt(100);
    }
  }
  return false;
}
 

@Before
public void setUp() throws Exception {
  // Create the test table (owner added to the _acl_ table)
  Admin admin = TEST_UTIL.getAdmin();
  TableDescriptorBuilder.ModifyableTableDescriptor tableDescriptor =
    new TableDescriptorBuilder.ModifyableTableDescriptor(testTable.getTableName());
  ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor familyDescriptor =
    new ColumnFamilyDescriptorBuilder.ModifyableColumnFamilyDescriptor(TEST_FAMILY);
  familyDescriptor.setMaxVersions(4);
  tableDescriptor.setOwner(USER_OWNER);
  tableDescriptor.setColumnFamily(familyDescriptor);
  admin.createTable(tableDescriptor, new byte[][] { Bytes.toBytes("s") });
  TEST_UTIL.waitTableEnabled(testTable.getTableName());
  LOG.info("Sleeping a second because of HBASE-12581");
  Threads.sleep(1000);
}
 

@Override
public void run() {
  try {
    this.user.runAs(new PrivilegedAction<Object>() {
      @Override
      public Object run() {
        runRegionServer();
        return null;
      }
    });
  } catch (Throwable t) {
    LOG.error("Exception in run", t);
  } finally {
    // Run this on the way out.
    if (this.shutdownThread != null) {
      this.shutdownThread.start();
      Threads.shutdown(this.shutdownThread, 30000);
    }
  }
}
 

/**
 * Starts a region server thread and waits until its processed by master. Throws an exception
 * when it can't start a region server or when the region server is not processed by master
 * within the timeout.
 *
 * @return New RegionServerThread
 */
public JVMClusterUtil.RegionServerThread startRegionServerAndWait(long timeout)
    throws IOException {

  JVMClusterUtil.RegionServerThread t =  startRegionServer();
  ServerName rsServerName = t.getRegionServer().getServerName();

  long start = System.currentTimeMillis();
  ClusterMetrics clusterStatus = getClusterMetrics();
  while ((System.currentTimeMillis() - start) < timeout) {
    if (clusterStatus != null && clusterStatus.getLiveServerMetrics().containsKey(rsServerName)) {
      return t;
    }
    Threads.sleep(100);
  }
  if (t.getRegionServer().isOnline()) {
    throw new IOException("RS: " + rsServerName + " online, but not processed by master");
  } else {
    throw new IOException("RS: " + rsServerName + " is offline");
  }
}
 

/**
 * Blocks until there is an active master and that master has completed
 * initialization.
 *
 * @return true if an active master becomes available.  false if there are no
 *         masters left.
 * @throws InterruptedException
 */
@Override
public boolean waitForActiveAndReadyMaster(long timeout) throws IOException {
  List<JVMClusterUtil.MasterThread> mts;
  long start = System.currentTimeMillis();
  while (!(mts = getMasterThreads()).isEmpty()
      && (System.currentTimeMillis() - start) < timeout) {
    for (JVMClusterUtil.MasterThread mt : mts) {
      if (mt.getMaster().isActiveMaster() && mt.getMaster().isInitialized()) {
        return true;
      }
    }

    Threads.sleep(100);
  }
  return false;
}
 

@Override
public boolean preScannerNext(ObserverContext<RegionCoprocessorEnvironment> c,
    InternalScanner s, List<Result> result, int limit, boolean hasNext) throws IOException {
  if (c.getEnvironment().getRegionInfo().isMetaRegion()) {
    int concurrency = CONCURRENCY.incrementAndGet();
    for (;;) {
      int max = MAX_CONCURRENCY.get();
      if (concurrency <= max) {
        break;
      }
      if (MAX_CONCURRENCY.compareAndSet(max, concurrency)) {
        break;
      }
    }
    Threads.sleepWithoutInterrupt(10);
  }
  return hasNext;
}
 

private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) {
  byte[] fam = Bytes.toBytes("testfamily");
  byte[] qf = Bytes.toBytes("testqualifier");
  MemStoreSizing memstoreSizing = new NonThreadSafeMemStoreSizing();
  for (int i = 0; i < keys.length; i++) {
    long timestamp = System.currentTimeMillis();
    Threads.sleep(1); // to make sure each kv gets a different ts
    byte[] row = Bytes.toBytes(keys[i]);
    byte[] val = Bytes.toBytes(keys[i] + i);
    KeyValue kv = new KeyValue(row, fam, qf, timestamp, val);
    hmc.add(kv, memstoreSizing);
    LOG.debug("added kv: " + kv.getKeyString() + ", timestamp" + kv.getTimestamp());
  }
  MemStoreSize mss = memstoreSizing.getMemStoreSize();
  regionServicesForStores.addMemStoreSize(mss.getDataSize(), mss.getHeapSize(),
    mss.getOffHeapSize(), mss.getCellsCount());
  return mss.getDataSize();
}
 

@Test
public void testCountOfCellsAfterFlatteningByIterator() throws IOException {
  String[] keys1 = { "A", "B", "C" }; // A, B, C
  addRowsByKeysWith50Cols(memstore, keys1);
  // this should only flatten as there are no duplicates
  ((CompactingMemStore) memstore).flushInMemory();
  while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) {
    Threads.sleep(10);
  }
  // Just doing the cnt operation here
  MemStoreSegmentsIterator itr = new MemStoreMergerSegmentsIterator(
      ((CompactingMemStore) memstore).getImmutableSegments().getStoreSegments(),
      CellComparatorImpl.COMPARATOR, 10);
  int cnt = 0;
  try {
    while (itr.next() != null) {
      cnt++;
    }
  } finally {
    itr.close();
  }
  assertEquals("the count should be ", 150, cnt);
}
 

/**
 * Wait until Master shows metrics counts for ReplicationLoadSourceList that are
 * greater than <code>greaterThan</code> for <code>serverName</code> before
 * returning. We want to avoid case where RS hasn't yet updated Master before
 * allowing test proceed.
 * @param greaterThan size of replicationLoadSourceList must be greater before we proceed
 */
private List<ReplicationLoadSource> waitOnMetricsReport(int greaterThan, ServerName serverName)
    throws IOException {
  ClusterMetrics metrics = hbaseAdmin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS));
  List<ReplicationLoadSource> list =
    metrics.getLiveServerMetrics().get(serverName).getReplicationLoadSourceList();
  while(list.size() <= greaterThan) {
    Threads.sleep(1000);
  }
  return list;
}
 

public Thread start() {
  Thread t = new Thread(this);
  t = Threads.setDaemonThreadRunning(t);
  try {
    startedLatch.await();
  } catch (InterruptedException e) {
    fail("Unexpected");
  }
  return t;
}
 

public static void blockUntilRegionIsInMeta(Connection conn, long timeout, RegionInfo hri)
    throws IOException, InterruptedException {
  log("blocking until region is in META: " + hri.getRegionNameAsString());
  long start = System.currentTimeMillis();
  while (System.currentTimeMillis() - start < timeout) {
    HRegionLocation loc = MetaTableAccessor.getRegionLocation(conn, hri);
    if (loc != null && !loc.getRegion().isOffline()) {
      log("found region in META: " + hri.getRegionNameAsString());
      break;
    }
    Threads.sleep(100);
  }
}
 

public BlockingCompactionContext waitForBlocking() {
  while (this.blocked == null || !this.blocked.isInCompact) {
    Threads.sleepWithoutInterrupt(50);
  }
  BlockingCompactionContext ctx = this.blocked;
  this.blocked = null;
  return ctx;
}
 

private void doAnswerTaskExecution(final CallRunner callTask,
    final ArrayList<Integer> results, final int value, final int sleepInterval) {
  callTask.setStatus(new MonitoredRPCHandlerImpl());
  doAnswer(new Answer<Object>() {
    @Override
    public Object answer(InvocationOnMock invocation) {
      synchronized (results) {
        results.add(value);
      }
      Threads.sleepWithoutInterrupt(sleepInterval);
      return null;
    }
  }).when(callTask).run();
}
 

@Test
public void testSplitSwitch() throws Exception {
  final TableName tableName = TableName.valueOf(name.getMethodName());
  Table t = TEST_UTIL.createTable(tableName, FAMILY);
  TEST_UTIL.loadTable(t, FAMILY, false);

  RegionLocator locator = TEST_UTIL.getConnection().getRegionLocator(t.getName());
  int originalCount = locator.getAllRegionLocations().size();

  Admin admin = TEST_UTIL.getAdmin();
  initSwitchStatus(admin);
  assertTrue(admin.splitSwitch(false, false));
  try {
    admin.split(t.getName());
    fail("Shouldn't get here");
  } catch (DoNotRetryIOException dnioe) {
    // Expected
  }
  int count = admin.getRegions(tableName).size();
  assertTrue(originalCount == count);
  assertFalse(admin.splitSwitch(true, false));
  admin.split(t.getName());
  while ((count = admin.getRegions(tableName).size()) == originalCount) {
    Threads.sleep(1);
  }
  count = admin.getRegions(tableName).size();
  assertTrue(originalCount < count);
  admin.close();
}
 

@Test
public void testFlushTableAndRegion() throws Exception {
  RegionInfo hri = createTableAndGetOneRegion(tableName);
  ServerName serverName =
      TEST_UTIL.getHBaseCluster().getMaster().getAssignmentManager().getRegionStates()
          .getRegionServerOfRegion(hri);
  HRegionServer regionServer =
      TEST_UTIL.getHBaseCluster().getLiveRegionServerThreads().stream()
          .map(rsThread -> rsThread.getRegionServer())
          .filter(rs -> rs.getServerName().equals(serverName)).findFirst().get();

  // write a put into the specific region
  ASYNC_CONN.getTable(tableName)
      .put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-1")))
      .join();
  assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize() > 0);
  // flush region and wait flush operation finished.
  LOG.info("flushing region: " + Bytes.toStringBinary(hri.getRegionName()));
  admin.flushRegion(hri.getRegionName()).get();
  LOG.info("blocking until flush is complete: " + Bytes.toStringBinary(hri.getRegionName()));
  Threads.sleepWithoutInterrupt(500);
  while (regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize() > 0) {
    Threads.sleep(50);
  }
  // check the memstore.
  assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize(), 0);

  // write another put into the specific region
  ASYNC_CONN.getTable(tableName)
      .put(new Put(hri.getStartKey()).addColumn(FAMILY, FAMILY_0, Bytes.toBytes("value-2")))
      .join();
  assertTrue(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize() > 0);
  admin.flush(tableName).get();
  Threads.sleepWithoutInterrupt(500);
  while (regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize() > 0) {
    Threads.sleep(50);
  }
  // check the memstore.
  assertEquals(regionServer.getOnlineRegion(hri.getRegionName()).getMemStoreDataSize(), 0);
}
 

@Override
public ReturnCode filterCell(final Cell c) throws IOException {
  Threads.sleep(TIMEOUT / 2 + 100);
  return Bytes.equals(CellUtil.cloneRow(c), ROWS[reversed ? 0 : NUM_ROWS - 1])
      ? ReturnCode.INCLUDE
      : ReturnCode.SKIP;
}
 

@Override
protected void beforeWaitOnSafePoint() {
  if (throwException) {
    LOG.info("COUNTDOWN");
    // Don't countdown latch until someone waiting on it otherwise, the above
    // afterCreatingZigZagLatch will get to the latch and no one will ever free it and we'll
    // be stuck; test won't go down
    while (this.latch.getCount() <= 0)
      Threads.sleep(1);
    this.latch.countDown();
  }
}
 

/**
 * Make it so a big overlap spans many Regions, some of which are non-contiguous. Make it so
 * we can fix this condition. HBASE-24247
 */
@Test
public void testOverlapWithMergeOfNonContiguous() throws Exception {
  TableName tn = TableName.valueOf(this.name.getMethodName());
  TEST_UTIL.createMultiRegionTable(tn, HConstants.CATALOG_FAMILY);
  List<RegionInfo> ris = MetaTableAccessor.getTableRegions(TEST_UTIL.getConnection(), tn);
  assertTrue(ris.size() > 5);
  MasterServices services = TEST_UTIL.getHBaseCluster().getMaster();
  services.getCatalogJanitor().scan();
  CatalogJanitor.Report report = services.getCatalogJanitor().getLastReport();
  assertTrue(report.isEmpty());
  // Make a simple overlap spanning second and third region.
  makeOverlap(services, ris.get(1), ris.get(5));
  // Now Delete a region under the overlap to manufacture non-contiguous sub regions.
  RegionInfo deletedRegion = ris.get(3);
  long pid = services.getAssignmentManager().unassign(deletedRegion);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  GCRegionProcedure procedure =
    new GCRegionProcedure(services.getMasterProcedureExecutor().getEnvironment(), ris.get(3));
  pid = services.getMasterProcedureExecutor().submitProcedure(procedure);
  while (!services.getMasterProcedureExecutor().isFinished(pid)) {
    Threads.sleep(100);
  }
  services.getCatalogJanitor().scan();
  report = services.getCatalogJanitor().getLastReport();
  assertEquals(1, MetaFixer.calculateMerges(10, report.getOverlaps()).size());
  MetaFixer fixer = new MetaFixer(services);
  fixer.fixOverlaps(report);
  HBaseTestingUtility.await(10, () -> {
    try {
      services.getCatalogJanitor().scan();
      final CatalogJanitor.Report postReport = services.getCatalogJanitor().getLastReport();
      return postReport.isEmpty();
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  });
}
 

@Override
public void preDelete(final ObserverContext<RegionCoprocessorEnvironment> e,
    final Delete delete, final WALEdit edit, final Durability durability) throws IOException {
  Threads.sleep(sleepTime.get());
  if (ct.incrementAndGet() == 1) {
    throw new IOException("first call I fail");
  }
}
 

public Listener(final String name) throws IOException {
  super(name);
  // The backlog of requests that we will have the serversocket carry.
  int backlogLength = conf.getInt("hbase.ipc.server.listen.queue.size", 128);
  readerPendingConnectionQueueLength =
      conf.getInt("hbase.ipc.server.read.connection-queue.size", 100);
  // Create a new server socket and set to non blocking mode
  acceptChannel = ServerSocketChannel.open();
  acceptChannel.configureBlocking(false);

  // Bind the server socket to the binding addrees (can be different from the default interface)
  bind(acceptChannel.socket(), bindAddress, backlogLength);
  port = acceptChannel.socket().getLocalPort(); //Could be an ephemeral port
  address = (InetSocketAddress)acceptChannel.socket().getLocalSocketAddress();
  // create a selector;
  selector = Selector.open();

  readers = new Reader[readThreads];
  // Why this executor thing? Why not like hadoop just start up all the threads? I suppose it
  // has an advantage in that it is easy to shutdown the pool.
  readPool = Executors.newFixedThreadPool(readThreads,
    new ThreadFactoryBuilder().setNameFormat(
      "Reader=%d,bindAddress=" + bindAddress.getHostName() +
      ",port=" + port).setDaemon(true)
    .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build());
  for (int i = 0; i < readThreads; ++i) {
    Reader reader = new Reader();
    readers[i] = reader;
    readPool.execute(reader);
  }
  LOG.info(getName() + ": started " + readThreads + " reader(s) listening on port=" + port);

  // Register accepts on the server socket with the selector.
  acceptChannel.register(selector, SelectionKey.OP_ACCEPT);
  this.setName("Listener,port=" + port);
  this.setDaemon(true);
}
 

/**
 * Wait on crash processing. Balancer won't run if processing a crashed server.
 */
private void waitOnCrashProcessing() {
  while (UTIL.getHBaseCluster().getMaster().getServerManager().areDeadServersInProgress()) {
    LOG.info("Waiting on processing of crashed server before proceeding...");
    Threads.sleep(1000);
  }
}
 

@Override
public void start() {
  LOG.info("Using {} as user call queue; handlerCount={}; maxQueueLength={}",
    this.getClass().getSimpleName(), handlerCount, maxQueueLength);
  this.executor = new ThreadPoolExecutor(
      handlerCount,
      handlerCount,
      60,
      TimeUnit.SECONDS,
      new ArrayBlockingQueue<>(maxQueueLength),
      Threads.newDaemonThreadFactory("FifoRpcScheduler.handler"),
      new ThreadPoolExecutor.CallerRunsPolicy());
}
 

SimpleRpcServerResponder(SimpleRpcServer simpleRpcServer) throws IOException {
  this.simpleRpcServer = simpleRpcServer;
  this.setName("RpcServer.responder");
  this.setDaemon(true);
  this.setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER);
  writeSelector = Selector.open(); // create a selector
}
 

@Override
public Result postAppend(ObserverContext<RegionCoprocessorEnvironment> c, Append append,
    Result result) throws IOException {
  if (CALLED.getAndIncrement() == 0) {
    Threads.sleepWithoutInterrupt(SLEEP_TIME);
  }
  return RegionObserver.super.postAppend(c, append, result);
}
 

@Override
public void prePut(final ObserverContext<RegionCoprocessorEnvironment> e, final Put put,
    final WALEdit edit, final Durability durability) throws IOException {
  Threads.sleep(sleepTime.get());
  if (ct.incrementAndGet() == 1) {
    throw new IOException("first call I fail");
  }
}
 

/**
 * Create an edit log at the given <code>dir</code> location. You should never have to load an
 * existing log. If there is a log at startup, it should have already been processed and deleted
 * by the time the WAL object is started up.
 * @param fs filesystem handle
 * @param rootDir path to where logs and oldlogs
 * @param logDir dir where wals are stored
 * @param archiveDir dir where wals are archived
 * @param conf configuration to use
 * @param listeners Listeners on WAL events. Listeners passed here will be registered before we do
 *          anything else; e.g. the Constructor {@link #rollWriter()}.
 * @param failIfWALExists If true IOException will be thrown if files related to this wal already
 *          exist.
 * @param prefix should always be hostname and port in distributed env and it will be URL encoded
 *          before being used. If prefix is null, "wal" will be used
 * @param suffix will be url encoded. null is treated as empty. non-empty must start with
 *          {@link org.apache.hadoop.hbase.wal.AbstractFSWALProvider#WAL_FILE_NAME_DELIMITER}
 */
public FSHLog(final FileSystem fs, final Path rootDir, final String logDir,
    final String archiveDir, final Configuration conf, final List<WALActionsListener> listeners,
    final boolean failIfWALExists, final String prefix, final String suffix) throws IOException {
  super(fs, rootDir, logDir, archiveDir, conf, listeners, failIfWALExists, prefix, suffix);
  this.minTolerableReplication = conf.getInt("hbase.regionserver.hlog.tolerable.lowreplication",
    CommonFSUtils.getDefaultReplication(fs, this.walDir));
  this.lowReplicationRollLimit = conf.getInt("hbase.regionserver.hlog.lowreplication.rolllimit",
    5);
  this.closeErrorsTolerated = conf.getInt("hbase.regionserver.logroll.errors.tolerated", 2);

  // This is the 'writer' -- a single threaded executor. This single thread 'consumes' what is
  // put on the ring buffer.
  String hostingThreadName = Thread.currentThread().getName();
  // Using BlockingWaitStrategy. Stuff that is going on here takes so long it makes no sense
  // spinning as other strategies do.
  this.disruptor = new Disruptor<>(RingBufferTruck::new,
      getPreallocatedEventCount(),
      Threads.newDaemonThreadFactory(hostingThreadName + ".append"),
      ProducerType.MULTI, new BlockingWaitStrategy());
  // Advance the ring buffer sequence so that it starts from 1 instead of 0,
  // because SyncFuture.NOT_DONE = 0.
  this.disruptor.getRingBuffer().next();
  int maxHandlersCount = conf.getInt(HConstants.REGION_SERVER_HANDLER_COUNT, 200);
  this.ringBufferEventHandler = new RingBufferEventHandler(
      conf.getInt("hbase.regionserver.hlog.syncer.count", 5), maxHandlersCount);
  this.disruptor.setDefaultExceptionHandler(new RingBufferExceptionHandler());
  this.disruptor.handleEventsWith(new RingBufferEventHandler[] { this.ringBufferEventHandler });
  // Starting up threads in constructor is a no no; Interface should have an init call.
  this.disruptor.start();
}
 

@Test
public void testYieldWhileHoldingLocks() {
  final AtomicBoolean lock = new AtomicBoolean(false);

  final TestLockProcedure p1 = new TestLockProcedure(lock, "key", true, false);
  final TestLockProcedure p2 = new TestLockProcedure(lock, "key", true, false);

  procExecutor.submitProcedure(p1);
  procExecutor.submitProcedure(p2);

  // try to execute a bunch of yield on p1, p2 should be blocked
  while (p1.getTimestamps().size() < 100) {
    Threads.sleep(10);
  }

  assertEquals(0, p2.getTimestamps().size());

  // wait until p1 is completed
  p1.setThrowYield(false);
  ProcedureTestingUtility.waitProcedure(procExecutor, p1);

  // try to execute a bunch of yield on p2
  while (p2.getTimestamps().size() < 100) {
    Threads.sleep(10);
  }

  assertEquals(p1.getTimestamps().get(p1.getTimestamps().size() - 1).longValue() + 1,
    p2.getTimestamps().get(0).longValue());

  // wait until p2 is completed
  p1.setThrowYield(false);
  ProcedureTestingUtility.waitProcedure(procExecutor, p1);
}
 

@Override
public void run() {
  boolean b = this.conf.getBoolean(RUN_SHUTDOWN_HOOK, true);
  LOG.info("Shutdown hook starting; " + RUN_SHUTDOWN_HOOK + "=" + b +
    "; fsShutdownHook=" + this.fsShutdownHook);
  if (b) {
    this.stop.stop("Shutdown hook");
    Threads.shutdown(this.threadToJoin);
    if (this.fsShutdownHook != null) {
      synchronized (fsShutdownHooks) {
        int refs = fsShutdownHooks.get(fsShutdownHook);
        if (refs == 1) {
          LOG.info("Starting fs shutdown hook thread.");
          Thread fsShutdownHookThread = (fsShutdownHook instanceof Thread) ?
            (Thread)fsShutdownHook : new Thread(fsShutdownHook,
              fsShutdownHook.getClass().getSimpleName() + "-shutdown-hook");
          fsShutdownHookThread.start();
          Threads.shutdown(fsShutdownHookThread,
          this.conf.getLong(FS_SHUTDOWN_HOOK_WAIT, 30000));
        }
        if (refs > 0) {
          fsShutdownHooks.put(fsShutdownHook, refs - 1);
        }
      }
    }
  }
  LOG.info("Shutdown hook finished.");
}
 

@Override
public void preGetOp(final ObserverContext<RegionCoprocessorEnvironment> e,
    final Get get, final List<Cell> results) throws IOException {
  // After first sleep, all requests are timeout except the last retry. If we handle
  // all the following requests, finally the last request is also timeout. If we drop all
  // timeout requests, we can handle the last request immediately and it will not timeout.
  if (ct.incrementAndGet() <= 1) {
    Threads.sleep(SLEEP_TIME * RPC_RETRY * 2);
  } else {
    Threads.sleep(SLEEP_TIME);
  }
}
 

private void waitForServiceToStart(ServiceType service, ServerName serverName, long timeout)
  throws IOException {
  LOG.info("Waiting for service: {} to start: ", service, serverName.getServerName());
  long start = System.currentTimeMillis();

  while ((System.currentTimeMillis() - start) < timeout) {
    if (clusterManager.isRunning(service, serverName.getHostname(), serverName.getPort())) {
      return;
    }
    Threads.sleep(100);
  }
  throw new IOException("did timeout waiting for service to start:" + serverName);
}