org.apache.hadoop.hbase.Waiter Java Examples

@BeforeClass
public static void setUpBeforeClass() throws Exception {
  conf = TEST_UTIL.getConfiguration();
  TEST_UTIL.startMiniCluster();
  TEST_UTIL.createTable(TableName.valueOf("TestStatusResource"), Bytes.toBytes("D"));
  TEST_UTIL.createTable(TableName.valueOf("TestStatusResource2"), Bytes.toBytes("D"));
  REST_TEST_UTIL.startServletContainer(conf);
  Cluster cluster = new Cluster();
  cluster.add("localhost", REST_TEST_UTIL.getServletPort());
  client = new Client(cluster);
  context = JAXBContext.newInstance(StorageClusterStatusModel.class);
  TEST_UTIL.waitFor(6000, new Waiter.Predicate<IOException>() {
    @Override
    public boolean evaluate() throws IOException {
      return TEST_UTIL.getMiniHBaseCluster().getClusterMetrics().getAverageLoad() > 0;
    }
  });
}
 

/**
 * Remove a peer and wait for it to get cleaned up
 */
private void removePeerAndWait(final String peerId) throws Exception {
  final ReplicationPeers rp = manager.getReplicationPeers();
  if (rp.getPeerStorage().listPeerIds().contains(peerId)) {
    rp.getPeerStorage().removePeer(peerId);
    try {
      manager.removePeer(peerId);
    } catch (Exception e) {
      // ignore the failed exception and continue.
    }
  }
  Waiter.waitFor(conf, 20000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      Collection<String> peers = rp.getPeerStorage().listPeerIds();
      return (!manager.getAllQueues().contains(peerId)) && (rp.getPeer(peerId) == null)
          && (!peers.contains(peerId)) && manager.getSource(peerId) == null;
    }
  });
}
 

private static void waitPeer(final String peerId,
    ReplicationSourceManager manager, final boolean waitForSource) {
  ReplicationPeers rp = manager.getReplicationPeers();
  Waiter.waitFor(conf, 20000, () -> {
    if (waitForSource) {
      ReplicationSourceInterface rs = manager.getSource(peerId);
      if (rs == null) {
        return false;
      }
      if (rs instanceof ReplicationSourceDummy) {
        return ((ReplicationSourceDummy)rs).isStartup();
      }
      return true;
    } else {
      return (rp.getPeer(peerId) != null);
    }
  });
}
 

@Test
public void testReplicationStatusSink() throws Exception {
  try (Admin admin = UTIL2.getConnection().getAdmin()) {
    ServerName server = UTIL2.getHBaseCluster().getRegionServer(0).getServerName();
    ReplicationLoadSink loadSink = getLatestSinkMetric(admin, server);
    //First checks if status of timestamp of last applied op is same as RS start, since no edits
    //were replicated yet
    Assert.assertEquals(loadSink.getTimestampStarted(), loadSink.getTimestampsOfLastAppliedOp());
    //now insert some rows on source, so that it gets delivered to target
    TestReplicationStatus.insertRowsOnSource();
    long wait =
      Waiter.waitFor(UTIL2.getConfiguration(), 10000, (Waiter.Predicate<Exception>) () -> {
        ReplicationLoadSink loadSink1 = getLatestSinkMetric(admin, server);
        return loadSink1.getTimestampsOfLastAppliedOp() > loadSink1.getTimestampStarted();
      });
    Assert.assertNotEquals(-1, wait);
  }
}
 

@Test
public void testRace() throws Exception {
  ProcedureExecutor<MasterProcedureEnv> executor =
    UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor();
  DummyProcedure p = new DummyProcedure();
  long procId = executor.submitProcedure(p);
  p.failureSet.await();
  assertEquals(GetProcedureResultResponse.State.RUNNING, getState(procId));
  p.canRollback.countDown();
  UTIL.waitFor(30000, new Waiter.ExplainingPredicate<Exception>() {

    @Override
    public boolean evaluate() throws Exception {
      return getState(procId) == GetProcedureResultResponse.State.FINISHED;
    }

    @Override
    public String explainFailure() throws Exception {
      return "Procedure pid=" + procId + " is still in " + getState(procId) +
        " state, expected " + GetProcedureResultResponse.State.FINISHED;
    }
  });
}
 

private void waitForTableToEnterQuotaViolation(TableName tn) throws Exception {
  // Verify that the RegionServer has the quota in violation
  final HRegionServer rs = TEST_UTIL.getHBaseCluster().getRegionServer(0);
  Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, 1000, new Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      Map<TableName,SpaceQuotaSnapshot> snapshots =
          rs.getRegionServerSpaceQuotaManager().copyQuotaSnapshots();
      SpaceQuotaSnapshot snapshot = snapshots.get(tn);
      if (snapshot == null) {
        LOG.info("Found no snapshot for " + tn);
        return false;
      }
      LOG.info("Found snapshot " + snapshot);
      return snapshot.getQuotaStatus().isInViolation();
    }
  });
}
 

@Test
public void testRegionSizesFromMaster() throws Exception {
  final long tableSize = 1024L * 10L; // 10KB
  final int numRegions = 10;
  final TableName tn = helper.createTableWithRegions(numRegions);
  // Will write at least `tableSize` data
  helper.writeData(tn, tableSize);

  final HMaster master = TEST_UTIL.getMiniHBaseCluster().getMaster();
  final MasterQuotaManager quotaManager = master.getMasterQuotaManager();
  // Make sure the master has all of the reports
  Waiter.waitFor(TEST_UTIL.getConfiguration(), 30 * 1000, new Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      Map<RegionInfo,Long> regionSizes = quotaManager.snapshotRegionSizes();
      LOG.trace("Region sizes=" + regionSizes);
      return numRegions == countRegionsForTable(tn, regionSizes) &&
          tableSize <= getTableSize(tn, regionSizes);
    }
  });

  Map<TableName, Long> sizes = TEST_UTIL.getAdmin().getSpaceQuotaTableSizes();
  Long size = sizes.get(tn);
  assertNotNull("No reported size for " + tn, size);
  assertTrue("Reported table size was " + size, size.longValue() >= tableSize);
}
 

@Test
public void testAsyncSnapshotWillNotBlockSnapshotHFileCleaner() throws Exception {
  // Write some data
  Table table = UTIL.getConnection().getTable(TABLE_NAME);
  for (int i = 0; i < 10; i++) {
    Put put = new Put(Bytes.toBytes(i)).addColumn(TEST_FAM, Bytes.toBytes("q"), Bytes.toBytes(i));
    table.put(put);
  }
  String snapshotName = "testAsyncSnapshotWillNotBlockSnapshotHFileCleaner01";
  Future<Void> future =
    UTIL.getAdmin().snapshotAsync(new org.apache.hadoop.hbase.client.SnapshotDescription(
      snapshotName, TABLE_NAME, SnapshotType.FLUSH));
  Waiter.waitFor(UTIL.getConfiguration(), 10 * 1000L, 200L,
    () -> UTIL.getAdmin().listSnapshots(Pattern.compile(snapshotName)).size() == 1);
  UTIL.waitFor(30000, () -> !master.getSnapshotManager().isTakingAnySnapshot());
}
 

private void stopServersAndWaitUntilProcessed(List<ServerName> currentFN) throws Exception {
  for (ServerName sn : currentFN) {
    for (JVMClusterUtil.RegionServerThread rst : cluster.getLiveRegionServerThreads()) {
      if (ServerName.isSameAddress(sn, rst.getRegionServer().getServerName())) {
        LOG.info("Shutting down server: " + sn);
        cluster.stopRegionServer(rst.getRegionServer().getServerName());
        cluster.waitForRegionServerToStop(rst.getRegionServer().getServerName(), 60000);
      }
    }
  }

  // Wait until dead servers are processed.
  TEST_UTIL.waitFor(60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return !master.getServerManager().areDeadServersInProgress();
    }
  });

  assertEquals("Not all servers killed",
      SLAVES - currentFN.size(), cluster.getLiveRegionServerThreads().size());
}
 

public static boolean waitForAssignment(AssignmentManager am, RegionInfo regionInfo)
    throws IOException {
  // This method can be called before the regionInfo has made it into the regionStateMap
  // so wait around here a while.
  Waiter.waitFor(am.getConfiguration(), 10000,
    () -> am.getRegionStates().getRegionStateNode(regionInfo) != null);
  RegionStateNode regionNode = am.getRegionStates().getRegionStateNode(regionInfo);
  // Wait until the region has already been open, or we have a TRSP along with it.
  Waiter.waitFor(am.getConfiguration(), 30000,
    () -> regionNode.isInState(State.OPEN) || regionNode.isInTransition());
  TransitRegionStateProcedure proc = regionNode.getProcedure();
  regionNode.lock();
  try {
    if (regionNode.isInState(State.OPEN)) {
      return true;
    }
    proc = regionNode.getProcedure();
  } finally {
    regionNode.unlock();
  }
  assertNotNull(proc);
  ProcedureSyncWait.waitForProcedureToCompleteIOE(am.getMaster().getMasterProcedureExecutor(),
    proc, 5L * 60 * 1000);
  return true;
}
 

@BeforeClass
public static void setUp() throws Exception {
  TEST_UTIL = new HBaseTestingUtility();
  RSGroupUtil.enableRSGroup(TEST_UTIL.getConfiguration());
  TEST_UTIL.getConfiguration().set(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, "1");
  StartMiniClusterOption option =
    StartMiniClusterOption.builder().numMasters(2).numRegionServers(3).numDataNodes(3).build();
  TEST_UTIL.startMiniCluster(option);
  cluster = TEST_UTIL.getHBaseCluster();
  master = ((MiniHBaseCluster) cluster).getMaster();
  master.balanceSwitch(false);
  hbaseAdmin = TEST_UTIL.getAdmin();
  // wait till the balancer is in online mode
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return master.isInitialized() &&
        ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline() &&
        master.getServerManager().getOnlineServersList().size() >= 3;
    }
  });
}
 

/**
 * Waits until there is only one log(the current writing one) in the replication queue
 * @param numRs number of regionservers
 */
private void waitForLogAdvance(int numRs) throws Exception {
  Waiter.waitFor(CONF1, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      for (int i = 0; i < numRs; i++) {
        HRegionServer hrs = UTIL1.getHBaseCluster().getRegionServer(i);
        RegionInfo regionInfo =
            UTIL1.getHBaseCluster().getRegions(htable1.getName()).get(0).getRegionInfo();
        WAL wal = hrs.getWAL(regionInfo);
        Path currentFile = ((AbstractFSWAL<?>) wal).getCurrentFileName();
        Replication replicationService = (Replication) UTIL1.getHBaseCluster()
            .getRegionServer(i).getReplicationSourceService();
        for (ReplicationSourceInterface rsi : replicationService.getReplicationManager()
            .getSources()) {
          ReplicationSource source = (ReplicationSource) rsi;
          if (!currentFile.equals(source.getCurrentPath())) {
            return false;
          }
        }
      }
      return true;
    }
  });
}
 

@Test
public void removeClosedRegionFromConfigurationManager() throws Exception {
  try (Connection connection = ConnectionFactory.createConnection(conf)) {
    Admin admin = connection.getAdmin();
    assertTrue("The open region doesn't register as a ConfigurationObserver",
      rs1.getConfigurationManager().containsObserver(r1));
    admin.move(r1name);
    hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
      @Override public boolean evaluate() throws Exception {
        return rs1.getOnlineRegion(r1name) == null;
      }
    });
    assertFalse("The closed region is not removed from ConfigurationManager",
      rs1.getConfigurationManager().containsObserver(r1));
    admin.move(r1name, rs1.getServerName());
    hbaseTestingUtility.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
      @Override public boolean evaluate() throws Exception {
        return rs1.getOnlineRegion(r1name) != null;
      }
    });
  }
}
 

private boolean waitForCounterBoolean(final LongAdder ctr, final long oldval, final long newval,
    long timems, boolean failIfTimeout) throws Exception {

  long timeWaited = TEST_UTIL.waitFor(timems, 10, failIfTimeout,
    new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
          return (ctr.sum() >= newval);
    }
  });

  if( timeWaited > 0) {
    // when not timed out
    assertEquals(newval, ctr.sum());
  }
  return true;
}
 

protected static void initialize() throws Exception {
  ADMIN = new VerifyingRSGroupAdmin(TEST_UTIL.getConfiguration());
  CLUSTER = TEST_UTIL.getHBaseCluster();
  MASTER = TEST_UTIL.getMiniHBaseCluster().getMaster();

  // wait for balancer to come online
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return MASTER.isInitialized() &&
        ((RSGroupBasedLoadBalancer) MASTER.getLoadBalancer()).isOnline();
    }
  });
  ADMIN.balancerSwitch(false, true);
  MasterCoprocessorHost host = MASTER.getMasterCoprocessorHost();
  OBSERVER = (CPMasterObserver) host.findCoprocessor(CPMasterObserver.class.getName());
}
 

private Pair<ServerName, RegionStateNode> createTableWithRegionSplitting(RSGroupInfo rsGroupInfo,
  int tableRegionCount) throws Exception {
  final byte[] familyNameBytes = Bytes.toBytes("f");
  // All the regions created below will be assigned to the default group.
  TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, tableRegionCount);
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      List<String> regions = getTableRegionMap().get(tableName);
      if (regions == null) {
        return false;
      }
      return getTableRegionMap().get(tableName).size() >= tableRegionCount;
    }
  });

  return randomlySetOneRegionStateToSplitting(rsGroupInfo);
}
 

@Before
@Override
public void setUp() throws Exception {
  utility = new HBaseTestingUtility();
  Configuration conf = utility.getConfiguration();
  RSGroupUtil.enableRSGroup(conf);
  conf.setInt(ServerManager.WAIT_ON_REGIONSERVERS_MINTOSTART, NUM_SLAVES_BASE);
  conf.setInt("hbase.hfile.compaction.discharger.interval", 10);
  utility.startMiniCluster(NUM_SLAVES_BASE);
  MiniHBaseCluster cluster = utility.getHBaseCluster();
  final HMaster master = cluster.getMaster();

  //wait for balancer to come online
  utility.waitFor(60000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() {
      return master.isInitialized() &&
          ((RSGroupBasedLoadBalancer) master.getLoadBalancer()).isOnline();
    }
  });
  admin = utility.getAdmin();
}
 

@Test
public void testDefaultNamespaceCreateAndAssign() throws Exception {
  LOG.info("testDefaultNamespaceCreateAndAssign");
  String tableName = TABLE_PREFIX + "_testCreateAndAssign";
  ADMIN.modifyNamespace(NamespaceDescriptor.create("default")
    .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "default").build());
  final TableDescriptor desc = TableDescriptorBuilder.newBuilder(TableName.valueOf(tableName))
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
  ADMIN.createTable(desc);
  // wait for created table to be assigned
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return getTableRegionMap().get(desc.getTableName()) != null;
    }
  });
}
 

@Test
public void testNamespaceCreateAndAssign() throws Exception {
  LOG.info("testNamespaceCreateAndAssign");
  String nsName = TABLE_PREFIX + "_foo";
  final TableName tableName = TableName.valueOf(nsName, TABLE_PREFIX + "_testCreateAndAssign");
  RSGroupInfo appInfo = addGroup("appInfo", 1);
  ADMIN.createNamespace(NamespaceDescriptor.create(nsName)
    .addConfiguration(RSGroupInfo.NAMESPACE_DESC_PROP_GROUP, "appInfo").build());
  final TableDescriptor desc = TableDescriptorBuilder.newBuilder(tableName)
    .setColumnFamily(ColumnFamilyDescriptorBuilder.of("f")).build();
  ADMIN.createTable(desc);
  // wait for created table to be assigned
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return getTableRegionMap().get(desc.getTableName()) != null;
    }
  });
  ServerName targetServer = getServerName(appInfo.getServers().iterator().next());
  // verify it was assigned to the right group
  Assert.assertEquals(1, ADMIN.getRegions(targetServer).size());
}
 

private void startCluster(int numRS) throws Exception {
  SplitLogCounters.resetCounters();
  LOG.info("Starting cluster");
  conf.setLong("hbase.splitlog.max.resubmit", 0);
  // Make the failure test faster
  conf.setInt("zookeeper.recovery.retry", 0);
  conf.setInt(HConstants.REGIONSERVER_INFO_PORT, -1);
  conf.setFloat(HConstants.LOAD_BALANCER_SLOP_KEY, (float) 100.0); // no load balancing
  conf.setInt(HBASE_SPLIT_WAL_MAX_SPLITTER, 3);
  conf.setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 10);
  conf.set("hbase.wal.provider", getWalProvider());
  StartMiniClusterOption option = StartMiniClusterOption.builder()
      .numMasters(NUM_MASTERS).numRegionServers(numRS).build();
  TEST_UTIL.startMiniHBaseCluster(option);
  cluster = TEST_UTIL.getHBaseCluster();
  LOG.info("Waiting for active/ready master");
  cluster.waitForActiveAndReadyMaster();
  master = cluster.getMaster();
  TEST_UTIL.waitFor(120000, 200, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return cluster.getLiveRegionServerThreads().size() >= numRS;
    }
  });
}
 

@Test
public void testCreateAndDrop() throws Exception {
  TEST_UTIL.createTable(tableName, Bytes.toBytes("cf"));
  // wait for created table to be assigned
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return getTableRegionMap().get(tableName) != null;
    }
  });
  TEST_UTIL.deleteTable(tableName);
}
 

private void assertTriggerFlushByChanges(int changes) throws InterruptedException {
  int currentFlushCalled = flushCalled.get();
  for (int i = 0; i < changes; i++) {
    flusher.onUpdate();
  }
  Thread.sleep(1000);
  assertEquals(currentFlushCalled, flushCalled.get());
  flusher.onUpdate();
  Waiter.waitFor(conf, 5000, () -> flushCalled.get() == currentFlushCalled + 1);
}
 

@Test
public void testThreeRSAbort() throws Exception {
  LOG.info("testThreeRSAbort");
  int numRegionsToCreate = 40;
  int numRowsPerRegion = 100;

  startCluster(NUM_RS); // NUM_RS=6.

  try (Table table = installTable(numRegionsToCreate)) {
    populateDataInTable(numRowsPerRegion);

    List<RegionServerThread> rsts = cluster.getLiveRegionServerThreads();
    assertEquals(NUM_RS, rsts.size());
    cluster.killRegionServer(rsts.get(0).getRegionServer().getServerName());
    cluster.killRegionServer(rsts.get(1).getRegionServer().getServerName());
    cluster.killRegionServer(rsts.get(2).getRegionServer().getServerName());

    TEST_UTIL.waitFor(60000, new Waiter.ExplainingPredicate<Exception>() {

      @Override
      public boolean evaluate() throws Exception {
        return cluster.getLiveRegionServerThreads().size() <= NUM_RS - 3;
      }

      @Override
      public String explainFailure() throws Exception {
        return "Timed out waiting for server aborts.";
      }
    });
    TEST_UTIL.waitUntilAllRegionsAssigned(tableName);
    int rows;
    try {
      rows = TEST_UTIL.countRows(table);
    } catch (Exception e) {
      Threads.printThreadInfo(System.out, "Thread dump before fail");
      throw e;
    }
    assertEquals(numRegionsToCreate * numRowsPerRegion, rows);
  }
}
 

@Test
public void testSafemodeBringsDownMaster() throws Exception {
  final TableName tableName = TableName.valueOf("testSafemodeBringsDownMaster");
  final byte[][] splitKeys = new byte[][] {
    Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")
  };
  RegionInfo[] regions = MasterProcedureTestingUtility.createTable(
      getMasterProcedureExecutor(), tableName, splitKeys, "f1", "f2");
  MiniDFSCluster dfsCluster = UTIL.getDFSCluster();
  DistributedFileSystem dfs = (DistributedFileSystem) dfsCluster.getFileSystem();
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
  final long timeOut = 180000;
  long startTime = System.currentTimeMillis();
  int index = -1;
  do {
    index = UTIL.getMiniHBaseCluster().getServerWithMeta();
  } while (index == -1 &&
    startTime + timeOut < System.currentTimeMillis());

  if (index != -1){
    UTIL.getMiniHBaseCluster().abortRegionServer(index);
    UTIL.getMiniHBaseCluster().waitOnRegionServer(index);
  }
  UTIL.waitFor(timeOut, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      List<JVMClusterUtil.MasterThread> threads = UTIL.getMiniHBaseCluster().getLiveMasterThreads();
      return threads == null || threads.isEmpty();
    }
  });
  dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
}
 

/**
 * Tests the replication scenario 0 -> 0. By default
 * {@link BaseReplicationEndpoint#canReplicateToSameCluster()} returns false, so the
 * ReplicationSource should terminate, and no further logs should get enqueued
 */
@Test
public void testLoopedReplication() throws Exception {
  LOG.info("testLoopedReplication");
  startMiniClusters(1);
  createTableOnClusters(table);
  addPeer("1", 0, 0);
  Thread.sleep(SLEEP_TIME);

  // wait for source to terminate
  final ServerName rsName = utilities[0].getHBaseCluster().getRegionServer(0).getServerName();
  Waiter.waitFor(baseConfiguration, 10000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      ClusterMetrics clusterStatus = utilities[0].getAdmin()
          .getClusterMetrics(EnumSet.of(ClusterMetrics.Option.LIVE_SERVERS));
      ServerMetrics serverLoad = clusterStatus.getLiveServerMetrics().get(rsName);
      List<ReplicationLoadSource> replicationLoadSourceList =
          serverLoad.getReplicationLoadSourceList();
      return replicationLoadSourceList.isEmpty();
    }
  });

  Table[] htables = getHTablesOnClusters(tableName);
  putAndWait(row, famName, htables[0], htables[0]);
  rollWALAndWait(utilities[0], table.getTableName(), row);
  ZKWatcher zkw = utilities[0].getZooKeeperWatcher();
  String queuesZnode = ZNodePaths.joinZNode(zkw.getZNodePaths().baseZNode,
    ZNodePaths.joinZNode("replication", "rs"));
  List<String> listChildrenNoWatch =
      ZKUtil.listChildrenNoWatch(zkw, ZNodePaths.joinZNode(queuesZnode, rsName.toString()));
  assertEquals(0, listChildrenNoWatch.size());
}
 

/**
 * Test no new Compaction requests are generated after calling stop compactions
 */
@Test
public void testStopStartCompaction() throws IOException {
  // setup a compact/split thread on a mock server
  HRegionServer mockServer = Mockito.mock(HRegionServer.class);
  Mockito.when(mockServer.getConfiguration()).thenReturn(r.getBaseConf());
  final CompactSplit thread = new CompactSplit(mockServer);
  Mockito.when(mockServer.getCompactSplitThread()).thenReturn(thread);
  // setup a region/store with some files
  HStore store = r.getStore(COLUMN_FAMILY);
  createStoreFile(r);
  for (int i = 0; i < HStore.DEFAULT_BLOCKING_STOREFILE_COUNT - 1; i++) {
    createStoreFile(r);
  }
  thread.switchCompaction(false);
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
    CompactionLifeCycleTracker.DUMMY, null);
  assertFalse(thread.isCompactionsEnabled());
  int longCompactions = thread.getLongCompactions().getActiveCount();
  int shortCompactions = thread.getShortCompactions().getActiveCount();
  assertEquals("longCompactions=" + longCompactions + "," +
      "shortCompactions=" + shortCompactions, 0, longCompactions + shortCompactions);
  thread.switchCompaction(true);
  assertTrue(thread.isCompactionsEnabled());
  // Make sure no compactions have run.
  assertEquals(0, thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount());
  // Request a compaction and make sure it is submitted successfully.
  thread.requestCompaction(r, store, "test", Store.PRIORITY_USER,
      CompactionLifeCycleTracker.DUMMY, null);
  // Wait until the compaction finishes.
  Waiter.waitFor(UTIL.getConfiguration(), 5000,
      (Waiter.Predicate<Exception>) () -> thread.getLongCompactions().getCompletedTaskCount() +
      thread.getShortCompactions().getCompletedTaskCount() == 1);
  // Make sure there are no compactions running.
  assertEquals(0, thread.getLongCompactions().getActiveCount()
      + thread.getShortCompactions().getActiveCount());
}
 

public void tearDownAfterMethod() throws Exception {
  deleteTableIfNecessary();
  deleteNamespaceIfNecessary();
  deleteGroups();

  for (ServerName sn : ADMIN.listDecommissionedRegionServers()) {
    ADMIN.recommissionRegionServer(sn, null);
  }
  assertTrue(ADMIN.listDecommissionedRegionServers().isEmpty());

  int missing = NUM_SLAVES_BASE - getNumServers();
  LOG.info("Restoring servers: " + missing);
  for (int i = 0; i < missing; i++) {
    ((MiniHBaseCluster) CLUSTER).startRegionServer();
  }
  ADMIN.addRSGroup("master");
  ServerName masterServerName = ((MiniHBaseCluster) CLUSTER).getMaster().getServerName();
  try {
    ADMIN.moveServersToRSGroup(Sets.newHashSet(masterServerName.getAddress()), "master");
  } catch (Exception ex) {
    LOG.warn("Got this on setup, FYI", ex);
  }
  assertTrue(OBSERVER.preMoveServersCalled);
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      LOG.info("Waiting for cleanup to finish " + ADMIN.listRSGroups());
      // Might be greater since moving servers back to default
      // is after starting a server

      return ADMIN.getRSGroup(RSGroupInfo.DEFAULT_GROUP).getServers().size() == NUM_SLAVES_BASE;
    }
  });
}
 

@Test
public void testFailedMoveBeforeRetryExhaustedWhenMoveServer() throws Exception {
  String groupName = getGroupName(name.getMethodName());
  ADMIN.addRSGroup(groupName);
  final RSGroupInfo newGroup = ADMIN.getRSGroup(groupName);
  Pair<ServerName, RegionStateNode> gotPair = createTableWithRegionSplitting(newGroup, 10);

  // start thread to recover region state
  final ServerName movedServer = gotPair.getFirst();
  final RegionStateNode rsn = gotPair.getSecond();
  AtomicBoolean changed = new AtomicBoolean(false);
  Thread t1 = recoverRegionStateThread(movedServer,
    server -> MASTER.getAssignmentManager().getRegionsOnServer(movedServer), rsn, changed);
  t1.start();

  // move target server to group
  Thread t2 = new Thread(() -> {
    LOG.info("thread2 start running, to move regions");
    try {
      ADMIN.moveServersToRSGroup(Sets.newHashSet(movedServer.getAddress()), newGroup.getName());
    } catch (IOException e) {
      LOG.error("move server error", e);
    }
  });
  t2.start();

  t1.join();
  t2.join();

  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() {
      if (changed.get()) {
        return MASTER.getAssignmentManager().getRegionsOnServer(movedServer).size() == 0 &&
          !rsn.getRegionLocation().equals(movedServer);
      }
      return false;
    }
  });
}
 

@Test
public void testNotMoveTableToNullRSGroupWhenCreatingExistingTable() throws Exception {
  // Trigger
  TableName tn1 = TableName.valueOf("t1");
  TEST_UTIL.createTable(tn1, "cf1");
  try {
    // Create an existing table to trigger HBASE-21866
    TEST_UTIL.createTable(tn1, "cf1");
  } catch (TableExistsException teex) {
    // Ignore
  }

  // Wait then verify
  // Could not verify until the rollback of CreateTableProcedure is done
  // (that is, the coprocessor finishes its work),
  // or the table is still in the "default" rsgroup even though HBASE-21866
  // is not fixed.
  TEST_UTIL.waitFor(5000, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      return MASTER.getMasterProcedureExecutor().getActiveExecutorCount() == 0;
    }
  });
  Set<TableName> tables = Sets.newHashSet(ADMIN.listTablesInRSGroup(RSGroupInfo.DEFAULT_GROUP));
  assertTrue("Table 't1' must be in 'default' rsgroup", tables.contains(tn1));

  // Cleanup
  TEST_UTIL.deleteTable(tn1);
}
 

@Test
public void testDisabledTableMove() throws Exception {
  final byte[] familyNameBytes = Bytes.toBytes("f");
  String newGroupName = getGroupName(getNameWithoutIndex(name.getMethodName()));
  final RSGroupInfo newGroup = addGroup(newGroupName, 2);

  TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5);
  TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() {
    @Override
    public boolean evaluate() throws Exception {
      List<String> regions = getTableRegionMap().get(tableName);
      if (regions == null) {
        return false;
      }
      return getTableRegionMap().get(tableName).size() >= 5;
    }
  });

  RSGroupInfo tableGrp = ADMIN.getRSGroup(tableName);
  assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP));

  // test disable table
  ADMIN.disableTable(tableName);

  // change table's group
  LOG.info("Moving table " + tableName + " to " + newGroup.getName());
  ADMIN.setRSGroup(Sets.newHashSet(tableName), newGroup.getName());

  // verify group change
  assertEquals(newGroup.getName(), ADMIN.getRSGroup(tableName).getName());
}