Java Code Examples for org.apache.solr.common.cloud.Slice#getName()

private void getCheckpoints() throws IOException {
  this.checkpoints = new HashMap<>();
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

  Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();

  for(Slice slice : slices) {
    String sliceName = slice.getName();
    long checkpoint;
    if(initialCheckpoint > -1) {
      checkpoint = initialCheckpoint;
    } else {
      checkpoint = getCheckpoint(slice, liveNodes);
    }

    this.checkpoints.put(sliceName, checkpoint);
  }
}
 

static List<ZkNodeProps> getReplicasOfNode(String source, ClusterState state) {
  List<ZkNodeProps> sourceReplicas = new ArrayList<>();
  for (Map.Entry<String, DocCollection> e : state.getCollectionsMap().entrySet()) {
    for (Slice slice : e.getValue().getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (source.equals(replica.getNodeName())) {
          ZkNodeProps props = new ZkNodeProps(
              COLLECTION_PROP, e.getKey(),
              SHARD_ID_PROP, slice.getName(),
              ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
              ZkStateReader.REPLICA_PROP, replica.getName(),
              ZkStateReader.REPLICA_TYPE, replica.getType().name(),
              ZkStateReader.LEADER_PROP, String.valueOf(replica.equals(slice.getLeader())),
              CoreAdminParams.NODE, source);
          sourceReplicas.add(props);
        }
      }
    }
  }
  return sourceReplicas;
}
 

public ZkWriteCommand removeRoutingRule(final ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  String shard = message.getStr(ZkStateReader.SHARD_ID_PROP);
  String routeKeyStr = message.getStr("routeKey");

  log.info("Overseer.removeRoutingRule invoked for collection: {} shard: {} routeKey: {}"
      , collectionName, shard, routeKeyStr);

  DocCollection collection = clusterState.getCollection(collectionName);
  Slice slice = collection.getSlice(shard);
  if (slice == null) {
    log.warn("Unknown collection: {} shard: {}", collectionName, shard);
    return ZkStateWriter.NO_OP;
  }
  Map<String, RoutingRule> routingRules = slice.getRoutingRules();
  if (routingRules != null) {
    routingRules.remove(routeKeyStr); // no rules left
    Map<String, Object> props = slice.shallowCopy();
    props.put("routingRules", routingRules);
    Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props,collectionName);
    return new ZkWriteCommand(collectionName,
        CollectionMutator.updateSlice(collectionName, collection, newSlice));
  }

  return ZkStateWriter.NO_OP;
}
 

private String getShardId(DocCollection collection, String coreNodeName) {
  if (collection == null) return null;
  Map<String,Slice> slices = collection.getSlicesMap();
  if (slices != null) {
    for (Slice slice : slices.values()) {
      for (Replica replica : slice.getReplicas()) {
        String cnn = replica.getName();
        if (coreNodeName.equals(cnn)) {
          return slice.getName();
        }
      }
    }
  }
  return null;
}
 

private Map<String, Integer> getShardToDocCountMap(CloudSolrClient client, DocCollection docCollection) throws SolrServerException, IOException {
  Map<String,Integer> shardToDocCount = new TreeMap<>();
  for (Slice slice : docCollection.getActiveSlices()) {
    String shardName = slice.getName();
    try (HttpSolrClient leaderClient = new HttpSolrClient.Builder(slice.getLeader().getCoreUrl()).withHttpClient(client.getHttpClient()).build()) {
      long docsInShard = leaderClient.query(new SolrQuery("*:*").setParam("distrib", "false"))
          .getResults().getNumFound();
      shardToDocCount.put(shardName, (int) docsInShard);
    }
  }
  return shardToDocCount;
}
 

private void verifyLeaderAssignment(CloudSolrClient client, String collectionName)
    throws InterruptedException, KeeperException {
  String lastFailMsg = "";
  for (int idx = 0; idx < 300; ++idx) { // Keep trying while Overseer writes the ZK state for up to 30 seconds.
    lastFailMsg = "";
    ClusterState clusterState = client.getZkStateReader().getClusterState();
    for (Slice slice : clusterState.getCollection(collectionName).getSlices()) {
      Boolean foundLeader = false;
      Boolean foundPreferred = false;
      for (Replica replica : slice.getReplicas()) {
        Boolean isLeader = replica.getBool("leader", false);
        Boolean isPreferred = replica.getBool("property.preferredleader", false);
        if (isLeader != isPreferred) {
          lastFailMsg = "Replica should NOT have preferredLeader != leader. Preferred: " + isPreferred.toString() +
              " leader is " + isLeader.toString();
        }
        if (foundLeader && isLeader) {
          lastFailMsg = "There should only be a single leader in _any_ shard! Replica " + replica.getName() +
              " is the second leader in slice " + slice.getName();
        }
        if (foundPreferred && isPreferred) {
          lastFailMsg = "There should only be a single preferredLeader in _any_ shard! Replica " + replica.getName() +
              " is the second preferredLeader in slice " + slice.getName();
        }
        foundLeader = foundLeader ? foundLeader : isLeader;
        foundPreferred = foundPreferred ? foundPreferred : isPreferred;
      }
    }
    if (lastFailMsg.length() == 0) return;
    Thread.sleep(100);
  }
  fail(lastFailMsg);
}
 

public QueryResponse simQuery(QueryRequest req) throws SolrException, InterruptedException, IOException {
  ensureNotClosed();
  String collection = req.getCollection();
  if (collection == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection not set");
  }
  ensureSystemCollection(collection);
  if (!colShardReplicaMap.containsKey(collection)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection does not exist");
  }
  String query = req.getParams().get(CommonParams.Q);
  if (query == null || !query.equals("*:*")) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Only '*:*' query is supported");
  }
  ClusterState clusterState = getClusterState();
  DocCollection coll = clusterState.getCollection(collection);
  AtomicLong count = new AtomicLong();
  for (Slice s : coll.getActiveSlicesArr()) {
    Replica r = s.getLeader();
    if (r == null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, collection + "/" + s.getName() + " has no leader");
    }
    ReplicaInfo ri = getReplicaInfo(r);
    Number numDocs = (Number)ri.getVariable("SEARCHER.searcher.numDocs", 0L);
    count.addAndGet(numDocs.longValue());
    AtomicLong bufferedUpdates = (AtomicLong)sliceProperties.get(collection).get(s.getName()).get(BUFFERED_UPDATES);
    if (bufferedUpdates != null) {
      count.addAndGet(bufferedUpdates.get());
    }
  }
  QueryResponse rsp = new QueryResponse();
  NamedList<Object> values = new NamedList<>();
  values.add("responseHeader", new NamedList<>());
  SolrDocumentList docs = new SolrDocumentList();
  docs.setNumFound(count.get());
  values.add("response", docs);
  rsp.setResponse(values);
  return rsp;
}
 

public static String getAssignedId(final DocCollection collection, final String nodeName) {
  Collection<Slice> slices = collection != null ? collection.getSlices() : null;
  if (slices != null) {
    for (Slice slice : slices) {
      if (slice.getReplicasMap().get(nodeName) != null) {
        return slice.getName();
      }
    }
  }
  return null;
}
 

public static DocCollection updateReplica(DocCollection collection, final Slice slice, String coreNodeName, final Replica replica) {
  Map<String, Replica> replicasCopy = slice.getReplicasCopy();
  if (replica == null) {
    replicasCopy.remove(coreNodeName);
  } else {
    replicasCopy.put(replica.getName(), replica);
  }
  Slice newSlice = new Slice(slice.getName(), replicasCopy, slice.getProperties(), collection.getName());
  log.debug("Old Slice: {}", slice);
  log.debug("New Slice: {}", newSlice);
  return CollectionMutator.updateSlice(collection.getName(), collection, newSlice);
}
 

/** Constructs a slices map from a collection of slices and handles disambiguation if multiple collections are being queried simultaneously */
public static void addSlices(Map<String,Slice> target, String collectionName, Collection<Slice> slices, boolean multiCollection) {
  for (Slice slice : slices) {
    String key = slice.getName();
    if (multiCollection) key = collectionName + "_" + key;
    target.put(key, slice);
  }
}
 

public ZkWriteCommand updateShardState(ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  log.info("Update shard state invoked for collection: {} with message: {}", collectionName, message);

  DocCollection collection = clusterState.getCollection(collectionName);
  Map<String, Slice> slicesCopy = new LinkedHashMap<>(collection.getSlicesMap());
  for (String key : message.keySet()) {
    if (ZkStateReader.COLLECTION_PROP.equals(key)) continue;
    if (Overseer.QUEUE_OPERATION.equals(key)) continue;

    Slice slice = collection.getSlice(key);
    if (slice == null) {
      throw new RuntimeException("Overseer.updateShardState unknown collection: " + collectionName + " slice: " + key);
    }
    if (log.isInfoEnabled()) {
      log.info("Update shard state {} to {}", key, message.getStr(key));
    }
    Map<String, Object> props = slice.shallowCopy();
    
    if (Slice.State.getState(message.getStr(key)) == Slice.State.ACTIVE) {
      props.remove(Slice.PARENT);
      props.remove("shard_parent_node");
      props.remove("shard_parent_zk_session");
    }
    props.put(ZkStateReader.STATE_PROP, message.getStr(key));
    // we need to use epoch time so that it's comparable across Overseer restarts
    props.put(ZkStateReader.STATE_TIMESTAMP_PROP, String.valueOf(cloudManager.getTimeSource().getEpochTimeNs()));
    Slice newSlice = new Slice(slice.getName(), slice.getReplicasCopy(), props,collectionName);
    slicesCopy.put(slice.getName(), newSlice);
  }

  return new ZkWriteCommand(collectionName, collection.copyWithSlices(slicesCopy));
}
 

private List<ZkNodeProps> getReplicasForSlice(String collectionName, Slice slice) {
  List<ZkNodeProps> sourceReplicas = new ArrayList<>();
  for (Replica replica : slice.getReplicas()) {
    ZkNodeProps props = new ZkNodeProps(
        COLLECTION_PROP, collectionName,
        SHARD_ID_PROP, slice.getName(),
        ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
        ZkStateReader.REPLICA_PROP, replica.getName(),
        CoreAdminParams.NODE, replica.getNodeName());
    sourceReplicas.add(props);
  }
  return sourceReplicas;
}
 

private SolrCmdDistributor.Node getLeaderNode(String collection, Slice slice) {
  //TODO when should we do StdNode vs RetryNode?
  final Replica leader = slice.getLeader();
  if (leader == null) {
    throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
        "No 'leader' replica available for shard " + slice.getName() + " of collection " + collection);
  }
  return new SolrCmdDistributor.ForwardNode(new ZkCoreNodeProps(leader), zkController.getZkStateReader(),
      collection, slice.getName(), DistributedUpdateProcessor.MAX_RETRIES_ON_FORWARD_DEAULT);
}
 

public List<ZkWriteCommand> downNode(ClusterState clusterState, ZkNodeProps message) {
  List<ZkWriteCommand> zkWriteCommands = new ArrayList<>();
  String nodeName = message.getStr(ZkStateReader.NODE_NAME_PROP);

  log.debug("DownNode state invoked for node: {}", nodeName);

  Map<String, DocCollection> collections = clusterState.getCollectionsMap();
  for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
    String collection = entry.getKey();
    DocCollection docCollection = entry.getValue();

    Map<String,Slice> slicesCopy = new LinkedHashMap<>(docCollection.getSlicesMap());

    boolean needToUpdateCollection = false;
    for (Entry<String, Slice> sliceEntry : slicesCopy.entrySet()) {
      Slice slice = sliceEntry.getValue();
      Map<String, Replica> newReplicas = slice.getReplicasCopy();

      Collection<Replica> replicas = slice.getReplicas();
      for (Replica replica : replicas) {
        String rNodeName = replica.getNodeName();
        if (rNodeName == null) {
          throw new RuntimeException("Replica without node name! " + replica);
        }
        if (rNodeName.equals(nodeName)) {
          log.debug("Update replica state for {} to {}", replica, Replica.State.DOWN);
          Map<String, Object> props = replica.shallowCopy();
          props.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
          Replica newReplica = new Replica(replica.getName(), props, collection, slice.getName());
          newReplicas.put(replica.getName(), newReplica);
          needToUpdateCollection = true;
        }
      }

      Slice newSlice = new Slice(slice.getName(), newReplicas, slice.shallowCopy(),collection);
      slicesCopy.put(slice.getName(), newSlice);
    }

    if (needToUpdateCollection) {
      zkWriteCommands.add(new ZkWriteCommand(collection, docCollection.copyWithSlices(slicesCopy)));
    }
  }

  return zkWriteCommands;
}
 

@SuppressWarnings({"unchecked"})
private void copyIndexFiles(URI backupPath, String collectionName, ZkNodeProps request, @SuppressWarnings({"rawtypes"})NamedList results) throws Exception {
  String backupName = request.getStr(NAME);
  String asyncId = request.getStr(ASYNC);
  String repoName = request.getStr(CoreAdminParams.BACKUP_REPOSITORY);
  ShardHandler shardHandler = ocmh.shardHandlerFactory.getShardHandler(ocmh.overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  String commitName = request.getStr(CoreAdminParams.COMMIT_NAME);
  Optional<CollectionSnapshotMetaData> snapshotMeta = Optional.empty();
  if (commitName != null) {
    SolrZkClient zkClient = ocmh.zkStateReader.getZkClient();
    snapshotMeta = SolrSnapshotManager.getCollectionLevelSnapshot(zkClient, collectionName, commitName);
    if (!snapshotMeta.isPresent()) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName
          + " does not exist for collection " + collectionName);
    }
    if (snapshotMeta.get().getStatus() != SnapshotStatus.Successful) {
      throw new SolrException(ErrorCode.BAD_REQUEST, "Snapshot with name " + commitName + " for collection " + collectionName
          + " has not completed successfully. The status is " + snapshotMeta.get().getStatus());
    }
  }

  log.info("Starting backup of collection={} with backupName={} at location={}", collectionName, backupName,
      backupPath);

  Collection<String> shardsToConsider = Collections.emptySet();
  if (snapshotMeta.isPresent()) {
    shardsToConsider = snapshotMeta.get().getShards();
  }

  final ShardRequestTracker shardRequestTracker = ocmh.asyncRequestTracker(asyncId);
  for (Slice slice : ocmh.zkStateReader.getClusterState().getCollection(collectionName).getActiveSlices()) {
    Replica replica = null;

    if (snapshotMeta.isPresent()) {
      if (!shardsToConsider.contains(slice.getName())) {
        log.warn("Skipping the backup for shard {} since it wasn't part of the collection {} when snapshot {} was created.",
            slice.getName(), collectionName, snapshotMeta.get().getName());
        continue;
      }
      replica = selectReplicaWithSnapshot(snapshotMeta.get(), slice);
    } else {
      // Note - Actually this can return a null value when there is no leader for this shard.
      replica = slice.getLeader();
      if (replica == null) {
        throw new SolrException(ErrorCode.SERVER_ERROR, "No 'leader' replica available for shard " + slice.getName() + " of collection " + collectionName);
      }
    }

    String coreName = replica.getStr(CORE_NAME_PROP);

    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.BACKUPCORE.toString());
    params.set(NAME, slice.getName());
    params.set(CoreAdminParams.BACKUP_REPOSITORY, repoName);
    params.set(CoreAdminParams.BACKUP_LOCATION, backupPath.toASCIIString()); // note: index dir will be here then the "snapshot." + slice name
    params.set(CORE_NAME_PROP, coreName);
    if (snapshotMeta.isPresent()) {
      params.set(CoreAdminParams.COMMIT_NAME, snapshotMeta.get().getName());
    }

    shardRequestTracker.sendShardRequest(replica.getNodeName(), params, shardHandler);
    log.debug("Sent backup request to core={} for backupName={}", coreName, backupName);
  }
  log.debug("Sent backup requests to all shard leaders for backupName={}", backupName);

  shardRequestTracker.processResponses(results, shardHandler, true, "Could not backup all shards");
}
 

@Override
protected void doDeleteByQuery(DeleteUpdateCommand cmd) throws IOException {
  zkCheck();

  // NONE: we are the first to receive this deleteByQuery
  //       - it must be forwarded to the leader of every shard
  // TO:   we are a leader receiving a forwarded deleteByQuery... we must:
  //       - block all updates (use VersionInfo)
  //       - flush *all* updates going to our replicas
  //       - forward the DBQ to our replicas and wait for the response
  //       - log + execute the local DBQ
  // FROM: we are a replica receiving a DBQ from our leader
  //       - log + execute the local DBQ
  DistribPhase phase = DistribPhase.parseParam(req.getParams().get(DISTRIB_UPDATE_PARAM));

  DocCollection coll = clusterState.getCollection(collection);

  if (DistribPhase.NONE == phase) {
    if (rollupReplicationTracker == null) {
      rollupReplicationTracker = new RollupRequestReplicationTracker();
    }
    boolean leaderForAnyShard = false;  // start off by assuming we are not a leader for any shard

    ModifiableSolrParams outParams = new ModifiableSolrParams(filterParams(req.getParams()));
    outParams.set(DISTRIB_UPDATE_PARAM, DistribPhase.TOLEADER.toString());
    outParams.set(DISTRIB_FROM, ZkCoreNodeProps.getCoreUrl(
        zkController.getBaseUrl(), req.getCore().getName()));

    SolrParams params = req.getParams();
    String route = params.get(ShardParams._ROUTE_);
    Collection<Slice> slices = coll.getRouter().getSearchSlices(route, params, coll);

    List<SolrCmdDistributor.Node> leaders =  new ArrayList<>(slices.size());
    for (Slice slice : slices) {
      String sliceName = slice.getName();
      Replica leader;
      try {
        leader = zkController.getZkStateReader().getLeaderRetry(collection, sliceName);
      } catch (InterruptedException e) {
        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Exception finding leader for shard " + sliceName, e);
      }

      // TODO: What if leaders changed in the meantime?
      // should we send out slice-at-a-time and if a node returns "hey, I'm not a leader" (or we get an error because it went down) then look up the new leader?

      // Am I the leader for this slice?
      ZkCoreNodeProps coreLeaderProps = new ZkCoreNodeProps(leader);
      String leaderCoreNodeName = leader.getName();
      String coreNodeName = cloudDesc.getCoreNodeName();
      isLeader = coreNodeName.equals(leaderCoreNodeName);

      if (isLeader) {
        // don't forward to ourself
        leaderForAnyShard = true;
      } else {
        leaders.add(new SolrCmdDistributor.ForwardNode(coreLeaderProps, zkController.getZkStateReader(), collection, sliceName, maxRetriesOnForward));
      }
    }

    outParams.remove("commit"); // this will be distributed from the local commit


    if (params.get(UpdateRequest.MIN_REPFACT) != null) {
      // TODO: Kept this for rolling upgrades. Remove in Solr 9
      outParams.add(UpdateRequest.MIN_REPFACT, req.getParams().get(UpdateRequest.MIN_REPFACT));
    }
    cmdDistrib.distribDelete(cmd, leaders, outParams, false, rollupReplicationTracker, null);

    if (!leaderForAnyShard) {
      return;
    }

    // change the phase to TOLEADER so we look up and forward to our own replicas (if any)
    phase = DistribPhase.TOLEADER;
  }
  List<SolrCmdDistributor.Node> replicas = null;

  if (DistribPhase.TOLEADER == phase) {
    // This core should be a leader
    isLeader = true;
    replicas = setupRequestForDBQ();
  } else if (DistribPhase.FROMLEADER == phase) {
    isLeader = false;
  }

  // check if client has requested minimum replication factor information. will set replicationTracker to null if
  // we aren't the leader or subShardLeader
  checkReplicationTracker(cmd);
  super.doDeleteByQuery(cmd, replicas, coll);
}
 

/**
 * Updates the mappings between the jetty's instances and the zookeeper cluster state.
 */
protected void updateMappingsFromZk(String collection) throws Exception {
  List<CloudJettyRunner> cloudJettys = new ArrayList<>();
  Map<String, List<CloudJettyRunner>> shardToJetty = new HashMap<>();
  Map<String, CloudJettyRunner> shardToLeaderJetty = new HashMap<>();

  CloudSolrClient cloudClient = this.createCloudClient(null);
  try {
    cloudClient.connect();
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection coll = clusterState.getCollection(collection);

    for (JettySolrRunner jetty : jettys) {
      int port = jetty.getLocalPort();
      if (port == -1) {
        throw new RuntimeException("Cannot find the port for jetty");
      }

      nextJetty:
      for (Slice shard : coll.getSlices()) {
        Set<Map.Entry<String, Replica>> entries = shard.getReplicasMap().entrySet();
        for (Map.Entry<String, Replica> entry : entries) {
          Replica replica = entry.getValue();
          if (replica.getStr(ZkStateReader.BASE_URL_PROP).contains(":" + port)) {
            if (!shardToJetty.containsKey(shard.getName())) {
              shardToJetty.put(shard.getName(), new ArrayList<CloudJettyRunner>());
            }
            boolean isLeader = shard.getLeader() == replica;
            CloudJettyRunner cjr = new CloudJettyRunner(jetty, replica, collection, shard.getName(), entry.getKey());
            shardToJetty.get(shard.getName()).add(cjr);
            if (isLeader) {
              shardToLeaderJetty.put(shard.getName(), cjr);
            }
            cloudJettys.add(cjr);
            break nextJetty;
          }
        }
      }
    }

    List<CloudJettyRunner> oldRunners = this.cloudJettys.putIfAbsent(collection, cloudJettys);
    if (oldRunners != null)  {
      // must close resources for the old entries
      for (CloudJettyRunner oldRunner : oldRunners) {
        IOUtils.closeQuietly(oldRunner.client);
      }
    }

    this.cloudJettys.put(collection, cloudJettys);
    this.shardToJetty.put(collection, shardToJetty);
    this.shardToLeaderJetty.put(collection, shardToLeaderJetty);
  } finally {
    cloudClient.close();
  }
}
 

protected static void ensureAllReplicasAreActive(String testCollectionName, int shards, int rf, int maxWaitSecs)
    throws Exception {
  long startMs = System.currentTimeMillis();

  ZkStateReader zkr = cloudSolrClient.getZkStateReader();
  zkr.updateClusterState(); // force the state to be fresh

  ClusterState cs = zkr.getClusterState();
  Collection<Slice> slices = cs.getActiveSlices(testCollectionName);
  assertTrue(slices.size() == shards);
  boolean allReplicasUp = false;
  long waitMs = 0L;
  long maxWaitMs = maxWaitSecs * 1000L;
  Replica leader = null;
  while (waitMs < maxWaitMs && !allReplicasUp) {
    // refresh state every 2 secs
    if (waitMs % 2000 == 0) {
      log.info("Updating ClusterState");
      cloudSolrClient.getZkStateReader().updateClusterState();
    }

    cs = cloudSolrClient.getZkStateReader().getClusterState();
    assertNotNull(cs);
    allReplicasUp = true; // assume true
    for (Slice shard : cs.getActiveSlices(testCollectionName)) {
      String shardId = shard.getName();
      assertNotNull("No Slice for " + shardId, shard);
      Collection<Replica> replicas = shard.getReplicas();
      assertTrue(replicas.size() == rf);
      leader = shard.getLeader();
      assertNotNull(leader);
      log.info("Found " + replicas.size() + " replicas and leader on " +
          leader.getNodeName() + " for " + shardId + " in " + testCollectionName);

      // ensure all replicas are "active"
      for (Replica replica : replicas) {
        String replicaState = replica.getStr(ZkStateReader.STATE_PROP);
        if (!"active".equals(replicaState)) {
          log.info("Replica " + replica.getName() + " for shard " + shardId + " is currently " + replicaState);
          allReplicasUp = false;
        }
      }
    }

    if (!allReplicasUp) {
      try {
        Thread.sleep(500L);
      } catch (Exception ignoreMe) {
      }
      waitMs += 500L;
    }
  } // end while

  if (!allReplicasUp)
    fail("Didn't see all replicas for " + testCollectionName +
        " come up within " + maxWaitMs + " ms! ClusterState: " + printClusterStateInfo(testCollectionName));

  long diffMs = (System.currentTimeMillis() - startMs);
  log.info("Took " + diffMs + " ms to see all replicas become active for " + testCollectionName);
}