org.apache.solr.common.cloud.Replica Java Examples

/** 
 * Helper method for picking a node that can safely be stoped
 * @see <a href="https://issues.apache.org/jira/browse/SOLR-13050">SOLR-13050</a>
 */
private JettySolrRunner pickNodeToStop() throws Exception {
  // first get the nodeName of the overser.
  // stopping the overseer is not something we want to hassle with in this test
  final String overseerNodeName = (String) cluster.getSolrClient().request
    (CollectionAdminRequest.getOverseerStatus()).get("leader");

  // now find a node that is *NOT* the overseer or the leader of a .system collection shard
  for (Replica r :  getCollectionState(CollectionAdminParams.SYSTEM_COLL).getReplicas()) {
    if ( ! (r.getBool("leader", false) || r.getNodeName().equals(overseerNodeName) ) ) {
      return cluster.getReplicaJetty(r);
    }
  }
  fail("Couldn't find non-leader, non-overseer, replica of .system collection to kill");
  return null;
}
 

@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
  waitForRecoveriesToFinish(false);
  assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
  index("id", "a!doc1");  // shard3
  index("id", "b!doc1");  // shard1
  index("id", "c!doc1");  // shard2
  index("id", "e!doc1");  // shard4
  commit();

  doQuery("a!doc1", "q", "*:*", ShardParams._ROUTE_, "a!"); // can go to any random node

  // query shard3 directly with _route_=a! so that we trigger the short circuited request path
  Replica shard3 = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getLeader("shard3");
  String nodeName = shard3.getNodeName();
  SolrClient shard3Client = getClient(nodeName);
  QueryResponse response = shard3Client.query(new SolrQuery("*:*").add(ShardParams._ROUTE_, "a!").add(ShardParams.SHARDS_INFO, "true"));

  assertEquals("Could not find doc", 1, response.getResults().getNumFound());
  NamedList<?> sinfo = (NamedList<?>) response.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("missing shard info for short circuited request", sinfo);
}
 

/**
 * Asserts that Update logs exist for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#NRT}, but not
 * for replicas of type {@link org.apache.solr.common.cloud.Replica.Type#PULL}
 */
private void assertUlogPresence(DocCollection collection) {
  for (Slice s:collection.getSlices()) {
    for (Replica r:s.getReplicas()) {
      SolrCore core = null;
      try {
        core = cluster.getReplicaJetty(r).getCoreContainer().getCore(r.getCoreName());
        assertNotNull(core);
        assertTrue("Update log should exist for replicas of type Append",
            new java.io.File(core.getUlogDir()).exists());
      } finally {
        core.close();
      }
    }
  }
}
 

protected long getIndexVersion(Replica replica) throws IOException {
  try (HttpSolrClient client = new HttpSolrClient.Builder(replica.getCoreUrl()).build()) {
    ModifiableSolrParams params = new ModifiableSolrParams();
    params.set("qt", "/replication");
    params.set(ReplicationHandler.COMMAND, ReplicationHandler.CMD_SHOW_COMMITS);
    try {
      QueryResponse response = client.query(params);
      @SuppressWarnings("unchecked")
      List<NamedList<Object>> commits = (List<NamedList<Object>>)response.getResponse().get(ReplicationHandler.CMD_SHOW_COMMITS);
      Collections.max(commits, (a,b)->((Long)a.get("indexVersion")).compareTo((Long)b.get("indexVersion")));
      return (long) Collections.max(commits, (a,b)->((Long)a.get("indexVersion")).compareTo((Long)b.get("indexVersion"))).get("indexVersion");
    } catch (SolrServerException e) {
      log.warn("Exception getting version from {}, will return an invalid version to retry.", replica.getName(), e);
      return -1;
    }
  }
}
 

private void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs) throws InterruptedException {
  TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
  while (!timeOut.hasTimedOut()) {
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Collection<Slice> slices = clusterState.getCollection(collection).getActiveSlices();
    if (slices.size() == numSlices) {
      boolean isMatch = true;
      for (Slice slice : slices) {
        int count = 0;
        for (Replica replica : slice.getReplicas()) {
          if (replica.getState() == Replica.State.ACTIVE && clusterState.liveNodesContain(replica.getNodeName())) {
            count++;
          }
        }
        if (count < numReplicas) {
          isMatch = false;
        }
      }
      if (isMatch) return;
    }
    Thread.sleep(200);
  }
  fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
 

/**
 * Tests if the 'shards.preference' parameter works with single-sharded collections.
 */
@Test
public void singleShardedPreferenceRules() throws Exception {
  String collectionName = "singleShardPreferenceTestColl";

  int liveNodes = cluster.getJettySolrRunners().size();

  // For testing replica.type, we want to have all replica types available for the collection
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, liveNodes/3, liveNodes/3, liveNodes/3)
      .setMaxShardsPerNode(liveNodes)
      .processAndWait(cluster.getSolrClient(), TIMEOUT);
  cluster.waitForActiveCollection(collectionName, 1, liveNodes);

  // Add some new documents
  new UpdateRequest()
      .add(id, "0", "a_t", "hello1")
      .add(id, "2", "a_t", "hello2")
      .add(id, "3", "a_t", "hello2")
      .commit(getRandomClient(), collectionName);

  // Run the actual test for 'queryReplicaType'
  queryReplicaType(getRandomClient(), Replica.Type.PULL, collectionName);
  queryReplicaType(getRandomClient(), Replica.Type.TLOG, collectionName);
  queryReplicaType(getRandomClient(), Replica.Type.NRT, collectionName);
}
 

public void testAddRemoveTlogReplica() throws Exception {
  DocCollection docCollection = createAndWaitForCollection(2, 0, 1, 0);
  assertEquals(2, docCollection.getSlices().size());

  addReplicaToShard("shard1", Replica.Type.TLOG);
  docCollection = assertNumberOfReplicas(0, 3, 0, true, false);
  addReplicaToShard("shard2", Replica.Type.TLOG);
  docCollection = assertNumberOfReplicas(0, 4, 0, true, false);

  waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));

  //Delete tlog replica from shard1
  CollectionAdminRequest.deleteReplica(
      collectionName,
      "shard1",
      docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.TLOG)).get(0).getName())
  .process(cluster.getSolrClient());
  assertNumberOfReplicas(0, 3, 0, true, true);
}
 

protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
  if (collection == null) {
    return null;
  }

  Set<String> liveNodes = clusterState.getLiveNodes();

  if (isPreferLeader) {
    List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
    SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
    if (core != null) return core;
  }

  List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
  return randomlyGetSolrCore(liveNodes, replicas);
}
 

public static CollectionStatePredicate expectedShardsAndActiveReplicas(int expectedShards, int expectedReplicas) {
  return (liveNodes, collectionState) -> {
    if (collectionState == null)
      return false;
    if (collectionState.getSlices().size() != expectedShards) {
      return false;
    }
    
    int activeReplicas = 0;
    for (Slice slice : collectionState) {
      for (Replica replica : slice) {
        if (replica.isActive(liveNodes)) {
          activeReplicas++;
        }
      }
    }
    if (activeReplicas == expectedReplicas) {
      return true;
    }

    return false;
  };
}
 

public void testAddRemovePullReplica() throws Exception {
  CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1, 0, 0)
    .setMaxShardsPerNode(100)
    .process(cluster.getSolrClient());
  waitForState("Expected collection to be created with 2 shards and 1 replica each", collectionName, clusterShape(2, 2));
  DocCollection docCollection = assertNumberOfReplicas(2, 0, 0, false, true);
  assertEquals(2, docCollection.getSlices().size());

  addReplicaToShard("shard1", Replica.Type.PULL);
  docCollection = assertNumberOfReplicas(2, 0, 1, true, false);
  addReplicaToShard("shard2", Replica.Type.PULL);
  docCollection = assertNumberOfReplicas(2, 0, 2, true, false);

  waitForState("Expecting collection to have 2 shards and 2 replica each", collectionName, clusterShape(2, 4));

  //Delete pull replica from shard1
  CollectionAdminRequest.deleteReplica(
      collectionName,
      "shard1",
      docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).get(0).getName())
  .process(cluster.getSolrClient());
  assertNumberOfReplicas(2, 0, 1, true, true);
}
 

@Test
public void deleteReplicaByCount() throws Exception {

  final String collectionName = "deleteByCount";

  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 3).process(cluster.getSolrClient());
  waitForState("Expected a single shard with three replicas", collectionName, clusterShape(1, 3));

  CollectionAdminRequest.deleteReplicasFromShard(collectionName, "shard1", 2).process(cluster.getSolrClient());
  waitForState("Expected a single shard with a single replica", collectionName, clusterShape(1, 1));

  SolrException e = expectThrows(SolrException.class,
      "Can't delete the last replica by count",
      () -> CollectionAdminRequest.deleteReplicasFromShard(collectionName, "shard1", 1).process(cluster.getSolrClient())
  );
  assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, e.code());
  assertTrue(e.getMessage().contains("There is only one replica available"));
  DocCollection docCollection = getCollectionState(collectionName);
  // We know that since leaders are preserved, PULL replicas should not be left alone in the shard
  assertEquals(0, docCollection.getSlice("shard1").getReplicas(EnumSet.of(Replica.Type.PULL)).size());
}
 

private CollectionStatePredicate clusterStateReflectsActiveAndDownReplicas() {
  return (liveNodes, collectionState) -> {
    for (Replica r:collectionState.getReplicas()) {
      if (r.getState() != Replica.State.DOWN && r.getState() != Replica.State.ACTIVE) {
        return false;
      }
      if (r.getState() == Replica.State.DOWN && liveNodes.contains(r.getNodeName())) {
        return false;
      }
      if (r.getState() == Replica.State.ACTIVE && !liveNodes.contains(r.getNodeName())) {
        return false;
      }
    }
    return true;
  };
}
 

public void projectAddReplica(Cell cell, ReplicaInfo ri, Consumer<Row.OperationInfo> opCollector, boolean strictMode) {
  if (strictMode) {
    // we do not want to add a replica of the 'withCollection' in strict mode
    return;
  }

  @SuppressWarnings({"unchecked"})
  Map<String, String> withCollectionMap = (Map<String, String>) cell.val;
  if (withCollectionMap == null || withCollectionMap.isEmpty()) return;

  Set<String> uniqueColls = new HashSet<>();
  Row row = cell.row;
  row.forEachReplica(replicaInfo -> uniqueColls.add(replicaInfo.getCollection()));

  for (Map.Entry<String, String> e : withCollectionMap.entrySet()) {
    if (uniqueColls.contains(e.getKey()) && !uniqueColls.contains(e.getValue())) {
      String withCollection = e.getValue();

      opCollector.accept(new Row.OperationInfo(withCollection, "shard1", row.node, cell.name, true, Replica.Type.NRT));
    }
  }
}
 

private void collectStartTimes(String collectionName, Map<String,Long> urlToTime)
    throws SolrServerException, IOException {

  DocCollection collectionState = getCollectionState(collectionName);
  if (collectionState != null) {
    for (Slice shard : collectionState) {
      for (Replica replica : shard) {
        ZkCoreNodeProps coreProps = new ZkCoreNodeProps(replica);
        CoreStatus coreStatus;
        try (HttpSolrClient server = getHttpSolrClient(coreProps.getBaseUrl())) {
          coreStatus = CoreAdminRequest.getCoreStatus(coreProps.getCoreName(), false, server);
        }
        long before = coreStatus.getCoreStartTime().getTime();
        urlToTime.put(coreProps.getCoreUrl(), before);
      }
    }
  } else {
    throw new IllegalArgumentException("Could not find collection " + collectionName);
  }
}
 

private CollectionStatePredicate activeReplicaCount(int numWriter, int numActive, int numPassive) {
  return (liveNodes, collectionState) -> {
    int writersFound = 0, activesFound = 0, passivesFound = 0;
    if (collectionState == null)
      return false;
    for (Slice slice : collectionState) {
      for (Replica replica : slice) {
        if (replica.isActive(liveNodes))
          switch (replica.getType()) {
            case TLOG:
              activesFound++;
              break;
            case PULL:
              passivesFound++;
              break;
            case NRT:
              writersFound++;
              break;
            default:
              throw new AssertionError("Unexpected replica type");
          }
      }
    }
    return numWriter == writersFound && numActive == activesFound && numPassive == passivesFound;
  };
}
 

/**
 * Walks the tree of collection status to verify that any replicas not reporting a "down" status is
 * on a live node, if any replicas reporting their status as "active" but the node is not live is
 * marked as "down"; used by CLUSTERSTATUS.
 * @param liveNodes List of currently live node names.
 * @param collectionProps Map of collection status information pulled directly from ZooKeeper.
 */

@SuppressWarnings("unchecked")
protected void crossCheckReplicaStateWithLiveNodes(List<String> liveNodes, NamedList<Object> collectionProps) {
  for (Map.Entry<String, Object> next : collectionProps) {
    Map<String, Object> collMap = (Map<String, Object>) next.getValue();
    Map<String, Object> shards = (Map<String, Object>) collMap.get("shards");
    for (Object nextShard : shards.values()) {
      Map<String, Object> shardMap = (Map<String, Object>) nextShard;
      Map<String, Object> replicas = (Map<String, Object>) shardMap.get("replicas");
      for (Object nextReplica : replicas.values()) {
        Map<String, Object> replicaMap = (Map<String, Object>) nextReplica;
        if (Replica.State.getState((String) replicaMap.get(ZkStateReader.STATE_PROP)) != Replica.State.DOWN) {
          // not down, so verify the node is live
          String node_name = (String) replicaMap.get(ZkStateReader.NODE_NAME_PROP);
          if (!liveNodes.contains(node_name)) {
            // node is not live, so this replica is actually down
            replicaMap.put(ZkStateReader.STATE_PROP, Replica.State.DOWN.toString());
          }
        }
      }
    }
  }
}
 

protected Replica getShardLeader(String testCollectionName, String shardId, int timeoutSecs) throws Exception {
  Replica leader = null;
  long timeout = System.nanoTime() + TimeUnit.NANOSECONDS.convert(timeoutSecs, TimeUnit.SECONDS);
  while (System.nanoTime() < timeout) {
    Replica tmp = null;
    try {
      tmp = cloudClient.getZkStateReader().getLeaderRetry(testCollectionName, shardId);
    } catch (Exception exc) {}
    if (tmp != null && "active".equals(tmp.getStr(ZkStateReader.STATE_PROP))) {
      leader = tmp;
      break;
    }
    Thread.sleep(1000);
  }
  assertNotNull("Could not find active leader for " + shardId + " of " +
      testCollectionName + " after "+timeoutSecs+" secs; clusterState: " +
      printClusterStateInfo(testCollectionName), leader);

  return leader;
}
 

private void nodeAddedTriggerWithAddReplicaPreferredOpReplicaType(String collectionNamePrefix, int numShards, int numCollections) throws Exception {
  String setTriggerCommand = "{" +
      "'set-trigger' : {" +
      "'name' : 'node_added_trigger'," +
      "'event' : 'nodeAdded'," +
      "'waitFor' : '1s'," +
      "'enabled' : true," +
      "'" + AutoScalingParams.PREFERRED_OP + "':'addreplica'," +
      "'" + AutoScalingParams.REPLICA_TYPE + "':'" + Replica.Type.PULL + "'," +
      "'actions' : [{'name':'compute_plan', 'class' : 'solr.ComputePlanAction'}," +
      "{'name':'test','class':'" + AssertingTriggerAction.class.getName() + "'}]" +
      "}}";

  String setClusterPolicyCommand = "{" +
      " 'set-cluster-policy': [" +
      "      {'cores':'<" + (1 + numCollections * numShards) + "', 'node':'#ANY'}," +
      "      {'replica':'<2', 'shard': '#EACH', 'node': '#ANY'}," +
      "      {'nodeRole':'overseer', 'replica':0}" +
      "    ]" +
      "}";

  nodeAddedTriggerWithAddReplicaPreferredOp(collectionNamePrefix, numShards, numCollections, setTriggerCommand, setClusterPolicyCommand, 0, 1, 0);
}
 

public DirectUpdateHandler2(SolrCore core) {
  super(core);
 
  solrCoreState = core.getSolrCoreState();
  
  UpdateHandlerInfo updateHandlerInfo = core.getSolrConfig()
      .getUpdateHandlerInfo();
  int docsUpperBound = updateHandlerInfo.autoCommmitMaxDocs;
  int timeUpperBound = updateHandlerInfo.autoCommmitMaxTime;
  long fileSizeUpperBound = updateHandlerInfo.autoCommitMaxSizeBytes;
  commitTracker = new CommitTracker("Hard", core, docsUpperBound, timeUpperBound, fileSizeUpperBound, updateHandlerInfo.openSearcher, false);
  
  int softCommitDocsUpperBound = updateHandlerInfo.autoSoftCommmitMaxDocs;
  int softCommitTimeUpperBound = updateHandlerInfo.autoSoftCommmitMaxTime;
  softCommitTracker = new CommitTracker("Soft", core, softCommitDocsUpperBound, softCommitTimeUpperBound, NO_FILE_SIZE_UPPER_BOUND_PLACEHOLDER, true, true);
  
  commitWithinSoftCommit = updateHandlerInfo.commitWithinSoftCommit;
  indexWriterCloseWaitsForMerges = updateHandlerInfo.indexWriterCloseWaitsForMerges;

  ZkController zkController = core.getCoreContainer().getZkController();
  if (zkController != null && core.getCoreDescriptor().getCloudDescriptor().getReplicaType() == Replica.Type.TLOG) {
    commitWithinSoftCommit = false;
    commitTracker.setOpenSearcher(true);
  }

}
 

/**
 * Send request to all replicas of a collection
 * @return List of replicas which is not live for receiving the request
 */
List<Replica> collectionCmd(ZkNodeProps message, ModifiableSolrParams params,
                   NamedList<Object> results, Replica.State stateMatcher, String asyncId, Set<String> okayExceptions) {
  log.info("Executing Collection Cmd={}, asyncId={}", params, asyncId);
  String collectionName = message.getStr(NAME);
  @SuppressWarnings("deprecation")
  ShardHandler shardHandler = shardHandlerFactory.getShardHandler(overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection coll = clusterState.getCollection(collectionName);
  List<Replica> notLivesReplicas = new ArrayList<>();
  final ShardRequestTracker shardRequestTracker = new ShardRequestTracker(asyncId);
  for (Slice slice : coll.getSlices()) {
    notLivesReplicas.addAll(shardRequestTracker.sliceCmd(clusterState, params, stateMatcher, slice, shardHandler));
  }

  shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions);
  return notLivesReplicas;
}
 

public static String getUrlFromZk(ClusterState clusterState, String collection) {
  Map<String,Slice> slices = clusterState.getCollection(collection).getSlicesMap();

  if (slices == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
  }

  for (Map.Entry<String,Slice> entry : slices.entrySet()) {
    Slice slice = entry.getValue();
    Map<String,Replica> shards = slice.getReplicasMap();
    Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
    for (Map.Entry<String,Replica> shardEntry : shardEntries) {
      final ZkNodeProps node = shardEntry.getValue();
      if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
        return ZkCoreNodeProps.getCoreUrl(node.getStr(ZkStateReader.BASE_URL_PROP), collection); //new ZkCoreNodeProps(node).getCoreUrl();
      }
    }
  }

  throw new RuntimeException("Could not find a live node for collection:" + collection);
}
 

private void waitForNumDocsInAllReplicas(int numDocs, Collection<Replica> replicas, String query) throws IOException, SolrServerException, InterruptedException {
  TimeOut t = new TimeOut(REPLICATION_TIMEOUT_SECS, TimeUnit.SECONDS, TimeSource.NANO_TIME);
  for (Replica r:replicas) {
    try (HttpSolrClient replicaClient = getHttpSolrClient(r.getCoreUrl())) {
      while (true) {
        try {
          assertEquals("Replica " + r.getName() + " not up to date after " + REPLICATION_TIMEOUT_SECS + " seconds",
              numDocs, replicaClient.query(new SolrQuery(query)).getResults().getNumFound());
          break;
        } catch (AssertionError e) {
          if (t.hasTimedOut()) {
            throw e;
          } else {
            Thread.sleep(100);
          }
        }
      }
    }
  }
}
 

private CollectionStatePredicate activeReplicaCount(int numNrtReplicas, int numTlogReplicas, int numPullReplicas) {
  return (liveNodes, collectionState) -> {
    int nrtFound = 0, tlogFound = 0, pullFound = 0;
    if (collectionState == null)
      return false;
    for (Slice slice : collectionState) {
      for (Replica replica : slice) {
        if (replica.isActive(liveNodes))
          switch (replica.getType()) {
            case TLOG:
              tlogFound++;
              break;
            case PULL:
              pullFound++;
              break;
            case NRT:
              nrtFound++;
              break;
            default:
              throw new AssertionError("Unexpected replica type");
          }
      }
    }
    return numNrtReplicas == nrtFound && numTlogReplicas == tlogFound && numPullReplicas == pullFound;
  };
}
 

@Test
public void deleteReplicaAndVerifyDirectoryCleanup() throws Exception {

  final String collectionName = "deletereplica_test";
  CollectionAdminRequest.createCollection(collectionName, "conf", 1, 2).process(cluster.getSolrClient());

  Replica leader = cluster.getSolrClient().getZkStateReader().getLeaderRetry(collectionName, "shard1");

  //Confirm that the instance and data directory exist
  CoreStatus coreStatus = getCoreStatus(leader);
  assertTrue("Instance directory doesn't exist", Files.exists(Paths.get(coreStatus.getInstanceDirectory())));
  assertTrue("DataDirectory doesn't exist", Files.exists(Paths.get(coreStatus.getDataDirectory())));

  CollectionAdminRequest.deleteReplica(collectionName, "shard1",leader.getName())
      .process(cluster.getSolrClient());

  Replica newLeader = cluster.getSolrClient().getZkStateReader().getLeaderRetry(collectionName, "shard1");

  assertFalse(leader.equals(newLeader));

  //Confirm that the instance and data directory were deleted by default
  assertFalse("Instance directory still exists", Files.exists(Paths.get(coreStatus.getInstanceDirectory())));
  assertFalse("DataDirectory still exists", Files.exists(Paths.get(coreStatus.getDataDirectory())));
}
 

private Map<String, String> getProps(CloudSolrClient client, String collectionName, String replicaName, String... props)
    throws KeeperException, InterruptedException {

  client.getZkStateReader().forceUpdateCollection(collectionName);
  ClusterState clusterState = client.getZkStateReader().getClusterState();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
  if (docCollection == null || docCollection.getReplica(replicaName) == null) {
    fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
  }
  Replica replica = docCollection.getReplica(replicaName);
  Map<String, String> propMap = new HashMap<>();
  for (String prop : props) {
    propMap.put(prop, replica.getProperty(prop));
  }
  return propMap;
}
 

/**
 * Method gets the SolrClient for the leader replica. This is needed for a workaround for SOLR-8733.
 */
public SolrClient getClientForLeader() throws KeeperException, InterruptedException {
  ZkStateReader zkStateReader = cloudClient.getZkStateReader();
  cloudClient.getZkStateReader().forceUpdateCollection(DEFAULT_COLLECTION);
  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  Replica leader = null;
  Slice shard1 = clusterState.getCollection(DEFAULT_COLLECTION).getSlice(SHARD1);
  leader = shard1.getLeader();

  for (int i = 0; i < clients.size(); i++) {
    String leaderBaseUrl = zkStateReader.getBaseUrlForNodeName(leader.getNodeName());
    if (((HttpSolrClient) clients.get(i)).getBaseURL().startsWith(leaderBaseUrl))
      return clients.get(i);
  }

  return null;
}
 

private void checkLeaderStatus() throws InterruptedException, KeeperException {
  for (int idx = 0; pendingOps.size() > 0 && idx < 600; ++idx) {
    ClusterState clusterState = coreContainer.getZkController().getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();
    DocCollection dc = clusterState.getCollection(collectionName);
    for (Slice slice : dc.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (replica.isActive(liveNodes) && replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false)) {
          if (replica.getBool(LEADER_PROP, false)) {
            if (pendingOps.containsKey(slice.getName())) {
              // Record for return that the leader changed successfully
              pendingOps.remove(slice.getName());
              addToSuccesses(slice, replica);
              break;
            }
          }
        }
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
    coreContainer.getZkController().getZkStateReader().forciblyRefreshAllClusterStateSlow();
  }
  addAnyFailures();
}
 

/**
 * On startup, the node already published all of its replicas as DOWN,
 * we can skip publish the replica as down
 * @return Should publish the replica as down on startup
 */
private boolean isPublishAsDownOnStartup(CloudDescriptor cloudDesc) {
    Replica replica = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName())
        .getSlice(cloudDesc.getShardId())
        .getReplica(cloudDesc.getCoreNodeName());
    return !replica.getNodeName().equals(getNodeName());
}
 

/**
 * Get a (reproducibly) random replica from a {@link Slice} matching a predicate
 */
protected static Replica getRandomReplica(Slice slice, Predicate<Replica> matchPredicate) {
  List<Replica> replicas = new ArrayList<>(slice.getReplicas());
  if (replicas.size() == 0)
    fail("Couldn't get random replica from shard as it has no replicas!\n" + slice.toString());
  Collections.shuffle(replicas, random());
  for (Replica replica : replicas) {
    if (matchPredicate.test(replica))
      return replica;
  }
  fail("Couldn't get random replica that matched conditions\n" + slice.toString());
  return null;  // just to keep the compiler happy - fail will always throw an Exception
}
 

public Replica.State getState() {
  if (variables.get(ZkStateReader.STATE_PROP) != null) {
    return Replica.State.getState((String) variables.get(ZkStateReader.STATE_PROP));
  } else {
    // default to ACTIVE
    variables.put(ZkStateReader.STATE_PROP, Replica.State.ACTIVE.toString());
    return Replica.State.ACTIVE;
  }
}