Java Code Examples for org.apache.solr.common.cloud.ClusterState#liveNodesContain()

public static String getUrlFromZk(ClusterState clusterState, String collection) {
  Map<String,Slice> slices = clusterState.getCollection(collection).getSlicesMap();

  if (slices == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Could not find collection:" + collection);
  }

  for (Map.Entry<String,Slice> entry : slices.entrySet()) {
    Slice slice = entry.getValue();
    Map<String,Replica> shards = slice.getReplicasMap();
    Set<Map.Entry<String,Replica>> shardEntries = shards.entrySet();
    for (Map.Entry<String,Replica> shardEntry : shardEntries) {
      final ZkNodeProps node = shardEntry.getValue();
      if (clusterState.liveNodesContain(node.getStr(ZkStateReader.NODE_NAME_PROP))) {
        return ZkCoreNodeProps.getCoreUrl(node.getStr(ZkStateReader.BASE_URL_PROP), collection); //new ZkCoreNodeProps(node).getCoreUrl();
      }
    }
  }

  throw new RuntimeException("Could not find a live node for collection:" + collection);
}
 

/**
 * Send request to all replicas of a slice
 * @return List of replicas which is not live for receiving the request
 */
public List<Replica> sliceCmd(ClusterState clusterState, ModifiableSolrParams params, Replica.State stateMatcher,
              Slice slice, ShardHandler shardHandler) {
  List<Replica> notLiveReplicas = new ArrayList<>();
  for (Replica replica : slice.getReplicas()) {
    if ((stateMatcher == null || Replica.State.getState(replica.getStr(ZkStateReader.STATE_PROP)) == stateMatcher)) {
      if (clusterState.liveNodesContain(replica.getStr(ZkStateReader.NODE_NAME_PROP))) {
        // For thread safety, only simple clone the ModifiableSolrParams
        ModifiableSolrParams cloneParams = new ModifiableSolrParams();
        cloneParams.add(params);
        cloneParams.set(CoreAdminParams.CORE, replica.getStr(ZkStateReader.CORE_NAME_PROP));

        sendShardRequest(replica.getStr(ZkStateReader.NODE_NAME_PROP), cloneParams, shardHandler);
      } else {
        notLiveReplicas.add(replica);
      }
    }
  }
  return notLiveReplicas;
}
 

private void waitTillRecovered() throws Exception {
  for (int i = 0; i < 30; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    zkStateReader.forceUpdateCollection("collection1");
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see recovered node");
}
 

private void waitTillAllNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) { 
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;
    for (Replica replica : replicas) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

private void assertSliceAndReplicaCount(String collection, int numSlices, int numReplicas, int timeOutInMs) throws InterruptedException {
  TimeOut timeOut = new TimeOut(timeOutInMs, TimeUnit.MILLISECONDS, TimeSource.NANO_TIME);
  while (!timeOut.hasTimedOut()) {
    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    Collection<Slice> slices = clusterState.getCollection(collection).getActiveSlices();
    if (slices.size() == numSlices) {
      boolean isMatch = true;
      for (Slice slice : slices) {
        int count = 0;
        for (Replica replica : slice.getReplicas()) {
          if (replica.getState() == Replica.State.ACTIVE && clusterState.liveNodesContain(replica.getNodeName())) {
            count++;
          }
        }
        if (count < numReplicas) {
          isMatch = false;
        }
      }
      if (isMatch) return;
    }
    Thread.sleep(200);
  }
  fail("Expected numSlices=" + numSlices + " numReplicas=" + numReplicas + " but found " + cloudClient.getZkStateReader().getClusterState().getCollection(collection) + " with /live_nodes: " + cloudClient.getZkStateReader().getClusterState().getLiveNodes());
}
 

/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    final ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.forceUpdateCollection(collection);
            final ClusterState clusterState = zkStateReader.getClusterState();
            final Map<String, Slice> slices = clusterState.getCollection(collection).getSlicesMap();
            Preconditions.checkNotNull(slices, "Could not find collection:" + collection);

            // change paths for Replica.State per Solr refactoring
            // remove SYNC state per: http://tinyurl.com/pag6rwt
            for (final Map.Entry<String, Slice> entry : slices.entrySet()) {
                final Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (final Map.Entry<String, Replica> shard : shards.entrySet()) {
                    final String state = shard.getValue().getStr(ZkStateReader.STATE_PROP).toUpperCase();
                    if ((Replica.State.RECOVERING.name().equals(state) || Replica.State.DOWN.name().equals(state))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }


            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 

public static CreateReplica assignReplicaDetails(SolrCloudManager cloudManager, ClusterState clusterState,
                                               ZkNodeProps message, ReplicaPosition replicaPosition) {
  boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);

  String collection = message.getStr(COLLECTION_PROP);
  String node = replicaPosition.node;
  String shard = message.getStr(SHARD_ID_PROP);
  String coreName = message.getStr(CoreAdminParams.NAME);
  String coreNodeName = message.getStr(CoreAdminParams.CORE_NODE_NAME);
  Replica.Type replicaType = replicaPosition.type;

  if (StringUtils.isBlank(coreName)) {
    coreName = message.getStr(CoreAdminParams.PROPERTY_PREFIX + CoreAdminParams.NAME);
  }

  log.info("Node Identified {} for creating new replica of shard {} for collection {}", node, shard, collection);
  if (!clusterState.liveNodesContain(node)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Node: " + node + " is not live");
  }
  DocCollection coll = clusterState.getCollection(collection);
  if (coreName == null) {
    coreName = Assign.buildSolrCoreName(cloudManager.getDistribStateManager(), coll, shard, replicaType);
  } else if (!skipCreateReplicaInClusterState) {
    //Validate that the core name is unique in that collection
    for (Slice slice : coll.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        String replicaCoreName = replica.getStr(CORE_NAME_PROP);
        if (coreName.equals(replicaCoreName)) {
          throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Another replica with the same core name already exists" +
              " for this collection");
        }
      }
    }
  }
  log.info("Returning CreateReplica command.");
  return new CreateReplica(collection, shard, node, replicaType, coreName, coreNodeName);
}
 

private void waitTillNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;

    Collection<String> nodesDownNames = nodesDown.stream()
        .map(n -> n.coreNodeName)
        .collect(Collectors.toList());
    
    Collection<Replica> replicasToCheck = null;
    replicasToCheck = replicas.stream()
        .filter(r -> !nodesDownNames.contains(r.getName()))
        .collect(Collectors.toList());

    for (Replica replica : replicasToCheck) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

private void waitTillNodesActive() throws Exception {
  for (int i = 0; i < 60; i++) {
    Thread.sleep(3000);
    ZkStateReader zkStateReader = cloudClient.getZkStateReader();
    ClusterState clusterState = zkStateReader.getClusterState();
    DocCollection collection1 = clusterState.getCollection("collection1");
    Slice slice = collection1.getSlice("shard1");
    Collection<Replica> replicas = slice.getReplicas();
    boolean allActive = true;

    Collection<String> nodesDownNames =
        nodesDown.stream()
            .map(n -> n.coreNodeName)
            .collect(Collectors.toList());

    Collection<Replica> replicasToCheck =
        replicas.stream()
            .filter(r -> !nodesDownNames.contains(r.getName()))
            .collect(Collectors.toList());

    for (Replica replica : replicasToCheck) {
      if (!clusterState.liveNodesContain(replica.getNodeName()) || replica.getState() != Replica.State.ACTIVE) {
        allActive = false;
        break;
      }
    }
    if (allActive) {
      return;
    }
  }
  printLayout();
  fail("timeout waiting to see all nodes active");
}
 

/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState(true);
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);

           // change paths for Replica.State per Solr refactoring
           // remove SYNC state per: http://tinyurl.com/pag6rwt
           for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING) || state.equals(Replica.State.DOWN))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }


            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 

/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState();
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);

            for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING.toString())
                            || state.equals(Replica.State.DOWN.toString()))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }
            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 

private boolean waitingForReplicasNotLive(ZkStateReader zkStateReader, int timeoutInMs, List<JettySolrRunner> jetties) {
  Set<String> nodeNames = jetties.stream()
      .filter(jetty -> jetty.getCoreContainer() != null)
      .map(JettySolrRunner::getNodeName)
      .collect(Collectors.toSet());
  long timeout = System.nanoTime()
      + TimeUnit.NANOSECONDS.convert(timeoutInMs, TimeUnit.MILLISECONDS);
  boolean success = false;
  while (!success && System.nanoTime() < timeout) {
    success = true;
    ClusterState clusterState = zkStateReader.getClusterState();
    if (clusterState != null) {
      Map<String, DocCollection> collections = clusterState.getCollectionsMap();
      for (Map.Entry<String, DocCollection> entry : collections.entrySet()) {
        DocCollection docCollection = entry.getValue();
        Collection<Slice> slices = docCollection.getSlices();
        for (Slice slice : slices) {
          // only look at active shards
          if (slice.getState() == Slice.State.ACTIVE) {
            Collection<Replica> replicas = slice.getReplicas();
            for (Replica replica : replicas) {
              if (nodeNames.contains(replica.getNodeName())) {
                boolean live = clusterState.liveNodesContain(replica
                    .getNodeName());
                if (live) {
                  success = false;
                }
              }
            }
          }
        }
      }
      if (!success) {
        try {
          Thread.sleep(500);
        } catch (InterruptedException e) {
          Thread.currentThread().interrupt();
          throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Interrupted");
        }
      }
    }
  }

  return success;
}
 

protected static void waitForRecoveriesToFinish(String collection,
                                                CloudSolrServer solrServer,
                                                boolean verbose,
                                                boolean failOnTimeout,
                                                int timeoutSeconds) throws Exception {
  LOG.info("Entering solr wait with timeout " + timeoutSeconds);
  ZkStateReader zkStateReader = solrServer.getZkStateReader();
  try {
    boolean cont = true;
    int cnt = 0;

    while (cont) {
      if (verbose) {
        LOG.debug("-");
      }
      boolean sawLiveRecovering = false;
      zkStateReader.updateClusterState(true);
      ClusterState clusterState = zkStateReader.getClusterState();
      Map<String, Slice> slices = clusterState.getSlicesMap(collection);
      assertNotNull("Could not find collection:" + collection, slices);
      for (Map.Entry<String, Slice> entry : slices.entrySet()) {
        Map<String, Replica> shards = entry.getValue().getReplicasMap();
        for (Map.Entry<String, Replica> shard : shards.entrySet()) {
          if (verbose) {
            LOG.debug("rstate:"
              + shard.getValue().getStr(ZkStateReader.STATE_PROP) + " live:"
              + clusterState.liveNodesContain(shard.getValue().getNodeName()));
          }
          String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
          if ((state.equals(ZkStateReader.RECOVERING)
              || state.equals(ZkStateReader.SYNC) || state
              .equals(ZkStateReader.DOWN))
              && clusterState.liveNodesContain(shard.getValue().getStr(
              ZkStateReader.NODE_NAME_PROP))) {
            sawLiveRecovering = true;
          }
        }
      }
      if (!sawLiveRecovering || cnt == timeoutSeconds) {
        if (!sawLiveRecovering) {
          if (verbose) {
            LOG.debug("no one is recovering");
          }
        } else {
          if (verbose) {
            LOG.debug("Gave up waiting for recovery to finish..");
          }
          if (failOnTimeout) {
            fail("There are still nodes recovering - waited for "
                + timeoutSeconds + " seconds");
            // won't get here
            return;
          }
        }
        cont = false;
      } else {
        Thread.sleep(1000);
      }
      cnt++;
    }
  } finally {
    LOG.info("Exiting solr wait");
  }
}