org.apache.solr.common.cloud.ClusterState Java Examples

The following examples show how to use org.apache.solr.common.cloud.ClusterState. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: RebalanceLeaders.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void checkLeaderStatus() throws InterruptedException, KeeperException {
  for (int idx = 0; pendingOps.size() > 0 && idx < 600; ++idx) {
    ClusterState clusterState = coreContainer.getZkController().getClusterState();
    Set<String> liveNodes = clusterState.getLiveNodes();
    DocCollection dc = clusterState.getCollection(collectionName);
    for (Slice slice : dc.getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (replica.isActive(liveNodes) && replica.getBool(SliceMutator.PREFERRED_LEADER_PROP, false)) {
          if (replica.getBool(LEADER_PROP, false)) {
            if (pendingOps.containsKey(slice.getName())) {
              // Record for return that the leader changed successfully
              pendingOps.remove(slice.getName());
              addToSuccesses(slice, replica);
              break;
            }
          }
        }
      }
    }
    TimeUnit.MILLISECONDS.sleep(100);
    coreContainer.getZkController().getZkStateReader().forciblyRefreshAllClusterStateSlow();
  }
  addAnyFailures();
}
 
Example #2
Source File: TestUpdateOperations.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
private void checkUpdateDistribPhase(CloudSolrServer server, String collectionName,
    String userName, DistribPhase distribPhase) throws Exception {
  String path = "/" + collectionName + "/update?commit=true";
  String updateDistribParam="";
  if (distribPhase != null) {
    updateDistribParam = distribPhase.toString();
    path += "&update.distrib="+updateDistribParam;
  }
  String docId = "testUpdateDistribDoc"+updateDistribParam;
  String body = "<add><doc><field name=\"id\">"+docId+"</field></doc></add>";

  String node = null;
  ClusterState clusterState = server.getZkStateReader().getClusterState();
  for (Slice slice : clusterState.getActiveSlices(collectionName)) {
    if(slice.getRange().includes(docId.hashCode())) {
      node = slice.getLeader().getNodeName().replace("_solr", "/solr");
    }
  }
  assertNotNull("Expected to find leader node for document", node);

  String ret = makeHttpRequest(server, node, "POST", path, body.getBytes("UTF-8"), "text/xml");
  assertTrue("Expected sentry exception",
    ret.contains("SentrySolrAuthorizationException: " +
      "User " + userName + " does not have privileges for " + collectionName));
}
 
Example #3
Source File: ShardLeaderElectionContext.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Do other replicas with higher term participated in the election
 *
 * @return true if other replicas with higher term participated in the election, false if otherwise
 */
private boolean replicasWithHigherTermParticipated(ZkShardTerms zkShardTerms, String coreNodeName) {
  ClusterState clusterState = zkController.getClusterState();
  DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  Slice slices = (docCollection == null) ? null : docCollection.getSlice(shardId);
  if (slices == null) return false;

  long replicaTerm = zkShardTerms.getTerm(coreNodeName);
  boolean isRecovering = zkShardTerms.isRecovering(coreNodeName);

  for (Replica replica : slices.getReplicas()) {
    if (replica.getName().equals(coreNodeName)) continue;

    if (clusterState.getLiveNodes().contains(replica.getNodeName())) {
      long otherTerm = zkShardTerms.getTerm(replica.getName());
      boolean isOtherReplicaRecovering = zkShardTerms.isRecovering(replica.getName());

      if (isRecovering && !isOtherReplicaRecovering) return true;
      if (otherTerm > replicaTerm) return true;
    }
  }
  return false;
}
 
Example #4
Source File: SolrClientNodeStateProvider.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected void readReplicaDetails() throws IOException {
  ClusterStateProvider clusterStateProvider = getClusterStateProvider();
  ClusterState clusterState = clusterStateProvider.getClusterState();
  if (clusterState == null) { // zkStateReader still initializing
    return;
  }
  Map<String, ClusterState.CollectionRef> all = clusterStateProvider.getClusterState().getCollectionStates();
  all.forEach((collName, ref) -> {
    DocCollection coll = ref.get();
    if (coll == null) return;
    if (coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION) != null) {
      withCollectionsMap.put(coll.getName(), (String) coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION));
    }
    coll.forEachReplica((shard, replica) -> {
      Map<String, Map<String, List<ReplicaInfo>>> nodeData = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>());
      Map<String, List<ReplicaInfo>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>());
      List<ReplicaInfo> replicas = collData.computeIfAbsent(shard, k -> new ArrayList<>());
      replicas.add(new ReplicaInfo(collName, shard, replica, new HashMap<>(replica.getProperties())));
    });
  });
}
 
Example #5
Source File: SnapshotClusterStateProvider.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked"})
public SnapshotClusterStateProvider(Map<String, Object> snapshot) {
  Objects.requireNonNull(snapshot);
  liveNodes = Set.copyOf((Collection<String>)snapshot.getOrDefault("liveNodes", Collections.emptySet()));
  clusterProperties = (Map<String, Object>)snapshot.getOrDefault("clusterProperties", Collections.emptyMap());
  Map<String, Object> stateMap = new HashMap<>((Map<String, Object>)snapshot.getOrDefault("clusterState", Collections.emptyMap()));
  Map<String, DocCollection> collectionStates = new HashMap<>();
  // back-compat with format = 1
  Integer stateVersion = Integer.valueOf(String.valueOf(stateMap.getOrDefault("version", 0)));
  stateMap.remove("version");
  stateMap.forEach((name, state) -> {
    Map<String, Object> mutableState = (Map<String, Object>)state;
    Map<String, Object> collMap = (Map<String, Object>) mutableState.get(name);
    if (collMap == null) {
      // snapshot in format 1
      collMap = mutableState;
      mutableState = Collections.singletonMap(name, state);
    }
    int version = Integer.parseInt(String.valueOf(collMap.getOrDefault("zNodeVersion", stateVersion)));
    collMap.remove("zNodeVersion");
    byte[] data = Utils.toJSON(mutableState);
    ClusterState collState = ClusterState.createFromJson(version, data, Collections.emptySet());
    collectionStates.put(name, collState.getCollection(name));
  });
  clusterState = new ClusterState(liveNodes, collectionStates);
}
 
Example #6
Source File: TestSolrCloudClusterSupport.java    From storm-solr with Apache License 2.0 6 votes vote down vote up
protected static String printClusterStateInfo(String collection) throws Exception {
  cloudSolrClient.getZkStateReader().updateClusterState();
  String cs = null;
  ClusterState clusterState = cloudSolrClient.getZkStateReader().getClusterState();
  if (collection != null) {
    cs = clusterState.getCollection(collection).toString();
  } else {
    Map<String, DocCollection> map = new HashMap<String, DocCollection>();
    for (String coll : clusterState.getCollections())
      map.put(coll, clusterState.getCollection(coll));
    CharArr out = new CharArr();
    new JSONWriter(out, 2).write(map);
    cs = out.toString();
  }
  return cs;
}
 
Example #7
Source File: ReplicaPropertiesBase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static void verifyPropertyNotPresent(CloudSolrClient client, String collectionName, String replicaName,
                              String property)
    throws KeeperException, InterruptedException {
  ClusterState clusterState = null;
  Replica replica = null;
  for (int idx = 0; idx < 300; ++idx) {
    clusterState = client.getZkStateReader().getClusterState();
    final DocCollection docCollection = clusterState.getCollectionOrNull(collectionName);
    replica = (docCollection == null) ? null : docCollection.getReplica(replicaName);
    if (replica == null) {
      fail("Could not find collection/replica pair! " + collectionName + "/" + replicaName);
    }
    if (StringUtils.isBlank(replica.getProperty(property))) return;
    Thread.sleep(100);
  }
  fail("Property " + property + " not set correctly for collection/replica pair: " +
      collectionName + "/" + replicaName + ". Replica props: " + replica.getProperties().toString() +
      ". Cluster state is " + clusterState.toString());

}
 
Example #8
Source File: OverseerCollectionMessageHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Send request to all replicas of a collection
 * @return List of replicas which is not live for receiving the request
 */
List<Replica> collectionCmd(ZkNodeProps message, ModifiableSolrParams params,
                   NamedList<Object> results, Replica.State stateMatcher, String asyncId, Set<String> okayExceptions) {
  log.info("Executing Collection Cmd={}, asyncId={}", params, asyncId);
  String collectionName = message.getStr(NAME);
  @SuppressWarnings("deprecation")
  ShardHandler shardHandler = shardHandlerFactory.getShardHandler(overseer.getCoreContainer().getUpdateShardHandler().getDefaultHttpClient());

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection coll = clusterState.getCollection(collectionName);
  List<Replica> notLivesReplicas = new ArrayList<>();
  final ShardRequestTracker shardRequestTracker = new ShardRequestTracker(asyncId);
  for (Slice slice : coll.getSlices()) {
    notLivesReplicas.addAll(shardRequestTracker.sliceCmd(clusterState, params, stateMatcher, slice, shardHandler));
  }

  shardRequestTracker.processResponses(results, shardHandler, false, null, okayExceptions);
  return notLivesReplicas;
}
 
Example #9
Source File: TestSimClusterStateProvider.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void assertClusterStateEquals(ClusterState one, ClusterState two) {
  assertEquals(one.getLiveNodes(), two.getLiveNodes());
  assertEquals(one.getCollectionsMap().keySet(), two.getCollectionsMap().keySet());
  one.forEachCollection(oneColl -> {
    DocCollection twoColl = two.getCollection(oneColl.getName());
    Map<String, Slice> oneSlices = oneColl.getSlicesMap();
    Map<String, Slice> twoSlices = twoColl.getSlicesMap();
    assertEquals(oneSlices.keySet(), twoSlices.keySet());
    oneSlices.forEach((s, slice) -> {
      Slice sTwo = twoSlices.get(s);
      for (Replica oneReplica : slice.getReplicas()) {
        Replica twoReplica = sTwo.getReplica(oneReplica.getName());
        assertNotNull(twoReplica);
        SimSolrCloudTestCase.assertReplicaEquals(oneReplica, twoReplica);
      }
    });
  });
}
 
Example #10
Source File: TopicStream.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void getCheckpoints() throws IOException {
  this.checkpoints = new HashMap<>();
  ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader();

  Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false);

  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();

  for(Slice slice : slices) {
    String sliceName = slice.getName();
    long checkpoint;
    if(initialCheckpoint > -1) {
      checkpoint = initialCheckpoint;
    } else {
      checkpoint = getCheckpoint(slice, liveNodes);
    }

    this.checkpoints.put(sliceName, checkpoint);
  }
}
 
Example #11
Source File: ShardSplitTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected void checkSubShardConsistency(String shard) throws SolrServerException, IOException {
  SolrQuery query = new SolrQuery("*:*").setRows(1000).setFields("id", "_version_");
  query.set("distrib", false);

  ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
  Slice slice = clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(shard);
  long[] numFound = new long[slice.getReplicasMap().size()];
  int c = 0;
  for (Replica replica : slice.getReplicas()) {
    String coreUrl = new ZkCoreNodeProps(replica).getCoreUrl();
    QueryResponse response;
    try (HttpSolrClient client = getHttpSolrClient(coreUrl)) {
      response = client.query(query);
    }
    numFound[c++] = response.getResults().getNumFound();
    if (log.isInfoEnabled()) {
      log.info("Shard: {} Replica: {} has {} docs", shard, coreUrl, String.valueOf(response.getResults().getNumFound()));
    }
    assertTrue("Shard: " + shard + " Replica: " + coreUrl + " has 0 docs", response.getResults().getNumFound() > 0);
  }
  for (int i = 0; i < slice.getReplicasMap().size(); i++) {
    assertEquals(shard + " is not consistent", numFound[0], numFound[i]);
  }
}
 
Example #12
Source File: ReplicaMutator.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Handles state updates
 */
public ZkWriteCommand setState(ClusterState clusterState, ZkNodeProps message) {
  String collectionName = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;
  String sliceName = message.getStr(ZkStateReader.SHARD_ID_PROP);

  if (collectionName == null || sliceName == null) {
    log.error("Invalid collection and slice {}", message);
    return ZkStateWriter.NO_OP;
  }
  DocCollection collection = clusterState.getCollectionOrNull(collectionName);
  Slice slice = collection != null ? collection.getSlice(sliceName) : null;
  if (slice == null) {
    log.error("No such slice exists {}", message);
    return ZkStateWriter.NO_OP;
  }

  return updateState(clusterState, message);
}
 
Example #13
Source File: ZkStateWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Applies the given {@link ZkWriteCommand} on the <code>prevState</code>. The modified
 * {@link ClusterState} is returned and it is expected that the caller will use the returned
 * cluster state for the subsequent invocation of this method.
 * <p>
 * The modified state may be buffered or flushed to ZooKeeper depending on the internal buffering
 * logic of this class. The {@link #hasPendingUpdates()} method may be used to determine if the
 * last enqueue operation resulted in buffered state. The method {@link #writePendingUpdates()} can
 * be used to force an immediate flush of pending cluster state changes.
 *
 * @param prevState the cluster state information on which the given <code>cmd</code> is applied
 * @param cmds       the list of {@link ZkWriteCommand} which specifies the change to be applied to cluster state in atomic
 * @param callback  a {@link org.apache.solr.cloud.overseer.ZkStateWriter.ZkWriteCallback} object to be used
 *                  for any callbacks
 * @return modified cluster state created after applying <code>cmd</code> to <code>prevState</code>. If
 * <code>cmd</code> is a no-op ({@link #NO_OP}) then the <code>prevState</code> is returned unmodified.
 * @throws IllegalStateException if the current instance is no longer usable. The current instance must be
 *                               discarded.
 * @throws Exception             on an error in ZK operations or callback. If a flush to ZooKeeper results
 *                               in a {@link org.apache.zookeeper.KeeperException.BadVersionException} this instance becomes unusable and
 *                               must be discarded
 */
public ClusterState enqueueUpdate(ClusterState prevState, List<ZkWriteCommand> cmds, ZkWriteCallback callback) throws IllegalStateException, Exception {
  if (invalidState) {
    throw new IllegalStateException("ZkStateWriter has seen a tragic error, this instance can no longer be used");
  }
  if (cmds.isEmpty()) return prevState;
  if (isNoOps(cmds)) return prevState;

  for (ZkWriteCommand cmd : cmds) {
    if (cmd == NO_OP) continue;
    prevState = prevState.copyWith(cmd.name, cmd.collection);
    updates.put(cmd.name, cmd.collection);
    numUpdates++;
  }
  clusterState = prevState;

  if (maybeFlushAfter()) {
    ClusterState state = writePendingUpdates();
    if (callback != null) {
      callback.onWrite();
    }
    return state;
  }

  return clusterState;
}
 
Example #14
Source File: ReplaceNodeCmd.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
static List<ZkNodeProps> getReplicasOfNode(String source, ClusterState state) {
  List<ZkNodeProps> sourceReplicas = new ArrayList<>();
  for (Map.Entry<String, DocCollection> e : state.getCollectionsMap().entrySet()) {
    for (Slice slice : e.getValue().getSlices()) {
      for (Replica replica : slice.getReplicas()) {
        if (source.equals(replica.getNodeName())) {
          ZkNodeProps props = new ZkNodeProps(
              COLLECTION_PROP, e.getKey(),
              SHARD_ID_PROP, slice.getName(),
              ZkStateReader.CORE_NAME_PROP, replica.getCoreName(),
              ZkStateReader.REPLICA_PROP, replica.getName(),
              ZkStateReader.REPLICA_TYPE, replica.getType().name(),
              ZkStateReader.LEADER_PROP, String.valueOf(replica.equals(slice.getLeader())),
              CoreAdminParams.NODE, source);
          sourceReplicas.add(props);
        }
      }
    }
  }
  return sourceReplicas;
}
 
Example #15
Source File: RebalanceLeaders.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private DocCollection checkParams() throws KeeperException, InterruptedException {
  req.getParams().required().check(COLLECTION_PROP);

  collectionName = req.getParams().get(COLLECTION_PROP);
  if (StringUtils.isBlank(collectionName)) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        String.format(Locale.ROOT, "The " + COLLECTION_PROP + " is required for the Rebalance Leaders command."));
  }
  coreContainer.getZkController().getZkStateReader().forceUpdateCollection(collectionName);
  ClusterState clusterState = coreContainer.getZkController().getClusterState();

  DocCollection dc = clusterState.getCollection(collectionName);
  if (dc == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection '" + collectionName + "' does not exist, no action taken.");
  }
  return dc;
}
 
Example #16
Source File: CdcrLeaderStateManager.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void process(WatchedEvent event) {
  if (isCancelled) return; // if the watcher is cancelled, do nothing.
  String collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
  String shard = core.getCoreDescriptor().getCloudDescriptor().getShardId();

  log.debug("The leader state has changed: {} @ {}:{}", event, collectionName, shard);
  // session events are not change events, and do not remove the watcher
  if (Event.EventType.None.equals(event.getType())) {
    return;
  }

  try {
    log.info("Received new leader state @ {}:{}", collectionName, shard);
    SolrZkClient zkClient = core.getCoreContainer().getZkController().getZkClient();
    ClusterState clusterState = core.getCoreContainer().getZkController().getClusterState();
    if (CdcrLeaderStateManager.this.isLeaderRegistered(zkClient, clusterState)) {
      CdcrLeaderStateManager.this.checkIfIAmLeader();
    }
  } catch (KeeperException | InterruptedException e) {
    log.warn("Failed updating leader state and setting watch @ {}: {}", collectionName, shard, e);
  }
}
 
Example #17
Source File: ManagedIndexSchema.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected static List<String> getActiveReplicaCoreUrls(ZkController zkController, String collection, String localCoreNodeName) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ZkStateReader zkStateReader = zkController.getZkStateReader();
  ClusterState clusterState = zkStateReader.getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlicesArr().length > 0) {
    final Slice[] activeSlices = docCollection.getActiveSlicesArr();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (!localCoreNodeName.equals(replica.getName()) &&
              replica.getState() == Replica.State.ACTIVE &&
              liveNodes.contains(replica.getNodeName())) {
            ZkCoreNodeProps replicaCoreProps = new ZkCoreNodeProps(replica);
            activeReplicaCoreUrls.add(replicaCoreProps.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 
Example #18
Source File: HttpSolrCall.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected SolrCore getCoreByCollection(String collectionName, boolean isPreferLeader) {
  ZkStateReader zkStateReader = cores.getZkController().getZkStateReader();

  ClusterState clusterState = zkStateReader.getClusterState();
  DocCollection collection = clusterState.getCollectionOrNull(collectionName, true);
  if (collection == null) {
    return null;
  }

  Set<String> liveNodes = clusterState.getLiveNodes();

  if (isPreferLeader) {
    List<Replica> leaderReplicas = collection.getLeaderReplicas(cores.getZkController().getNodeName());
    SolrCore core = randomlyGetSolrCore(liveNodes, leaderReplicas);
    if (core != null) return core;
  }

  List<Replica> replicas = collection.getReplicas(cores.getZkController().getNodeName());
  return randomlyGetSolrCore(liveNodes, replicas);
}
 
Example #19
Source File: SolrShardReporter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public String get() {
  CloudDescriptor cd = core.getCoreDescriptor().getCloudDescriptor();
  if (cd == null) {
    return null;
  }
  ClusterState state = core.getCoreContainer().getZkController().getClusterState();
  DocCollection collection = state.getCollection(core.getCoreDescriptor().getCollectionName());
  Replica replica = collection.getLeader(core.getCoreDescriptor().getCloudDescriptor().getShardId());
  if (replica == null) {
    log.warn("No leader for {}/{}", collection.getName(), core.getCoreDescriptor().getCloudDescriptor().getShardId());
    return null;
  }
  String baseUrl = replica.getStr("base_url");
  if (baseUrl == null) {
    log.warn("No base_url for replica {}", replica);
  }
  return baseUrl;
}
 
Example #20
Source File: SimClusterStateProvider.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public ClusterState getClusterState() throws IOException {
  ensureNotClosed();
  try {
    lock.lockInterruptibly();
    try {
      Map<String, DocCollection> states = getCollectionStates();
      ClusterState state = new ClusterState(liveNodes.get(), states);
      return state;
    } finally {
      lock.unlock();
    }
  } catch (InterruptedException e) {
    throw new IOException(e);
  }
}
 
Example #21
Source File: AbstractDistribZkTestBase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected static void assertAllActive(String collection, ZkStateReader zkStateReader)
    throws KeeperException, InterruptedException {

    zkStateReader.forceUpdateCollection(collection);
    ClusterState clusterState = zkStateReader.getClusterState();
    final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
    if (docCollection == null || docCollection.getSlices() == null) {
      throw new IllegalArgumentException("Cannot find collection:" + collection);
    }

    Map<String,Slice> slices = docCollection.getSlicesMap();
    for (Map.Entry<String,Slice> entry : slices.entrySet()) {
      Slice slice = entry.getValue();
      if (slice.getState() != Slice.State.ACTIVE) {
        fail("Not all shards are ACTIVE - found a shard " + slice.getName() + " that is: " + slice.getState());
      }
      Map<String,Replica> shards = slice.getReplicasMap();
      for (Map.Entry<String,Replica> shard : shards.entrySet()) {
        Replica replica = shard.getValue();
        if (replica.getState() != Replica.State.ACTIVE) {
          fail("Not all replicas are ACTIVE - found a replica " + replica.getName() + " that is: " + replica.getState());
        }
      }
    }
}
 
Example #22
Source File: CloudReplicaSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
private void withShardsParam(Builder builder, String shardsParam) {
  List<String> sliceOrUrls = StrUtils.splitSmart(shardsParam, ",", true);
  this.slices = new String[sliceOrUrls.size()];
  this.replicas = new List[sliceOrUrls.size()];

  ClusterState clusterState = builder.zkStateReader.getClusterState();

  for (int i = 0; i < sliceOrUrls.size(); i++) {
    String sliceOrUrl = sliceOrUrls.get(i);
    if (sliceOrUrl.indexOf('/') < 0) {
      // this is a logical shard
      this.slices[i] = sliceOrUrl;
      replicas[i] = findReplicas(builder, shardsParam, clusterState, clusterState.getCollection(builder.collection).getSlice(sliceOrUrl));
    } else {
      // this has urls
      this.replicas[i] = StrUtils.splitSmart(sliceOrUrl, "|", true);
      builder.replicaListTransformer.transform(replicas[i]);
      builder.hostChecker.checkWhitelist(clusterState, shardsParam, replicas[i]);
    }
  }
}
 
Example #23
Source File: SolrConfigHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public static List<String> getActiveReplicaCoreUrls(ZkController zkController,
                                                    String collection) {
  List<String> activeReplicaCoreUrls = new ArrayList<>();
  ClusterState clusterState = zkController.getZkStateReader().getClusterState();
  Set<String> liveNodes = clusterState.getLiveNodes();
  final DocCollection docCollection = clusterState.getCollectionOrNull(collection);
  if (docCollection != null && docCollection.getActiveSlices() != null && docCollection.getActiveSlices().size() > 0) {
    final Collection<Slice> activeSlices = docCollection.getActiveSlices();
    for (Slice next : activeSlices) {
      Map<String, Replica> replicasMap = next.getReplicasMap();
      if (replicasMap != null) {
        for (Map.Entry<String, Replica> entry : replicasMap.entrySet()) {
          Replica replica = entry.getValue();
          if (replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) {
            activeReplicaCoreUrls.add(replica.getCoreUrl());
          }
        }
      }
    }
  }
  return activeReplicaCoreUrls;
}
 
Example #24
Source File: OverseerCollectionMessageHandler.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unchecked")
private void processReplicaAddPropertyCommand(ClusterState clusterState, ZkNodeProps message, @SuppressWarnings({"rawtypes"})NamedList results)
    throws Exception {
  checkRequired(message, COLLECTION_PROP, SHARD_ID_PROP, REPLICA_PROP, PROPERTY_PROP, PROPERTY_VALUE_PROP);
  SolrZkClient zkClient = zkStateReader.getZkClient();
  Map<String, Object> propMap = new HashMap<>();
  propMap.put(Overseer.QUEUE_OPERATION, ADDREPLICAPROP.toLower());
  propMap.putAll(message.getProperties());
  ZkNodeProps m = new ZkNodeProps(propMap);
  overseer.offerStateUpdate(Utils.toJSON(m));
}
 
Example #25
Source File: CollectionMutator.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public ZkWriteCommand deleteShard(final ClusterState clusterState, ZkNodeProps message) {
  final String sliceId = message.getStr(ZkStateReader.SHARD_ID_PROP);
  final String collection = message.getStr(ZkStateReader.COLLECTION_PROP);
  if (!checkCollectionKeyExistence(message)) return ZkStateWriter.NO_OP;

  log.info("Removing collection: {} shard: {}  from clusterstate", collection, sliceId);

  DocCollection coll = clusterState.getCollection(collection);

  Map<String, Slice> newSlices = new LinkedHashMap<>(coll.getSlicesMap());
  newSlices.remove(sliceId);

  DocCollection newCollection = coll.copyWithSlices(newSlices);
  return new ZkWriteCommand(collection, newCollection);
}
 
Example #26
Source File: CoreSorter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
CoreSorter init(CoreContainer cc, Collection<CoreDescriptor> coreDescriptors) {
  String myNodeName = cc.getNodeConfig().getNodeName();
  ClusterState state = cc.getZkController().getClusterState();
  for (CoreDescriptor coreDescriptor : coreDescriptors) {
    CloudDescriptor cloudDescriptor = coreDescriptor.getCloudDescriptor();
    String coll = cloudDescriptor.getCollectionName();
    String sliceName = getShardName(cloudDescriptor);
    if (shardsVsReplicaCounts.containsKey(sliceName)) continue;
    CountsForEachShard c = new CountsForEachShard(0, 0, 0);
    for (Replica replica : getReplicas(state, coll, cloudDescriptor.getShardId())) {
      if (replica.getNodeName().equals(myNodeName)) {
        c.myReplicas++;
      } else {
        Set<String> liveNodes = state.getLiveNodes();
        if (liveNodes.contains(replica.getNodeName())) {
          c.totalReplicasInLiveNodes++;
        } else {
          c.totalReplicasInDownNodes++;
        }
      }
    }
    shardsVsReplicaCounts.put(sliceName, c);
  }

  return this;

}
 
Example #27
Source File: Solr5Index.java    From incubator-atlas with Apache License 2.0 5 votes vote down vote up
/**
 * Wait for all the collection shards to be ready.
 */
private static void waitForRecoveriesToFinish(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    ZkStateReader zkStateReader = server.getZkStateReader();
    try {
        boolean cont = true;

        while (cont) {
            boolean sawLiveRecovering = false;
            zkStateReader.updateClusterState();
            ClusterState clusterState = zkStateReader.getClusterState();
            Map<String, Slice> slices = clusterState.getSlicesMap(collection);
            Preconditions.checkNotNull("Could not find collection:" + collection, slices);

            for (Map.Entry<String, Slice> entry : slices.entrySet()) {
                Map<String, Replica> shards = entry.getValue().getReplicasMap();
                for (Map.Entry<String, Replica> shard : shards.entrySet()) {
                    String state = shard.getValue().getStr(ZkStateReader.STATE_PROP);
                    if ((state.equals(Replica.State.RECOVERING.toString())
                            || state.equals(Replica.State.DOWN.toString()))
                            && clusterState.liveNodesContain(shard.getValue().getStr(
                            ZkStateReader.NODE_NAME_PROP))) {
                        sawLiveRecovering = true;
                    }
                }
            }
            if (!sawLiveRecovering) {
                cont = false;
            } else {
                Thread.sleep(1000);
            }
        }
    } finally {
        logger.info("Exiting solr wait");
    }
}
 
Example #28
Source File: Solr6Index.java    From atlas with Apache License 2.0 5 votes vote down vote up
/**
 * Checks if the collection has already been created in Solr.
 */
private static boolean checkIfCollectionExists(CloudSolrClient server, String collection) throws KeeperException, InterruptedException {
    final ZkStateReader zkStateReader = server.getZkStateReader();
    zkStateReader.forceUpdateCollection(collection);
    final ClusterState clusterState = zkStateReader.getClusterState();
    return clusterState.getCollectionOrNull(collection) != null;
}
 
Example #29
Source File: MaintainRoutedAliasCmd.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void deleteTargetCollection(ClusterState clusterState, @SuppressWarnings({"rawtypes"})NamedList results, String aliasName, ZkStateReader.AliasesManager aliasesManager, RoutedAlias.Action action) throws Exception {
  Map<String, Object> delProps = new HashMap<>();
  delProps.put(INVOKED_BY_ROUTED_ALIAS,
      (Runnable) () -> removeCollectionFromAlias(aliasName, aliasesManager, action.targetCollection));
  delProps.put(NAME, action.targetCollection);
  ZkNodeProps messageDelete = new ZkNodeProps(delProps);
  new DeleteCollectionCmd(ocmh).call(clusterState, messageDelete, results);
}
 
Example #30
Source File: BasicDistributedZkTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected ZkCoreNodeProps getLeaderUrlFromZk(String collection, String slice) {
  ClusterState clusterState = getCommonCloudSolrClient().getZkStateReader().getClusterState();
  ZkNodeProps leader = clusterState.getCollection(collection).getLeader(slice);
  if (leader == null) {
    throw new RuntimeException("Could not find leader:" + collection + " " + slice);
  }
  return new ZkCoreNodeProps(leader);
}