Java Code Examples for org.apache.solr.common.cloud.Slice#getRange()

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
protected boolean amISubShardLeader(DocCollection coll, Slice parentSlice, String id, SolrInputDocument doc) throws InterruptedException {
  // Am I the leader of a shard in "construction/recovery" state?
  String myShardId = cloudDesc.getShardId();
  Slice mySlice = coll.getSlice(myShardId);
  final Slice.State state = mySlice.getState();
  if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY) {
    Replica myLeader = zkController.getZkStateReader().getLeaderRetry(collection, myShardId);
    boolean amILeader = myLeader.getName().equals(cloudDesc.getCoreNodeName());
    if (amILeader) {
      // Does the document belong to my hash range as well?
      DocRouter.Range myRange = mySlice.getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      if (parentSlice != null)  {
        boolean isSubset = parentSlice.getRange() != null && myRange.isSubsetOf(parentSlice.getRange());
        return isSubset && coll.getRouter().isTargetSlice(id, doc, req.getParams(), myShardId, coll);
      } else  {
        // delete by query case -- as long as I am a sub shard leader we're fine
        return true;
      }
    }
  }
  return false;
}
 

/** For {@link org.apache.solr.common.params.CollectionParams.CollectionAction#SPLITSHARD} */
protected List<SolrCmdDistributor.Node> getSubShardLeaders(DocCollection coll, String shardId, String docId, SolrInputDocument doc) {
  Collection<Slice> allSlices = coll.getSlices();
  List<SolrCmdDistributor.Node> nodes = null;
  for (Slice aslice : allSlices) {
    final Slice.State state = aslice.getState();
    if (state == Slice.State.CONSTRUCTION || state == Slice.State.RECOVERY)  {
      DocRouter.Range myRange = coll.getSlice(shardId).getRange();
      if (myRange == null) myRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      boolean isSubset = aslice.getRange() != null && aslice.getRange().isSubsetOf(myRange);
      if (isSubset &&
          (docId == null // in case of deletes
              || coll.getRouter().isTargetSlice(docId, doc, req.getParams(), aslice.getName(), coll))) {
        Replica sliceLeader = aslice.getLeader();
        // slice leader can be null because node/shard is created zk before leader election
        if (sliceLeader != null && clusterState.liveNodesContain(sliceLeader.getNodeName()))  {
          if (nodes == null) nodes = new ArrayList<>();
          ZkCoreNodeProps nodeProps = new ZkCoreNodeProps(sliceLeader);
          nodes.add(new SolrCmdDistributor.StdNode(nodeProps, coll.getName(), aslice.getName()));
        }
      }
    }
  }
  return nodes;
}
 

private String createHashRangeFq() {
  if (routedByJoinKey) {
    ClusterState clusterState = searcher.getCore().getCoreContainer().getZkController().getClusterState();
    CloudDescriptor desc = searcher.getCore().getCoreDescriptor().getCloudDescriptor();
    Slice slice = clusterState.getCollection(desc.getCollectionName()).getSlicesMap().get(desc.getShardId());
    DocRouter.Range range = slice.getRange();

    // In CompositeIdRouter, the routing prefix only affects the top 16 bits
    int min = range.min & 0xffff0000;
    int max = range.max | 0x0000ffff;

    return String.format(Locale.ROOT, "{!hash_range f=%s l=%d u=%d}", fromField, min, max);
  } else {
    return null;
  }
}
 

@Test
public void testSplitFuzz() throws Exception {
  String collectionName = "splitFuzzCollection";
  CollectionAdminRequest
      .createCollection(collectionName, "conf", 2, 1)
      .setMaxShardsPerNode(100)
      .process(cluster.getSolrClient());

  cluster.waitForActiveCollection(collectionName, 2, 2);

  CollectionAdminRequest.SplitShard splitShard = CollectionAdminRequest.splitShard(collectionName)
      .setSplitFuzz(0.5f)
      .setShardName("shard1");
  splitShard.process(cluster.getSolrClient());
  waitForState("Timed out waiting for sub shards to be active. Number of active shards=" +
          cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(collectionName).getActiveSlices().size(),
      collectionName, activeClusterShape(3, 4));
  DocCollection coll = cluster.getSolrClient().getZkStateReader().getClusterState().getCollection(collectionName);
  Slice s1_0 = coll.getSlice("shard1_0");
  Slice s1_1 = coll.getSlice("shard1_1");
  long fuzz = ((long)Integer.MAX_VALUE >> 3) + 1L;
  long delta0 = s1_0.getRange().max - s1_0.getRange().min;
  long delta1 = s1_1.getRange().max - s1_1.getRange().min;
  long expected0 = (Integer.MAX_VALUE >> 1) + fuzz;
  long expected1 = (Integer.MAX_VALUE >> 1) - fuzz;
  assertEquals("wrong range in s1_0", expected0, delta0);
  assertEquals("wrong range in s1_1", expected1, delta1);
}
 

private void doDefensiveChecks(DistribPhase phase) {
  boolean isReplayOrPeersync = (updateCommand.getFlags() & (UpdateCommand.REPLAY | UpdateCommand.PEER_SYNC)) != 0;
  if (isReplayOrPeersync) return;

  String from = req.getParams().get(DISTRIB_FROM);

  DocCollection docCollection = clusterState.getCollection(collection);
  Slice mySlice = docCollection.getSlice(cloudDesc.getShardId());
  boolean localIsLeader = cloudDesc.isLeader();
  if (DistribPhase.FROMLEADER == phase && localIsLeader && from != null) { // from will be null on log replay
    String fromShard = req.getParams().get(DISTRIB_FROM_PARENT);
    if (fromShard != null) {
      if (mySlice.getState() == Slice.State.ACTIVE)  {
        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
            "Request says it is coming from parent shard leader but we are in active state");
      }
      // shard splitting case -- check ranges to see if we are a sub-shard
      Slice fromSlice = docCollection.getSlice(fromShard);
      DocRouter.Range parentRange = fromSlice.getRange();
      if (parentRange == null) parentRange = new DocRouter.Range(Integer.MIN_VALUE, Integer.MAX_VALUE);
      if (mySlice.getRange() != null && !mySlice.getRange().isSubsetOf(parentRange)) {
        throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
            "Request says it is coming from parent shard leader but parent hash range is not superset of my range");
      }
    } else {
      String fromCollection = req.getParams().get(DISTRIB_FROM_COLLECTION); // is it because of a routing rule?
      if (fromCollection == null)  {
        log.error("Request says it is coming from leader, but we are the leader: {}", req.getParamString());
        SolrException solrExc = new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE, "Request says it is coming from leader, but we are the leader");
        solrExc.setMetadata("cause", "LeaderChanged");
        throw solrExc;
      }
    }
  }

  int count = 0;
  while (((isLeader && !localIsLeader) || (isSubShardLeader && !localIsLeader)) && count < 5) {
    count++;
    // re-getting localIsLeader since we published to ZK first before setting localIsLeader value
    localIsLeader = cloudDesc.isLeader();
    try {
      Thread.sleep(500);
    } catch (InterruptedException e) {
      Thread.currentThread().interrupt();
    }
  }

  if ((isLeader && !localIsLeader) || (isSubShardLeader && !localIsLeader)) {
    log.error("ClusterState says we are the leader, but locally we don't think so");
    throw new SolrException(SolrException.ErrorCode.SERVICE_UNAVAILABLE,
        "ClusterState says we are the leader (" + zkController.getBaseUrl()
            + "/" + req.getCore().getName() + "), but locally we don't think so. Request came from " + from);
  }
}
 

/**
 *   This is called when splitByPrefix is used.
 *   The overseer called us to get recommended splits taking into
 *   account actual document distribution over the hash space.
 */
private void handleGetRanges(CoreAdminHandler.CallInfo it, String coreName) throws Exception {

  SolrCore parentCore = it.handler.coreContainer.getCore(coreName);
  if (parentCore == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unknown core " + coreName);
  }

  RefCounted<SolrIndexSearcher> searcherHolder = parentCore.getRealtimeSearcher();

  try {
    if (!it.handler.coreContainer.isZooKeeperAware()) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Shard splitByPrefix requires SolrCloud mode.");
    } else {
      SolrIndexSearcher searcher = searcherHolder.get();

      String routeFieldName = null;
      String prefixField = "id_prefix";

      ClusterState clusterState = it.handler.coreContainer.getZkController().getClusterState();
      String collectionName = parentCore.getCoreDescriptor().getCloudDescriptor().getCollectionName();
      DocCollection collection = clusterState.getCollection(collectionName);
      String sliceName = parentCore.getCoreDescriptor().getCloudDescriptor().getShardId();
      Slice slice = collection.getSlice(sliceName);
      DocRouter router = collection.getRouter() != null ? collection.getRouter() : DocRouter.DEFAULT;
      DocRouter.Range currentRange = slice.getRange();

      Object routerObj = collection.get(DOC_ROUTER); // for back-compat with Solr 4.4
      if (routerObj instanceof Map) {
        @SuppressWarnings({"rawtypes"})
        Map routerProps = (Map) routerObj;
        routeFieldName = (String) routerProps.get("field");
      }
      if (routeFieldName == null) {
        routeFieldName = searcher.getSchema().getUniqueKeyField().getName();
      }

      Collection<RangeCount> counts = getHashHistogram(searcher, prefixField, router, collection);

      if (counts.size() == 0) {
        // How to determine if we should look at the id field to figure out the prefix buckets?
        // There may legitimately be no indexed terms in id_prefix if no ids have a prefix yet.
        // For now, avoid using splitByPrefix unless you are actually using prefixes.
        counts = getHashHistogramFromId(searcher, searcher.getSchema().getUniqueKeyField().getName(), router, collection);
      }

      Collection<DocRouter.Range> splits = getSplits(counts, currentRange);
      String splitString = toSplitString(splits);

      if (splitString == null) {
        return;
      }

      it.rsp.add(CoreAdminParams.RANGES, splitString);
    }
  } finally {
    if (searcherHolder != null) searcherHolder.decref();
    if (parentCore != null) parentCore.close();
  }
}
 

public void splitByRouteFieldTest() throws Exception  {
  log.info("Starting testSplitWithRouteField");
  String collectionName = "routeFieldColl";
  int numShards = 4;
  int replicationFactor = 2;
  int maxShardsPerNode = (((numShards * replicationFactor) / getCommonCloudSolrClient()
      .getZkStateReader().getClusterState().getLiveNodes().size())) + 1;

  HashMap<String, List<Integer>> collectionInfos = new HashMap<>();
  String shard_fld = "shard_s";
  try (CloudSolrClient client = createCloudClient(null)) {
    Map<String, Object> props = Utils.makeMap(
        REPLICATION_FACTOR, replicationFactor,
        MAX_SHARDS_PER_NODE, maxShardsPerNode,
        OverseerCollectionMessageHandler.NUM_SLICES, numShards,
        "router.field", shard_fld);

    createCollection(collectionInfos, collectionName, props, client);
  }

  List<Integer> list = collectionInfos.get(collectionName);
  checkForCollection(collectionName, list, null);

  waitForRecoveriesToFinish(false);

  String url = getUrlFromZk(getCommonCloudSolrClient().getZkStateReader().getClusterState(), collectionName);

  try (HttpSolrClient collectionClient = getHttpSolrClient(url)) {

    ClusterState clusterState = cloudClient.getZkStateReader().getClusterState();
    final DocRouter router = clusterState.getCollection(collectionName).getRouter();
    Slice shard1 = clusterState.getCollection(collectionName).getSlice(SHARD1);
    DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange();
    final List<DocRouter.Range> ranges = router.partitionRange(2, shard1Range);
    final int[] docCounts = new int[ranges.size()];

    for (int i = 100; i <= 200; i++) {
      String shardKey = "" + (char) ('a' + (i % 26)); // See comment in ShardRoutingTest for hash distribution

      collectionClient.add(getDoc(id, i, "n_ti", i, shard_fld, shardKey));
      int idx = getHashRangeIdx(router, ranges, shardKey);
      if (idx != -1) {
        docCounts[idx]++;
      }
    }

    for (int i = 0; i < docCounts.length; i++) {
      int docCount = docCounts[i];
      log.info("Shard shard1_{} docCount = {}", i, docCount);
    }

    collectionClient.commit();

    trySplit(collectionName, null, SHARD1, 3);

    waitForRecoveriesToFinish(collectionName, false);

    assertEquals(docCounts[0], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_0")).getResults().getNumFound());
    assertEquals(docCounts[1], collectionClient.query(new SolrQuery("*:*").setParam("shards", "shard1_1")).getResults().getNumFound());
  }
}