org.apache.solr.common.params.ShardParams Java Examples

The following examples show how to use org.apache.solr.common.params.ShardParams. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TermsComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(ResponseBuilder rb) throws IOException {
  SolrParams params = rb.req.getParams();

  //the terms parameter is also used by json facet API. So we will get errors if we try to parse as boolean
  if (params.get(TermsParams.TERMS, "false").equals("true")) {
    rb.doTerms = true;
  } else {
    return;
  }

  // TODO: temporary... this should go in a different component.
  String shards = params.get(ShardParams.SHARDS);
  if (shards != null) {
    rb.isDistrib = true;
    if (params.get(ShardParams.SHARDS_QT) == null) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No shards.qt parameter specified");
    }
    List<String> lst = StrUtils.splitSmart(shards, ",", true);
    checkShardsWhitelist(rb, lst);
    rb.shards = lst.toArray(new String[lst.size()]);
  }
}
 
Example #2
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
/** Check that if the mincount is set as 0 then it is updated to be 1. */
@Test
public void rewriteMincountFacetFieldOption_perFieldMincountSetZero_shouldSetPerFieldMincountToOne()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("f.NAME.facet.mincount","f.CONTENT.facet.mincount").iterator());
    when(mockParams.getParams("f.NAME.facet.mincount")).thenReturn(new String[]{"0"});
    when(mockParams.getParams("f.CONTENT.facet.mincount")).thenReturn(new String[]{"0"});
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);

    Map<String, String> fieldMappings = ImmutableMap.of(
        "NAME",
                "{!afts key=SEARCH.FACET_FIELDS.LOCATION}text@s____@{http://www.alfresco.org/model/content/1.0}name","CONTENT",
        "{!afts key=SEARCH.FACET_FIELDS.LOCATION}text@s____@{http://www.alfresco.org/model/content/1.0}content");

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualNameCount = fixed.get("f.{!afts key=SEARCH.FACET_FIELDS.LOCATION}text@s____@{http://www.alfresco.org/model/content/1.0}name.facet.mincount");
    assertEquals("Expected the mincount to be 1.", "1", actualNameCount);
    String actualContentCount = fixed.get("f.{!afts key=SEARCH.FACET_FIELDS.LOCATION}text@s____@{http://www.alfresco.org/model/content/1.0}content.facet.mincount");
    assertEquals("Expected the mincount to be 1.", "1", actualContentCount);
}
 
Example #3
Source File: PhrasesIdentificationComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void process(ResponseBuilder rb) throws IOException {
  final PhrasesContextData contextData = (PhrasesContextData) rb.req.getContext().get(this.getClass());
  if (null == contextData) {
    // if prepare didn't give us anything to work with, then we should do nothing
    return;
  }

  // regardless of single node / shard, we need local stats...
  Phrase.populateStats(contextData.allPhrases, contextData.fieldWeights.keySet(), rb.req.getSearcher());

  if ( rb.req.getParams().getBool(ShardParams.IS_SHARD, false) ) {
    // shard request, return stats for all phrases (in original order)
    SimpleOrderedMap<Object> output = new SimpleOrderedMap<>();
    output.add("_all", Phrase.formatShardResponse(contextData.allPhrases));
    // TODO: might want to add numDocs() & getSumTotalTermFreq(f)/getDocCount(f) stats from each field...
    // so that we can sum/merge them for use in scoring?
    rb.rsp.add("phrases", output);
  } else {
    // full single node request...
    scoreAndAddResultsToResponse(rb, contextData);
  }
}
 
Example #4
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
@Test
public void rewriteMincountFacetPivotOption_mincountMissing_shouldSetPivotMinCountToOne()
{
    // There are no existing facet parameters.
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    when(mockParams.getParameterNamesIterator()).thenReturn(Iterators.empty());
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);

    Map<String, String> fieldMappings = new HashMap<>();

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.pivot.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1.
    String actual = fixed.get("facet.pivot.mincount");
    assertEquals("Expected the existing mincount to be preserved.", "1", actual);
}
 
Example #5
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
@Test
public void rewriteMincountFacetFieldOption_mincountSetTwo_shouldKeepIt()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("facet.mincount").iterator());
    when(mockParams.get("facet.mincount")).thenReturn("2");
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);

    Map<String, String> fieldMappings = new HashMap<>();

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualCount = fixed.get("facet.mincount");
    assertEquals("Expected the mincount to be 2.", "2", actualCount);
}
 
Example #6
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
@Test
public void rewriteShardedRequestParameters_mincountSetZero_shouldKeepMincountToZero()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("facet.mincount").iterator());
    when(mockParams.get("facet.mincount")).thenReturn("0");
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(String.valueOf(ShardRequest.PURPOSE_GET_FACETS));

    Map<String, String> fieldMappings = new HashMap<>();

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualCount = fixed.get("facet.mincount");
    assertEquals("Expected no fixed value", null, actualCount);
}
 
Example #7
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
/** Check that if the mincount is set as 0 then it is updated to be 1. */
@Test
public void rewriteMincountFacetFieldOption_mincountSetZero_shouldSetMincountToOne()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("facet.mincount").iterator());
    when(mockParams.get("facet.mincount")).thenReturn("0");
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);
    
    Map<String, String> fieldMappings = new HashMap<>();
    
    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualCount = fixed.get("facet.mincount");
    assertEquals("Expected the mincount to be 1.", "1", actualCount);
}
 
Example #8
Source File: TestDistributedDymReSearcher.java    From solr-researcher with Apache License 2.0 6 votes vote down vote up
@Test
@ShardsFixed(num = 2)
public void test() throws Exception {
  handle.clear();
  handle.put("QTime", SKIPVAL);
  handle.put("timestamp", SKIPVAL);
  handle.put("maxScore", SKIPVAL);
  handle.put("responseHeader", SKIP);
  handle.put("spellchecked_response", UNORDERED);

  query(CommonParams.QT, "standardResWithCommonMisspellings", 
      ShardParams.SHARDS_QT, "standardResWithCommonMisspellings",
      CommonParams.Q, "foo:bobo AND foo:marley", 
      SpellingParams.SPELLCHECK_COLLATE, "true",
      SpellingParams.SPELLCHECK_BUILD, "true", 
      SpellingParams.SPELLCHECK_COUNT, "10",
      SpellingParams.SPELLCHECK_EXTENDED_RESULTS, "true", 
      DymReSearcher.COMPONENT_NAME, "true",
      SpellCheckComponent.COMPONENT_NAME, "true");
}
 
Example #9
Source File: AlfrescoSearchHandler.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
private ShardHandler getAndPrepShardHandler(SolrQueryRequest req,
		ResponseBuilder rb) {
	ShardHandler shardHandler = null;

	rb.isDistrib = req.getParams().getBool(
			"distrib",
			req.getCore().getCoreContainer().isZooKeeperAware());
	if (!rb.isDistrib) {
		// for back compat, a shards param with URLs like
		// localhost:8983/solr will mean that this
		// search is distributed.
		final String shards = req.getParams().get(ShardParams.SHARDS);
		rb.isDistrib = ((shards != null) && (shards.indexOf('/') > 0));
	}

	if (rb.isDistrib) {
		shardHandler = shardHandlerFactory.getShardHandler();
		shardHandler.prepDistributed(rb);
		if (!rb.isDistrib) {
			shardHandler = null; // request is not distributed after all and
									// so the shard handler is not needed
		}
	}

	return shardHandler;
}
 
Example #10
Source File: AsyncBuildSuggestComponent.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
/** Dispatch shard request in <code>STAGE_EXECUTE_QUERY</code> stage */
@Override
public int distributedProcess(ResponseBuilder rb) {
  SolrParams params = rb.req.getParams();
  LOG.debug("SuggestComponent distributedProcess with : " + params);
  if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) 
    return ResponseBuilder.STAGE_EXECUTE_QUERY;
  if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
    ShardRequest sreq = new ShardRequest();
    sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
    sreq.params = new ModifiableSolrParams(rb.req.getParams());
    sreq.params.remove(ShardParams.SHARDS);
    rb.addRequest(this, sreq);
    return ResponseBuilder.STAGE_GET_FIELDS;
  }

  return ResponseBuilder.STAGE_DONE;
}
 
Example #11
Source File: DistributedDebugComponentTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testTolerantSearch() throws SolrServerException, IOException {
  String badShard = DEAD_HOST_1;
  SolrQuery query = new SolrQuery();
  query.setQuery("*:*");
  query.set("debug",  "true");
  query.set("distrib", "true");
  query.setFields("id", "text");
  query.set("shards", shard1 + "," + shard2 + "," + badShard);

  // verify that the request would fail if shards.tolerant=false
  ignoreException("Server refused connection");
  expectThrows(SolrException.class, () -> collection1.query(query));

  query.set(ShardParams.SHARDS_TOLERANT, "true");
  QueryResponse response = collection1.query(query);
  assertTrue((Boolean)response.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
  @SuppressWarnings("unchecked")
  NamedList<String> badShardTrack =
          (((NamedList<NamedList<NamedList<String>>>)response.getDebugMap().get("track")).get("EXECUTE_QUERY")).get(badShard);
  assertEquals("Unexpected response size for shard", 1, badShardTrack.size());
  Entry<String, String> exception = badShardTrack.iterator().next();
  assertEquals("Expected key 'Exception' not found", "Exception", exception.getKey());
  assertNotNull("Exception message should not be null", exception.getValue());
  unIgnoreException("Server refused connection");
}
 
Example #12
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
/** Check that if the mincount is set as 0 then it is updated to be 1. */
@Test
public void rewriteMincountFacetPivotOption_mincountSetZero_shouldSetMincountToOne()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("facet.pivot.mincount").iterator());
    when(mockParams.get("facet.pivot.mincount")).thenReturn("0");
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);

    Map<String, String> fieldMappings = new HashMap<>();

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.pivot.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualCount = fixed.get("facet.pivot.mincount");
    assertEquals("Expected the mincount to be 1.", "1", actualCount);
}
 
Example #13
Source File: PhrasesIdentificationComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void prepare(ResponseBuilder rb) throws IOException {
  final SolrParams params = rb.req.getParams();
  if (!params.getBool(COMPONENT_NAME, false)) {
    return;
  }
  if (params.getBool(ShardParams.IS_SHARD, false)) {
    // only one stage/purpose where we should do any work on a shard
    if (0 == (SHARD_PURPOSE & params.getInt(ShardParams.SHARDS_PURPOSE, 0))) {
      return;
    }
  }

  // if we're still here, then we should parse & validate our input, 
  // putting it in the request context so our process method knows it should do work
  rb.req.getContext().put(this.getClass(), PhrasesContextData.parseAndValidateRequest(rb.req));
}
 
Example #14
Source File: NodePreferenceRulesComparator.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private boolean hasCoreUrlPrefix(Object o, String prefix) {
  final String s;
  if (o instanceof String) {
    s = (String)o;
  }
  else if (o instanceof Replica) {
    s = ((Replica)o).getCoreUrl();
  } else {
    return false;
  }
  if (prefix.equals(ShardParams.REPLICA_LOCAL)) {
    return !StringUtils.isEmpty(localHostAddress) && s.startsWith(localHostAddress);
  } else {
    return s.startsWith(prefix);
  }
}
 
Example #15
Source File: AffinityReplicaListTransformerFactory.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public ReplicaListTransformer getInstance(String configSpec, SolrParams requestParams, ReplicaListTransformerFactory fallback) {
  ReplicaListTransformer rlt;
  if (configSpec == null) {
    rlt = AffinityReplicaListTransformer.getInstance(defaultDividendParam, defaultHashParam, requestParams);
  } else {
    String[] parts = configSpec.split(":", 2);
    switch (parts[0]) {
      case ShardParams.ROUTING_DIVIDEND:
        rlt = AffinityReplicaListTransformer.getInstance(parts.length == 1 ? defaultDividendParam : parts[1], defaultHashParam, requestParams);
        break;
      case ShardParams.ROUTING_HASH:
        rlt = AffinityReplicaListTransformer.getInstance(null, parts.length == 1 ? defaultHashParam : parts[1], requestParams);
        break;
      default:
        throw new IllegalArgumentException("Invalid routing spec: \"" + configSpec + '"');
    }
  }
  return rlt != null ? rlt : fallback.getInstance(null, requestParams, null);
}
 
Example #16
Source File: RealTimeGetComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Helper method for creating a new ShardRequest for the specified ids, based on the params 
 * specified for the current request.  The new ShardRequest does not yet know anything about 
 * which shard/slice it will be sent to.
 */
private ShardRequest createShardRequest(final ResponseBuilder rb, final List<String> ids) {
  final ShardRequest sreq = new ShardRequest();
  sreq.purpose = 1;
  sreq.params = new ModifiableSolrParams(rb.req.getParams());

  // TODO: how to avoid hardcoding this and hit the same handler?
  sreq.params.set(ShardParams.SHARDS_QT,"/get");      
  sreq.params.set(DISTRIB,false);

  sreq.params.remove(ShardParams.SHARDS);
  sreq.params.remove(ID);
  sreq.params.remove("ids");
  sreq.params.set("ids", StrUtils.join(ids, ','));
  
  return sreq;
}
 
Example #17
Source File: NodePreferenceRulesComparatorTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void replicaTypeTest() {
  List<Replica> replicas = getBasicReplicaList();

  List<PreferenceRule> rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT," +
      ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG");
  NodePreferenceRulesComparator comparator = new NodePreferenceRulesComparator(rules, null);

  replicas.sort(comparator);
  assertEquals("node1", replicas.get(0).getNodeName());
  assertEquals("node2", replicas.get(1).getNodeName());

  // reversed rule
  rules = PreferenceRule.from(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG," +
      ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":NRT");
  comparator = new NodePreferenceRulesComparator(rules, null);

  replicas.sort(comparator);
  assertEquals("node2", replicas.get(0).getNodeName());
  assertEquals("node1", replicas.get(1).getNodeName());
}
 
Example #18
Source File: SuggestComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Dispatch shard request in <code>STAGE_EXECUTE_QUERY</code> stage */
@Override
public int distributedProcess(ResponseBuilder rb) {
  SolrParams params = rb.req.getParams();
  log.info("SuggestComponent distributedProcess with : {}", params);
  if (rb.stage < ResponseBuilder.STAGE_EXECUTE_QUERY) 
    return ResponseBuilder.STAGE_EXECUTE_QUERY;
  if (rb.stage == ResponseBuilder.STAGE_EXECUTE_QUERY) {
    ShardRequest sreq = new ShardRequest();
    sreq.purpose = ShardRequest.PURPOSE_GET_TOP_IDS;
    sreq.params = new ModifiableSolrParams(rb.req.getParams());
    sreq.params.remove(ShardParams.SHARDS);
    rb.addRequest(this, sreq);
    return ResponseBuilder.STAGE_GET_FIELDS;
  }

  return ResponseBuilder.STAGE_DONE;
}
 
Example #19
Source File: TestShortCircuitedRequests.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
@ShardsFixed(num = 4)
public void test() throws Exception {
  waitForRecoveriesToFinish(false);
  assertEquals(4, cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getSlices().size());
  index("id", "a!doc1");  // shard3
  index("id", "b!doc1");  // shard1
  index("id", "c!doc1");  // shard2
  index("id", "e!doc1");  // shard4
  commit();

  doQuery("a!doc1", "q", "*:*", ShardParams._ROUTE_, "a!"); // can go to any random node

  // query shard3 directly with _route_=a! so that we trigger the short circuited request path
  Replica shard3 = cloudClient.getZkStateReader().getClusterState().getCollection(DEFAULT_COLLECTION).getLeader("shard3");
  String nodeName = shard3.getNodeName();
  SolrClient shard3Client = getClient(nodeName);
  QueryResponse response = shard3Client.query(new SolrQuery("*:*").add(ShardParams._ROUTE_, "a!").add(ShardParams.SHARDS_INFO, "true"));

  assertEquals("Could not find doc", 1, response.getResults().getNumFound());
  NamedList<?> sinfo = (NamedList<?>) response.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("missing shard info for short circuited request", sinfo);
}
 
Example #20
Source File: RewriteFacetParametersComponentTest.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
@Test
public void rewriteMincountFacetPivotOption_mincountSetTwo_shouldKeepIt()
{
    ModifiableSolrParams fixed = new ModifiableSolrParams();
    // The user has tried to set the mincount to zero.
    when(mockParams.getParameterNamesIterator()).thenReturn(asList("facet.pivot.mincount").iterator());
    when(mockParams.get("facet.pivot.mincount")).thenReturn("2");
    when(mockParams.get(ShardParams.SHARDS_PURPOSE)).thenReturn(null);

    Map<String, String> fieldMappings = new HashMap<>();

    // Call the method under test.
    rewriteFacetParametersComponent.rewriteMincountFacetFieldOption(fixed, mockParams, "facet.pivot.mincount", fieldMappings,
        mockRequest);

    // Check that the mincount is set to 1 and the field name is converted to the format stored by Solr.
    String actualCount = fixed.get("facet.pivot.mincount");
    assertEquals("Expected the mincount to be 2.", "2", actualCount);
}
 
Example #21
Source File: DistributedQueryComponentOptimizationTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Test
public void testWildcardFieldList() throws Exception {

  QueryResponse nonDistribRsp = queryWithAsserts("q", "id:19", "fl", "id,*a_sS", "sort", "payload asc");
  QueryResponse rsp = queryWithAsserts("q", "id:19", "fl", "id,*a_sS", "sort", "payload asc", "distrib.singlePass", "true");

  assertFieldValues(nonDistribRsp.getResults(), "id", "19");
  assertFieldValues(rsp.getResults(), "id", "19");

  nonDistribRsp = queryWithAsserts("q", "id:19", "fl", "id,dynamic_s,cat*", "sort", "payload asc");
  rsp = queryWithAsserts("q", "id:19", "fl", "id,dynamic_s,cat*", "sort", "payload asc", "distrib.singlePass", "true");
  assertFieldValues(nonDistribRsp.getResults(), "id", "19");
  assertFieldValues(rsp.getResults(), "id", "19");

  queryWithAsserts("q", "id:19", "fl", "id,*a_sS", "sort", "payload asc", "distrib.singlePass", "true");
  queryWithAsserts("q", "id:19", "fl", "id,dynamic_s,cat*", "sort", "payload asc", "distrib.singlePass", "true");

  // fl=*
  queryWithAsserts("q", "*:*", "fl", "*", "sort", "payload desc", ShardParams.DISTRIB_SINGLE_PASS, "true");
  queryWithAsserts("q", "*:*", "fl", "*", "sort", "payload desc");

  // fl=*,score
  queryWithAsserts("q", "*:*", "fl", "*,score", "sort", "payload desc", ShardParams.DISTRIB_SINGLE_PASS, "true");
  queryWithAsserts("q", "*:*", "fl", "*,score", "sort", "payload desc");
}
 
Example #22
Source File: HttpShardHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private boolean canShortCircuit(String[] slices, boolean onlyNrtReplicas, SolrParams params, CloudDescriptor cloudDescriptor) {
  // Are we hosting the shard that this request is for, and are we active? If so, then handle it ourselves
  // and make it a non-distributed request.
  String ourSlice = cloudDescriptor.getShardId();
  String ourCollection = cloudDescriptor.getCollectionName();
  // Some requests may only be fulfilled by replicas of type Replica.Type.NRT
  if (slices.length == 1 && slices[0] != null
      && (slices[0].equals(ourSlice) || slices[0].equals(ourCollection + "_" + ourSlice))  // handle the <collection>_<slice> format
      && cloudDescriptor.getLastPublished() == Replica.State.ACTIVE
      && (!onlyNrtReplicas || cloudDescriptor.getReplicaType() == Replica.Type.NRT)) {
    boolean shortCircuit = params.getBool("shortCircuit", true);       // currently just a debugging parameter to check distrib search on a single node

    String targetHandler = params.get(ShardParams.SHARDS_QT);
    shortCircuit = shortCircuit && targetHandler == null;             // if a different handler is specified, don't short-circuit

    return shortCircuit;
  }
  return false;
}
 
Example #23
Source File: TermsComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected void checkShardsWhitelist(final ResponseBuilder rb, final List<String> lst) {
  final List<String> urls = new LinkedList<String>();
  for (final String ele : lst) {
    urls.addAll(StrUtils.splitSmart(ele, '|'));
  }
  
  if (whitelistHostChecker.isWhitelistHostCheckingEnabled() && rb.req.getCore().getCoreContainer().getZkController() == null && !whitelistHostChecker.hasExplicitWhitelist()) {
    throw new SolrException(ErrorCode.FORBIDDEN, "TermsComponent "+HttpShardHandlerFactory.INIT_SHARDS_WHITELIST
        +" not configured but required when using the '"+ShardParams.SHARDS+"' parameter with the TermsComponent."
        +HttpShardHandlerFactory.SET_SOLR_DISABLE_SHARDS_WHITELIST_CLUE);
  } else {
    ClusterState cs = null;
    if (rb.req.getCore().getCoreContainer().getZkController() != null) {
      cs = rb.req.getCore().getCoreContainer().getZkController().getClusterState();
    }
    whitelistHostChecker.checkWhitelist(cs, urls.toString(), urls);
  }
}
 
Example #24
Source File: CloudReplicaSource.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked", "rawtypes"})
private void withClusterState(Builder builder, SolrParams params) {
  ClusterState clusterState = builder.zkStateReader.getClusterState();
  String shardKeys = params.get(ShardParams._ROUTE_);

  // This will be the complete list of slices we need to query for this request.
  Map<String, Slice> sliceMap = new HashMap<>();

  // we need to find out what collections this request is for.

  // A comma-separated list of specified collections.
  // Eg: "collection1,collection2,collection3"
  String collections = params.get("collection");
  if (collections != null) {
    // If there were one or more collections specified in the query, split
    // each parameter and store as a separate member of a List.
    List<String> collectionList = StrUtils.splitSmart(collections, ",",
        true);
    // In turn, retrieve the slices that cover each collection from the
    // cloud state and add them to the Map 'slices'.
    for (String collectionName : collectionList) {
      // The original code produced <collection-name>_<shard-name> when the collections
      // parameter was specified (see ClientUtils.appendMap)
      // Is this necessary if ony one collection is specified?
      // i.e. should we change multiCollection to collectionList.size() > 1?
      addSlices(sliceMap, clusterState, params, collectionName, shardKeys, true);
    }
  } else {
    // just this collection
    addSlices(sliceMap, clusterState, params, builder.collection, shardKeys, false);
  }

  this.slices = sliceMap.keySet().toArray(new String[sliceMap.size()]);
  this.replicas = new List[slices.length];
  for (int i = 0; i < slices.length; i++) {
    String sliceName = slices[i];
    replicas[i] = findReplicas(builder, null, clusterState, sliceMap.get(sliceName));
  }
}
 
Example #25
Source File: ConsistencyComponent.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
@Override
public void process(ResponseBuilder rb) throws IOException
{
    SolrQueryRequest req = rb.req;
    AlfrescoCoreAdminHandler adminHandler = (AlfrescoCoreAdminHandler)
                req.getCore().
                getCoreContainer().
                getMultiCoreHandler();

    boolean isShard = rb.req.getParams().getBool(ShardParams.IS_SHARD, false);
    MetadataTracker metaTrkr = adminHandler.getTrackerRegistry().getTrackerForCore(req.getCore().getName(), MetadataTracker.class);
    if(metaTrkr != null && !isShard)
    {
        TrackerState metadataTrkrState = metaTrkr.getTrackerState();

        long lastIndexedTx = metadataTrkrState.getLastIndexedTxId();
        long lastIndexTxCommitTime = metadataTrkrState.getLastIndexedTxCommitTime();
        long lastTxIdOnServer = metadataTrkrState.getLastTxIdOnServer();
        long transactionsToDo = lastTxIdOnServer - lastIndexedTx;
        if (transactionsToDo < 0)
        {
            transactionsToDo = 0;
        }
        rb.rsp.add("lastIndexedTx", lastIndexedTx);
        rb.rsp.add("lastIndexedTxTime", lastIndexTxCommitTime);
        rb.rsp.add("txRemaining", transactionsToDo);
    }
}
 
Example #26
Source File: StreamingTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testTupleStreamGetShardsPreference() throws Exception {
  StreamContext streamContext = new StreamContext();
  streamContext.setSolrClientCache(new SolrClientCache());
  streamContext.setRequestReplicaListTransformerGenerator(new RequestReplicaListTransformerGenerator(ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":TLOG", null, null, null));

  streamContext.setRequestParams(mapParams(ShardParams.SHARDS_PREFERENCE, ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":nrt"));

  try {
    ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader();
    List<String> strings = zkStateReader.aliasesManager.getAliases().resolveAliases(MULTI_REPLICA_COLLECTIONORALIAS);
    String collName = strings.size() > 0 ? strings.get(0) : MULTI_REPLICA_COLLECTIONORALIAS;
    Map<String, String> replicaTypeMap = mapReplicasToReplicaType(zkStateReader.getClusterState().getCollectionOrNull(collName));

    // Test from extra params
    SolrParams sParams = mapParams("q", "*:*", ShardParams.SHARDS_PREFERENCE, ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":pull");
    testTupleStreamSorting(streamContext, sParams, "PULL", replicaTypeMap);

    // Test defaults from streamContext.getParams()
    testTupleStreamSorting(streamContext, new ModifiableSolrParams(), "NRT", replicaTypeMap);

    // Test defaults from the RLTG
    streamContext.setRequestParams(new ModifiableSolrParams());
    testTupleStreamSorting(streamContext, new ModifiableSolrParams(), "TLOG", replicaTypeMap);
  } finally {
    streamContext.getSolrClientCache().close();
  }
}
 
Example #27
Source File: CloudReplicaSource.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private CloudReplicaSource(Builder builder) {
  final String shards = builder.params.get(ShardParams.SHARDS);
  if (shards != null) {
    withShardsParam(builder, shards);
  } else {
    withClusterState(builder, builder.params);
  }
}
 
Example #28
Source File: CloudHttp2SolrClientTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void queryReplicaType(CloudHttp2SolrClient cloudClient,
                              Replica.Type typeToQuery,
                              String collectionName)
    throws Exception
{
  SolrQuery qRequest = new SolrQuery("*:*");

  ModifiableSolrParams qParams = new ModifiableSolrParams();
  qParams.add(ShardParams.SHARDS_PREFERENCE, ShardParams.SHARDS_PREFERENCE_REPLICA_TYPE + ":" + typeToQuery.toString());
  qParams.add(ShardParams.SHARDS_INFO, "true");
  qRequest.add(qParams);

  Map<String, String> replicaTypeToReplicas = mapReplicasToReplicaType(getCollectionState(collectionName));

  QueryResponse qResponse = cloudClient.query(collectionName, qRequest);

  Object shardsInfo = qResponse.getResponse().get(ShardParams.SHARDS_INFO);
  assertNotNull("Unable to obtain "+ShardParams.SHARDS_INFO, shardsInfo);

  // Iterate over shards-info and check what cores responded
  SimpleOrderedMap<?> shardsInfoMap = (SimpleOrderedMap<?>)shardsInfo;
  @SuppressWarnings({"unchecked"})
  Iterator<Map.Entry<String, ?>> itr = shardsInfoMap.asMap(100).entrySet().iterator();
  List<String> shardAddresses = new ArrayList<String>();
  while (itr.hasNext()) {
    Map.Entry<String, ?> e = itr.next();
    assertTrue("Did not find map-type value in "+ShardParams.SHARDS_INFO, e.getValue() instanceof Map);
    String shardAddress = (String)((Map)e.getValue()).get("shardAddress");
    if (shardAddress.endsWith("/")) {
      shardAddress = shardAddress.substring(0, shardAddress.length() - 1);
    }
    assertNotNull(ShardParams.SHARDS_INFO+" did not return 'shardAddress' parameter", shardAddress);
    shardAddresses.add(shardAddress);
  }
  assertEquals("Shard addresses must be of size 1, since there is only 1 shard in the collection", 1, shardAddresses.size());

  assertEquals("Make sure that the replica queried was the replicaType desired", typeToQuery.toString().toUpperCase(Locale.ROOT), replicaTypeToReplicas.get(shardAddresses.get(0)).toUpperCase(Locale.ROOT));
}
 
Example #29
Source File: SearchHandler.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private ShardHandler getAndPrepShardHandler(SolrQueryRequest req, ResponseBuilder rb) {
  ShardHandler shardHandler = null;

  CoreContainer cc = req.getCore().getCoreContainer();
  boolean isZkAware = cc.isZooKeeperAware();
  rb.isDistrib = req.getParams().getBool(DISTRIB, isZkAware);
  if (!rb.isDistrib) {
    // for back compat, a shards param with URLs like localhost:8983/solr will mean that this
    // search is distributed.
    final String shards = req.getParams().get(ShardParams.SHARDS);
    rb.isDistrib = ((shards != null) && (shards.indexOf('/') > 0));
  }
  
  if (rb.isDistrib) {
    shardHandler = shardHandlerFactory.getShardHandler();
    shardHandler.prepDistributed(rb);
    if (!rb.isDistrib) {
      shardHandler = null; // request is not distributed after all and so the shard handler is not needed
    }
  }

  if (isZkAware) {
    String shardsTolerant = req.getParams().get(ShardParams.SHARDS_TOLERANT);
    boolean requireZkConnected = shardsTolerant != null && shardsTolerant.equals(ShardParams.REQUIRE_ZK_CONNECTED);
    ZkController zkController = cc.getZkController();
    boolean zkConnected = zkController != null && ! zkController.getZkClient().getConnectionManager().isLikelyExpired();
    if (requireZkConnected && false == zkConnected) {
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "ZooKeeper is not connected");
    } else {
      NamedList<Object> headers = rb.rsp.getResponseHeader();
      if (headers != null) {
        headers.add("zkConnected", zkConnected);
      }
    }
  }

  return shardHandler;
}
 
Example #30
Source File: StatsComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
@SuppressWarnings({"unchecked", "rawtypes"})
public void handleResponses(ResponseBuilder rb, ShardRequest sreq) {
  if (!rb.doStats || (sreq.purpose & ShardRequest.PURPOSE_GET_STATS) == 0) return;

  Map<String, StatsValues> allStatsValues = rb._statsInfo.getAggregateStatsValues();

  for (ShardResponse srsp : sreq.responses) {
    NamedList stats = null;
    try {
      stats = (NamedList<NamedList<NamedList<?>>>)
          srsp.getSolrResponse().getResponse().get("stats");
    } catch (Exception e) {
      if (ShardParams.getShardsTolerantAsBool(rb.req.getParams())) {
        continue; // looks like a shard did not return anything
      }
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
          "Unable to read stats info for shard: " + srsp.getShard(), e);
    }

    NamedList stats_fields = unwrapStats(stats);
    if (stats_fields != null) {
      for (int i = 0; i < stats_fields.size(); i++) {
        String key = stats_fields.getName(i);
        StatsValues stv = allStatsValues.get(key);
        @SuppressWarnings({"rawtypes"})
        NamedList shardStv = (NamedList) stats_fields.get(key);
        stv.accumulate(shardStv);
      }
    }
  }
}