Java Code Examples for org.apache.solr.client.solrj.SolrQuery#setParam()

private SolrQuery runCommonQuery(RawQuery query, KeyInformation.IndexRetriever information, BaseTransaction tx,
                                 String collection, String keyIdField) throws BackendException {
    final SolrQuery solrQuery = new SolrQuery(query.getQuery())
            .addField(keyIdField)
            .setIncludeScore(true)
            .setStart(query.getOffset());
    if (query.hasLimit()) {
        solrQuery.setRows(Math.min(query.getLimit(), batchSize));
    } else {
        solrQuery.setRows(batchSize);
    }
    if (!query.getOrders().isEmpty()) {
        addOrderToQuery(solrQuery, query.getOrders());
    }

    for(final Parameter parameter: query.getParameters()) {
        if (parameter.value() instanceof String[]) {
            solrQuery.setParam(parameter.key(), (String[]) parameter.value());
        } else if (parameter.value() instanceof String) {
            solrQuery.setParam(parameter.key(), (String) parameter.value());
        }
    }
    return solrQuery;
}
 

/**
 * Convert given Query into a SolrQuery executable via
 * {@link org.apache.solr.client.solrj.SolrClient}
 *
 * @param query
 * @return
 */
@Override
public final SolrQuery doConstructSolrQuery(SolrDataQuery query) {
	Assert.notNull(query, "Cannot construct solrQuery from null value.");
	Assert.notNull(query.getCriteria(), "Query has to have a criteria.");

	SolrQuery solrQuery = new SolrQuery();
	solrQuery.setParam(CommonParams.Q, getQueryString(query));
	if (query instanceof Query) {
		processQueryOptions(solrQuery, (Query) query);
	}
	if (query instanceof FacetQuery) {
		processFacetOptions(solrQuery, (FacetQuery) query);
	}
	if (query instanceof HighlightQuery) {
		processHighlightOptions(solrQuery, (HighlightQuery) query);
	}
	return solrQuery;
}
 

protected long getVersionFromIndex(Replica replica, String docId) throws IOException, SolrServerException {
  Long vers = null;
  String queryStr = (docId != null) ? "id:" + docId : "_version_:[0 TO *]";
  SolrQuery query = new SolrQuery(queryStr);
  query.setRows(1);
  query.setFields("id", "_version_");
  query.addSort(new SolrQuery.SortClause("_version_", SolrQuery.ORDER.desc));
  query.setParam("distrib", false);

  try (SolrClient client = getHttpSolrClient(replica.getCoreUrl())) {
    QueryResponse qr = client.query(query);
    SolrDocumentList hits = qr.getResults();
    if (hits.isEmpty())
      fail("No results returned from query: "+query);

    vers = (Long) hits.get(0).getFirstValue("_version_");
  }

  if (vers == null)
    fail("Failed to get version using query " + query + " from " + replica.getCoreUrl());

  return vers.longValue();
}
 

@Test
public void testRerankedExplain() throws Exception {
  loadModel("linear2", LinearModel.class.getName(), new String[] {
      "constant1", "constant2", "pop"},
      "{\"weights\":{\"pop\":1.0,\"constant1\":1.5,\"constant2\":3.5}}");

  final SolrQuery query = new SolrQuery();
  query.setQuery("title:bloomberg");
  query.setParam("debugQuery", "on");
  query.add("rows", "2");
  query.add("rq", "{!ltr reRankDocs=2 model=linear2}");
  query.add("fl", "*,score");

  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/9=='\n13.5 = LinearModel(name=linear2,featureWeights=[constant1=1.5,constant2=3.5,pop=1.0]) model applied to features, sum of:\n  1.5 = prod of:\n    1.5 = weight on feature\n    1.0 = ValueFeature [name=constant1, params={value=1}]\n  7.0 = prod of:\n    3.5 = weight on feature\n    2.0 = ValueFeature [name=constant2, params={value=2}]\n  5.0 = prod of:\n    1.0 = weight on feature\n    5.0 = FieldValueFeature [name=pop, params={field=popularity}]\n'");
}
 

private void doTestMultipleAdditiveTreesExplain() throws Exception {

    final SolrQuery query = new SolrQuery();
    query.setQuery("*:*");
    query.add("fl", "*,score,[fv]");
    query.add("rows", "3");

    query.add("rq",
        "{!ltr reRankDocs=3 model=multipleadditivetreesmodel efi.user_query=w3}");

    // test out the explain feature, make sure it returns something
    query.setParam("debugQuery", "on");

    String qryResult = JQ("/query" + query.toQueryString());
    qryResult = qryResult.replaceAll("\n", " ");

    assertThat(qryResult, containsString("\"debug\":{"));
    qryResult = qryResult.substring(qryResult.indexOf("debug"));

    assertThat(qryResult, containsString("\"explain\":{"));
    qryResult = qryResult.substring(qryResult.indexOf("explain"));

    assertThat(qryResult, containsString("multipleadditivetreesmodel"));
    assertThat(qryResult, containsString(MultipleAdditiveTreesModel.class.getSimpleName()));

    assertThat(qryResult, containsString("-100.0 = tree 0"));
    assertThat(qryResult, containsString("50.0 = tree 0"));
    assertThat(qryResult, containsString("-20.0 = tree 1"));
    assertThat(qryResult, containsString("'matchedTitle':1.0 > 0.5"));
    assertThat(qryResult, containsString("'matchedTitle':0.0 <= 0.5"));

    assertThat(qryResult, containsString(" Go Right "));
    assertThat(qryResult, containsString(" Go Left "));
  }
 

@Test
public void multipleAdditiveTreesScoreExplainMissingEfiFeatureShouldReturnDefaultScore() throws Exception {
  loadFeatures("external_features_for_sparse_processing.json");
  loadModels("multipleadditivetreesmodel_external_binary_features.json");

  SolrQuery query = new SolrQuery();
  query.setQuery("title:bloomberg");
  query.setParam("debugQuery", "on");
  query.add("rows", "4");
  query.add("rq", "{!ltr reRankDocs=4 model=external_model_binary_feature efi.user_device_tablet=1}");
  query.add("fl", "*,score");

  final String tree1 = "(weight=1.0,root=(feature=user_device_smartphone,threshold=0.5,left=0.0,right=50.0))";
  final String tree2 = "(weight=1.0,root=(feature=user_device_tablet,threshold=0.5,left=0.0,right=65.0))";
  final String trees = "["+tree1+","+tree2+"]";

  query.add("wt", "json");
  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/7=='\n" +
          "65.0 = MultipleAdditiveTreesModel(name=external_model_binary_feature,trees="+trees+") model applied to features, sum of:\n" +
          "  0.0 = tree 0 | \\'user_device_smartphone\\':0.0 <= 0.500001, Go Left | val: 0.0\n" +
          "  65.0 = tree 1 | \\'user_device_tablet\\':1.0 > 0.500001, Go Right | val: 65.0\n'}");
  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/9=='\n" +
          "65.0 = MultipleAdditiveTreesModel(name=external_model_binary_feature,trees="+trees+") model applied to features, sum of:\n" +
          "  0.0 = tree 0 | \\'user_device_smartphone\\':0.0 <= 0.500001, Go Left | val: 0.0\n" +
          "  65.0 = tree 1 | \\'user_device_tablet\\':1.0 > 0.500001, Go Right | val: 65.0\n'}");
}
 

@Test
public void LinearScoreExplainMissingEfiFeatureShouldReturnDefaultScore() throws Exception {
  loadFeatures("features-linear-efi.json");
  loadModels("linear-model-efi.json");

  SolrQuery query = new SolrQuery();
  query.setQuery("title:bloomberg");
  query.setParam("debugQuery", "on");
  query.add("rows", "4");
  query.add("rq", "{!ltr reRankDocs=4 model=linear-efi}");
  query.add("fl", "*,score");
  query.add("wt", "xml");

  final String linearModelEfiString = "LinearModel(name=linear-efi,featureWeights=["
    + "sampleConstant=1.0,"
    + "search_number_of_nights=2.0])";

  query.remove("wt");
  query.add("wt", "json");
  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/7=='\n5.0 = "+linearModelEfiString+" model applied to features, sum of:\n  5.0 = prod of:\n    1.0 = weight on feature\n    5.0 = ValueFeature [name=sampleConstant, params={value=5}]\n" +
          "  0.0 = prod of:\n" +
          "    2.0 = weight on feature\n" +
          "    0.0 = The feature has no value\n'}");
  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/9=='\n5.0 = "+linearModelEfiString+" model applied to features, sum of:\n  5.0 = prod of:\n    1.0 = weight on feature\n    5.0 = ValueFeature [name=sampleConstant, params={value=5}]\n" +
          "  0.0 = prod of:\n" +
          "    2.0 = weight on feature\n" +
          "    0.0 = The feature has no value\n'}");
}
 

@Test
public void testRerankedExplainSameBetweenDifferentDocsWithSameFeatures() throws Exception {
  loadFeatures("features-linear.json");
  loadModels("linear-model.json");

  final SolrQuery query = new SolrQuery();
  query.setQuery("title:bloomberg");
  query.setParam("debugQuery", "on");
  query.add("rows", "4");
  query.add("rq", "{!ltr reRankDocs=4 model=6029760550880411648}");
  query.add("fl", "*,score");
  query.add("wt", "json");
  final String expectedExplainNormalizer = "normalized using MinMaxNormalizer(min=0.0,max=10.0)";
  final String expectedExplain = "\n3.5116758 = LinearModel(name=6029760550880411648,featureWeights=["
      + "title=0.0,"
      + "description=0.1,"
      + "keywords=0.2,"
      + "popularity=0.3,"
      + "text=0.4,"
      + "queryIntentPerson=0.1231231,"
      + "queryIntentCompany=0.12121211"
      + "]) model applied to features, sum of:\n  0.0 = prod of:\n    0.0 = weight on feature\n    1.0 = ValueFeature [name=title, params={value=1}]\n  0.2 = prod of:\n    0.1 = weight on feature\n    2.0 = ValueFeature [name=description, params={value=2}]\n  0.4 = prod of:\n    0.2 = weight on feature\n    2.0 = ValueFeature [name=keywords, params={value=2}]\n  0.09 = prod of:\n    0.3 = weight on feature\n    0.3 = "+expectedExplainNormalizer+"\n      3.0 = ValueFeature [name=popularity, params={value=3}]\n  1.6 = prod of:\n    0.4 = weight on feature\n    4.0 = ValueFeature [name=text, params={value=4}]\n  0.6156155 = prod of:\n    0.1231231 = weight on feature\n    5.0 = ValueFeature [name=queryIntentPerson, params={value=5}]\n  0.60606056 = prod of:\n    0.12121211 = weight on feature\n    5.0 = ValueFeature [name=queryIntentCompany, params={value=5}]\n";

  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/7=='"+expectedExplain+"'}");
  assertJQ(
      "/query" + query.toQueryString(),
      "/debug/explain/9=='"+expectedExplain+"'}");
}
 

/**
 * Set cookies via interceptor
 * Change the request via an interceptor
 * Ensure cookies are actually set and that request is actually changed
 */
@Test
public void testInterceptors() {
  DebugServlet.clear();
  HttpClientUtil.addRequestInterceptor(changeRequestInterceptor);
  HttpClientUtil.addRequestInterceptor(cookieSettingRequestInterceptor);    

  final String clientUrl = jetty.getBaseUrl().toString() + "/debug/foo";
  try(HttpSolrClient server = getHttpSolrClient(clientUrl)) {

    SolrQuery q = new SolrQuery("foo");
    q.setParam("a", "\u1234");
    expectThrows(Exception.class, () -> server.query(q, random().nextBoolean()?METHOD.POST:METHOD.GET));

    // Assert cookies from UseContextCallback 
    assertNotNull(DebugServlet.cookies);
    boolean foundCookie = false;
    for (javax.servlet.http.Cookie cookie : DebugServlet.cookies) {
      if (cookieName.equals(cookie.getName())
          && cookieValue.equals(cookie.getValue())) {
        foundCookie = true;
        break;
      }
    }
    assertTrue(foundCookie);

    // Assert request changes by ChangeRequestCallback
    assertEquals("\u1234", DebugServlet.parameters.get("a")[0]);
    assertEquals("\u4321", DebugServlet.parameters.get("b")[0]);

  } catch (IOException ex) {
    throw new RuntimeException(ex);
  } finally {
    HttpClientUtil.removeRequestInterceptor(changeRequestInterceptor);
    HttpClientUtil.removeRequestInterceptor(cookieSettingRequestInterceptor);    
  }
}
 

@SuppressWarnings("rawtypes")
public static SolrQuery getSolrQuery(MCRQuery query, Document input, HttpServletRequest request) {
    int rows = query.getNumPerPage();
    List<String> returnFields = query.getReturnFields();
    MCRCondition condition = query.getCondition();
    HashMap<String, List<MCRCondition>> table;

    if (condition instanceof MCRSetCondition) {
        table = MCRConditionTransformer.groupConditionsByIndex((MCRSetCondition) condition);
    } else {
        // if there is only one condition its no set condition. we don't need to group
        LOGGER.warn("Condition is not SetCondition.");
        table = new HashMap<>();

        ArrayList<MCRCondition> conditionList = new ArrayList<>();
        conditionList.add(condition);

        table.put("metadata", conditionList);

    }

    boolean booleanAnd = !(condition instanceof MCROrCondition<?>);
    SolrQuery mergedSolrQuery = MCRConditionTransformer.buildMergedSolrQuery(query.getSortBy(), false, booleanAnd,
        table, rows, returnFields);
    String mask = input.getRootElement().getAttributeValue("mask");
    if (mask != null) {
        mergedSolrQuery.setParam("mask", mask);
        mergedSolrQuery.setParam("_session", request.getParameter("_session"));
    }
    return mergedSolrQuery;
}
 

/**
 * The method computes the number of sitemap files. If we have less than
 * <em>numberOfURLs</em> URLs and only one MyCoRe type the sitemap_google.xml
 * contained all URLs. Otherwise it split the sitemap in an sitemap_google.xml
 * index file and a lot of sitemap_google_xxxx.xml URL files.
 *
 * @return the number of files, one for a single sitemap_google.xml file, more than
 *         one for the index and all parts.
 *
 */
protected int checkSitemapFile() throws IOException {
    int number = 0;
    QueryResponse response;
    SolrQuery query = new SolrQuery();
    query.setQuery(SOLR_QUERY);
    query.setRows(Integer.MAX_VALUE);
    query.setParam("fl", "id,modified");

    try {
        response = MCRSolrClientFactory.getMainSolrClient().query(query);
        objidlist = response.getResults().stream().map((document) -> {
            String id = (String) document.getFieldValue("id");
            Date modified = (Date) document.getFieldValue("modified");

            return new MCRObjectIDDateImpl(modified, id);
        }).collect(Collectors.toList());

    } catch (SolrServerException e) {
        LOGGER.error(e);
    }
    number = objidlist.size() / numberOfURLs;
    if (objidlist.size() % numberOfURLs != 0) {
        number++;
    }
    return number;
}
 

/**
 * Append field list to {@link org.apache.solr.client.solrj.SolrQuery}
 *
 * @param solrQuery
 * @param fields
 */
protected void appendProjectionOnFields(SolrQuery solrQuery, List<Field> fields) {
	if (CollectionUtils.isEmpty(fields)) {
		return;
	}
	List<String> solrReadableFields = new ArrayList<String>();
	for (Field field : fields) {
		if (field instanceof CalculatedField) {
			solrReadableFields.add(createCalculatedFieldFragment((CalculatedField) field));
		} else {
			solrReadableFields.add(field.getName());
		}
	}
	solrQuery.setParam(CommonParams.FL, StringUtils.join(solrReadableFields, ","));
}
 

@Override
public SolrQuery doConstructSolrQuery(TermsQuery query) {
	Assert.notNull(query, "Cannot construct solrQuery from null value.");

	SolrQuery solrQuery = new SolrQuery();
	String queryString = getQueryString(query);
	if (StringUtils.hasText(queryString)) {
		solrQuery.setParam(CommonParams.Q, queryString);
	}
	appendTermsOptionsToSolrQuery(query.getTermsOptions(), solrQuery);
	processTermsFields(solrQuery, query);
	appendRequestHandler(solrQuery, query.getRequestHandler());
	return solrQuery;
}
 

@Override
public List<String> getSuggestions(String prefixString, String indexFieldName) {
    SolrClient solrClient = null;

    try {
        solrClient = Solr6Index.getSolrClient(); // get solr client using same settings as that of Janus Graph

        if (solrClient == null) {
            LOG.warn("The indexing system is not solr based. Suggestions feature will not be available.");

            return Collections.EMPTY_LIST;
        }

        SolrQuery solrQuery = new SolrQuery();

        solrQuery.setRequestHandler(Constants.TERMS_REQUEST_HANDLER)
                 .setParam(TERMS_PREFIX, prefixString)
                 .setParam(CommonParams.OMIT_HEADER, true);

        if (StringUtils.isNotEmpty(indexFieldName)) {
            solrQuery.setParam(TERMS_FIELD, indexFieldName);
        }

        QueryResponse queryResponse = solrClient.query(VERTEX_INDEX, solrQuery);
        TermsResponse termsResponse = queryResponse == null? null: queryResponse.getTermsResponse();

        if(termsResponse == null) {
            LOG.info("Received null for terms response. Will return no suggestions.");

            return Collections.EMPTY_LIST;
        }

        Map<String, TermFreq> termsMap = new HashMap<>();

        for (List<TermsResponse.Term> fieldTerms : termsResponse.getTermMap().values()) {
            for (TermsResponse.Term fieldTerm : fieldTerms) {
                TermFreq term = termsMap.get(fieldTerm.getTerm());

                if (term == null) {
                    term = new TermFreq(fieldTerm.getTerm(), fieldTerm.getFrequency());

                    termsMap.put(term.getTerm(), term);
                } else {
                    term.addFreq(fieldTerm.getFrequency());
                }
            }
        }

        return getTopTerms(termsMap);
    } catch (SolrServerException | IOException e) {
        String msg = String.format("Error encountered in generating the suggestions. Ignoring the error", e);

        LOG.error(msg);
    } finally {
        Solr6Index.releaseSolrClient(solrClient);
    }

    return Collections.EMPTY_LIST;
}
 

protected QueryResponse doQuery() {
  SolrEntityProcessor.this.queryString = context.getResolvedEntityAttribute(QUERY);
  if (SolrEntityProcessor.this.queryString == null) {
    throw new DataImportHandlerException(
        DataImportHandlerException.SEVERE,
        "SolrEntityProcessor: parameter 'query' is required"
    );
  }

  String rowsP = context.getResolvedEntityAttribute(CommonParams.ROWS);
  if (rowsP != null) {
    rows = Integer.parseInt(rowsP);
  }

  String sortParam = context.getResolvedEntityAttribute(CommonParams.SORT);
  
  String fqAsString = context.getResolvedEntityAttribute(CommonParams.FQ);
  if (fqAsString != null) {
    SolrEntityProcessor.this.filterQueries = fqAsString.split(",");
  }

  String fieldsAsString = context.getResolvedEntityAttribute(CommonParams.FL);
  if (fieldsAsString != null) {
    SolrEntityProcessor.this.fields = fieldsAsString.split(",");
  }
  SolrEntityProcessor.this.requestHandler = context.getResolvedEntityAttribute(CommonParams.QT);
 

  SolrQuery solrQuery = new SolrQuery(queryString);
  solrQuery.setRows(rows);
  
  if (sortParam!=null) {
    solrQuery.setParam(CommonParams.SORT, sortParam);
  }
  
  passNextPage(solrQuery);
  
  if (fields != null) {
    for (String field : fields) {
      solrQuery.addField(field);
    }
  }
  solrQuery.setRequestHandler(requestHandler);
  solrQuery.setFilterQueries(filterQueries);
  
  
  QueryResponse response = null;
  try {
    response = solrClient.query(solrQuery);
  } catch (SolrServerException | IOException | SolrException e) {
    if (ABORT.equals(onError)) {
      wrapAndThrow(SEVERE, e);
    } else if (SKIP.equals(onError)) {
      wrapAndThrow(DataImportHandlerException.SKIP_ROW, e);
    }
  }
  
  if (response != null) {
    SolrEntityProcessor.this.rowIterator = createNextPageIterator(response);
  }
  return response;
}
 

@Override
public ResultsList<Document> searchDocuments(String term, int start, int rows, List<String> additionalFields,
		List<String> filters) throws SearchEngineException {
	ResultsList<Document> results;

	try {
		SolrQuery query = new SolrQuery(term);
		query.setStart(start);
		query.setRows(rows);
		query.setRequestHandler(config.getDocumentRequestHandler());
		List<String> queryFields = new ArrayList<>(DEFAULT_SEARCH_FIELDS);
		if (additionalFields != null) {
			queryFields.addAll(additionalFields);
		}
		if (filters != null) {
			query.addFilterQuery(filters.toArray(new String[filters.size()]));
		}
		query.setParam(DisMaxParams.QF, queryFields.toArray(new String[queryFields.size()]));
		
		LOGGER.debug("Query: {}", query);

		QueryResponse response = server.query(query);
		List<Document> docs;
		long total = 0;
		
		if (response.getGroupResponse() != null) {
			docs = new ArrayList<>(rows);
			GroupResponse gResponse = response.getGroupResponse();
			for (GroupCommand gCommand : gResponse.getValues()) {
				total += gCommand.getNGroups();
				for (Group group : gCommand.getValues()) {
					docs.addAll(server.getBinder().getBeans(Document.class, group.getResult()));
				}
			}
		} else if (response.getResults().getNumFound() == 0) {
			docs = new ArrayList<>();
		} else {
			docs = response.getBeans(Document.class);
			total = response.getResults().getNumFound();
		}
		
		results = new ResultsList<>(docs, start, (start / rows), total);
	} catch (SolrServerException | IOException e) {
		throw new SearchEngineException(e);
	}

	return results;
}
 

@Override
public ResultsList<Document> searchDocuments(String term, int start, int rows, List<String> additionalFields,
		List<String> filters, FacetStyle facetStyle) throws SearchEngineException {
	ResultsList<Document> results = null;

	try {
		SolrQuery query = new SolrQuery(term);
		query.setStart(start);
		query.setRows(rows);
		query.setRequestHandler(config.getDocumentRequestHandler());
		List<String> queryFields = new ArrayList<>(DEFAULT_SEARCH_FIELDS);
		if (additionalFields != null) {
			queryFields.addAll(additionalFields);
		}
		if (filters != null) {
			query.addFilterQuery(filters.toArray(new String[0]));
		}
		query.setParam(DisMaxParams.QF, queryFields.toArray(new String[queryFields.size()]));
		
		if (facetStyle == FacetStyle.NONE) {
			query.addFacetField(config.getFacetFields().toArray(new String[config.getFacetFields().size()]));
		} else {
			// Add the facet tree params
			query.setFacet(true);
			query.setParam("facet.tree", true);
			query.setParam("facet.tree.field", buildFacetTreeQueryParameter(facetStyle));
		}
		
		LOGGER.debug("Query: {}", query);

		QueryResponse response = server.query(query);
		List<Document> docs;
		long total = 0;
		
		if (response.getGroupResponse() != null) {
			docs = new ArrayList<>(rows);
			GroupResponse gResponse = response.getGroupResponse();
			for (GroupCommand gCommand : gResponse.getValues()) {
				total += gCommand.getNGroups();
				for (Group group : gCommand.getValues()) {
					docs.addAll(server.getBinder().getBeans(Document.class, group.getResult()));
				}
			}
		} else if (response.getResults().getNumFound() == 0) {
			docs = new ArrayList<>();
		} else {
			docs = response.getBeans(Document.class);
			total = response.getResults().getNumFound();
		}
		
		results = new ResultsList<>(docs, start, (start / rows), total, extractFacets(response, facetStyle));
	} catch (SolrServerException e) {
		throw new SearchEngineException(e);
	}

	return results;
}