Java Code Examples for org.apache.solr.search.SolrIndexSearcher

The following examples show how to use org.apache.solr.search.SolrIndexSearcher. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
protected BitsFilter getACLFilter(String[] auths, String field, SolrIndexSearcher searcher) throws IOException
{
    HybridBitSet aclBits = getACLSet(auths, field, searcher);
    List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();
    List<FixedBitSet> bitSets = new ArrayList<FixedBitSet>(leaves.size());

    for(LeafReaderContext readerContext :  leaves)
    {
    	LeafReader reader = readerContext.reader();
        int maxDoc = reader.maxDoc();
        FixedBitSet bits = new FixedBitSet(maxDoc);
        bitSets.add(bits);

        NumericDocValues fieldValues = DocValuesCache.getNumericDocValues(QueryConstants.FIELD_ACLID, reader);
        if (fieldValues != null) {
            for (int i = 0; i < maxDoc; i++) {
                long aclID = fieldValues.get(i);
                if (aclBits.get(aclID)) {
                    bits.set(i);
                }
            }
        }
    }

    return new BitsFilter(bitSets);
}
 
Example 2
Source Project: liresolr   Source File: HashTermStatistics.java    License: GNU General Public License v2.0 6 votes vote down vote up
public static void addToStatistics(SolrIndexSearcher searcher, String field) throws IOException {
        // check if this field is already in the stats.
//        synchronized (instance) {
            if (termstats.get(field)!=null) return;
//        }
        // else add it to the stats.
        Terms terms = searcher.getSlowAtomicReader().terms(field);
        HashMap<String, Integer> term2docFreq = new HashMap<String, Integer>(1000);
        termstats.put(field, term2docFreq);
        if (terms!=null) {
            TermsEnum termsEnum = terms.iterator();
            BytesRef term;
            while ((term = termsEnum.next()) != null) {
                term2docFreq.put(term.utf8ToString(), termsEnum.docFreq());
            }
        }
    }
 
Example 3
Source Project: lucene-solr   Source File: SolrPluginUtils.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Generates an NamedList of Explanations for each item in a list of docs.
 *
 * @param query The Query you want explanations in the context of
 * @param docs The Documents you want explained relative that query
 */
public static NamedList<Explanation> getExplanations
  (Query query,
   DocList docs,
   SolrIndexSearcher searcher,
   IndexSchema schema) throws IOException {

  NamedList<Explanation> explainList = new SimpleOrderedMap<>();
  DocIterator iterator = docs.iterator();
  for (int i=0; i<docs.size(); i++) {
    int id = iterator.nextDoc();

    Document doc = searcher.doc(id);
    String strid = schema.printableUniqueKey(doc);

    explainList.add(strid, searcher.explain(query, id) );
  }
  return explainList;
}
 
Example 4
private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      SchemaField field = schema.getField(fieldName);
      if (field.stored() && field.getType() instanceof StrField ) {
        strFields.add( fieldName );
      }
    }
  }
    
  return strFields;
}
 
Example 5
public DelegatingCollector getFilterCollector(IndexSearcher searcher)
{
    String[] auths = authorities.substring(1).split(authorities.substring(0, 1));
    try
    {
        HybridBitSet denySet = getACLSet(auths, QueryConstants.FIELD_DENIED, (SolrIndexSearcher) searcher);
        if(denySet instanceof EmptyHybridBitSet)
        {
            return new AllAccessCollector();
        }
        else
        {
            return new AccessControlCollector(denySet);
        }
    }
    catch(Exception e)
    {
        throw new RuntimeException(e);
    }
}
 
Example 6
/**
 * @param doc SolrDocument to check
 * @param idField field where the id is stored
 * @param fieldType type of id field
 * @param filterQuery Query to filter by
 * @param searcher SolrIndexSearcher on which to apply the filter query
 * @returns the internal docid, or -1 if doc is not found or doesn't match filter
 */
private static int getFilteredInternalDocId(SolrDocument doc, SchemaField idField, FieldType fieldType,
      Query filterQuery, SolrIndexSearcher searcher) throws IOException {
  int docid = -1;
  Field f = (Field)doc.getFieldValue(idField.getName());
  String idStr = f.stringValue();
  BytesRef idBytes = new BytesRef();
  fieldType.readableToIndexed(idStr, idBytes);
  // get the internal document id
  long segAndId = searcher.lookupId(idBytes);

    // if docid is valid, run it through the filter
  if (segAndId >= 0) {
    int segid = (int) segAndId;
    AtomicReaderContext ctx = searcher.getTopReaderContext().leaves().get((int) (segAndId >> 32));
    docid = segid + ctx.docBase;
    Weight weight = filterQuery.createWeight(searcher);
    Scorer scorer = weight.scorer(ctx, null);
    if (scorer == null || segid != scorer.advance(segid)) {
      // filter doesn't match.
      docid = -1;
    }
  }
  return docid;
}
 
Example 7
Source Project: lucene-solr   Source File: SolrSuggester.java    License: Apache License 2.0 6 votes vote down vote up
/** Build the underlying Lucene Suggester */
public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
  log.info("SolrSuggester.build({})", name);

  dictionary = dictionaryFactory.create(core, searcher);
  try {
    lookup.build(dictionary);
  } catch (AlreadyClosedException e) {
    RuntimeException e2 = new SolrCoreState.CoreIsClosedException
        ("Suggester build has been interrupted by a core reload or shutdown.");
    e2.initCause(e);
    throw e2;
  }
  if (storeDir != null) {
    File target = getStoreFile();
    if(!lookup.store(new FileOutputStream(target))) {
      log.error("Store Lookup build failed");
    } else {
      if (log.isInfoEnabled()) {
        log.info("Stored suggest data to: {}", target.getAbsolutePath());
      }
    }
  }
}
 
Example 8
Source Project: BioSolr   Source File: FacetTreeGenerator.java    License: Apache License 2.0 6 votes vote down vote up
public List<SimpleOrderedMap<Object>> generateTree(ResponseBuilder rb, NamedList<Integer> facetValues) throws IOException {
	List<SimpleOrderedMap<Object>> retVal = null;
	
	// First get the searcher for the required collection
	RefCounted<SolrIndexSearcher> searcherRef = getSearcherReference(rb);
	
	try {
		// Build the facet tree(s)
		Collection<TreeFacetField> fTrees = treeBuilder.processFacetTree(searcherRef.get(), extractFacetValues(facetValues));
		LOGGER.debug("Extracted {} facet trees", fTrees.size());
		
		if (pruner != null) {
			// Prune the trees
			fTrees = pruner.prune(fTrees);
		}

		// Convert the trees into a SimpleOrderedMap
		retVal = convertTreeFacetFields(fTrees);
	} finally {
		// Make sure the search ref count is decreased
		searcherRef.decref();
	}
	
	return retVal;
}
 
Example 9
Source Project: liresolr   Source File: LireRequestHandler.java    License: GNU General Public License v2.0 6 votes vote down vote up
/**
 * Returns a random set of documents from the index. Mainly for testing purposes.
 *
 * @param req
 * @param rsp
 * @throws IOException
 */
private void handleRandomSearch(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
    SolrIndexSearcher searcher = req.getSearcher();
    Query query = new MatchAllDocsQuery();
    DocList docList = searcher.getDocList(query, getFilterQueries(req), Sort.RELEVANCE, 0, numberOfCandidateResults, 0);
    int paramRows = Math.min(req.getParams().getInt("rows", defaultNumberOfResults), docList.size());
    if (docList.size() < 1) {
        rsp.add("Error", "No documents in index");
    } else {
        LinkedList list = new LinkedList();
        while (list.size() < paramRows) {
            DocList auxList = docList.subset((int) (Math.random() * docList.size()), 1);
            Document doc = null;
            for (DocIterator it = auxList.iterator(); it.hasNext(); ) {
                doc = searcher.doc(it.nextDoc());
            }
            if (!list.contains(doc)) {
                list.add(doc);
            }
        }
        rsp.addResponse(list);
    }
}
 
Example 10
Source Project: lucene-solr   Source File: RandomTestDictionaryFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public RandomTestDictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if(params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  String name = (String)params.get(CommonParams.NAME);
  if (name == null) { // Shouldn't happen since this is the component name
    throw new IllegalArgumentException(CommonParams.NAME + " is a mandatory parameter");
  }
  long maxItems = DEFAULT_MAX_ITEMS;
  Object specifiedMaxItems = params.get(RAND_DICT_MAX_ITEMS);
  if (specifiedMaxItems != null) {
    maxItems = Long.parseLong(specifiedMaxItems.toString());
  }
  return new RandomTestDictionary(name, maxItems);
}
 
Example 11
Source Project: BioSolr   Source File: FacetTreeGenerator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Get a reference to the searcher for the required collection. If the collection is
 * not the same as the search collection, we assume it is under the same Solr instance.
 * @param rb the response builder holding the facets.
 * @return a counted reference to the searcher.
 * @throws SolrException if the collection cannot be found.
 */
private RefCounted<SolrIndexSearcher> getSearcherReference(ResponseBuilder rb) throws SolrException {
	RefCounted<SolrIndexSearcher> searcherRef;
	
	SolrCore currentCore = rb.req.getCore();
	if (StringUtils.isBlank(collection)) {
		searcherRef = currentCore.getSearcher();
	} else {
		// Using an alternative core - find it
		SolrCore reqCore = currentCore.getCoreDescriptor().getCoreContainer().getCore(collection);
		if (reqCore == null) {
			throw new SolrException(ErrorCode.BAD_REQUEST, "Collection \"" + collection
					+ "\" cannot be found");
		}
		searcherRef = reqCore.getSearcher();
	}
	
	return searcherRef;
}
 
Example 12
Source Project: lucene-solr   Source File: FacetProcessor.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings("unused")
static DocSet getFieldMissing(SolrIndexSearcher searcher, DocSet docs, String fieldName) throws IOException {
  SchemaField sf = searcher.getSchema().getField(fieldName);
  DocSet hasVal = searcher.getDocSet(sf.getType().getRangeQuery(null, sf, null, null, false, false));
  DocSet answer = docs.andNot(hasVal);
  // hasVal.decref(); // OFF-HEAP
  return answer;
}
 
Example 13
Source Project: lucene-solr   Source File: QueryComponent.java    License: Apache License 2.0 5 votes vote down vote up
private void doProcessUngroupedSearch(ResponseBuilder rb, QueryCommand cmd, QueryResult result) throws IOException {

    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;

    SolrIndexSearcher searcher = req.getSearcher();

    try {
      searcher.search(result, cmd);
    } catch (FuzzyTermsEnum.FuzzyTermsException e) {
      throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
    }
    rb.setResult(result);

    ResultContext ctx = new BasicResultContext(rb);
    rsp.addResponse(ctx);
    rsp.getToLog().add("hits", rb.getResults()==null || rb.getResults().docList==null ? 0 : rb.getResults().docList.matches());

    if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
      if (null != rb.getNextCursorMark()) {
        rb.rsp.add(CursorMarkParams.CURSOR_MARK_NEXT,
                   rb.getNextCursorMark().getSerializedTotem());
      }
    }

    if(rb.mergeFieldHandler != null) {
      rb.mergeFieldHandler.handleMergeFields(rb, searcher);
    } else {
      doFieldSortValues(rb, searcher);
    }

    doPrefetch(rb);
  }
 
Example 14
Source Project: lucene-solr   Source File: QueryElevationComponent.java    License: Apache License 2.0 5 votes vote down vote up
@SuppressWarnings({"unchecked"})
public static IntIntHashMap getBoostDocs(SolrIndexSearcher indexSearcher, Set<BytesRef> boosted,
                                         @SuppressWarnings({"rawtypes"})Map context) throws IOException {

  IntIntHashMap boostDocs = null;

  if (boosted != null) {

    //First see if it's already in the request context. Could have been put there by another caller.
    if (context != null) {
      boostDocs = (IntIntHashMap) context.get(BOOSTED_DOCIDS);
      if (boostDocs != null) {
        return boostDocs;
      }
    }

    //Not in the context yet so load it.
    boostDocs = new IntIntHashMap(boosted.size()); // docId to boost
    int priority = boosted.size() + 1; // the corresponding priority for each boosted key (starts at this; decrements down)
    for (BytesRef uniqueKey : boosted) {
      priority--; // therefore first == bosted.size(); last will be 1
      long segAndId = indexSearcher.lookupId(uniqueKey); // higher 32 bits == segment ID, low 32 bits == doc ID
      if (segAndId == -1) { // not found
        continue;
      }
      int seg = (int) (segAndId >> 32);
      int localDocId = (int) segAndId;
      final IndexReaderContext indexReaderContext = indexSearcher.getTopReaderContext().children().get(seg);
      int docId = indexReaderContext.docBaseInParent + localDocId;
      boostDocs.put(docId, priority);
    }
    assert priority == 1; // the last priority (lowest)
  }

  if (context != null) {
    context.put(BOOSTED_DOCIDS, boostDocs);
  }

  return boostDocs;
}
 
Example 15
Source Project: lucene-solr   Source File: IndexFetcher.java    License: Apache License 2.0 5 votes vote down vote up
private void openNewSearcherAndUpdateCommitPoint() throws IOException {
  RefCounted<SolrIndexSearcher> searcher = null;
  IndexCommit commitPoint;
  // must get the latest solrCore object because the one we have might be closed because of a reload
  // todo stop keeping solrCore around
  SolrCore core = solrCore.getCoreContainer().getCore(solrCore.getName());
  try {
    @SuppressWarnings({"rawtypes"})
    Future[] waitSearcher = new Future[1];
    searcher = core.getSearcher(true, true, waitSearcher, true);
    if (waitSearcher[0] != null) {
      try {
        waitSearcher[0].get();
      } catch (InterruptedException | ExecutionException e) {
        SolrException.log(log, e);
      }
    }
    commitPoint = searcher.get().getIndexReader().getIndexCommit();
  } finally {
    if (searcher != null) {
      searcher.decref();
    }
    core.close();
  }

  // update the commit point in replication handler
  replicationHandler.indexCommitPoint = commitPoint;

}
 
Example 16
public static SolrOwnerSetScorer createOwnerSetScorer(Weight weight, LeafReaderContext context, SolrIndexSearcher searcher, String authorities) throws IOException
{
    
    DocSet authorityOwnedDocs = (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_OWNERLOOKUP_CACHE, authorities);
    
    if(authorityOwnedDocs == null)
    {
        // Split the authorities. The first character in the authorities String
        // specifies the separator, e.g. ",jbloggs,abeecher"
        String[] auths = authorities.substring(1).split(authorities.substring(0, 1));

        BooleanQuery.Builder bQuery = new BooleanQuery.Builder();
        for(String current : auths)
        {
            if (AuthorityType.getAuthorityType(current) == AuthorityType.USER)
            {
                bQuery.add(new TermQuery(new Term(QueryConstants.FIELD_OWNER, current)), Occur.SHOULD);
            }
        }
        
        WrappedQuery wrapped = new WrappedQuery(bQuery.build());
        wrapped.setCache(false);
        authorityOwnedDocs = searcher.getDocSet(wrapped);
    
        searcher.cacheInsert(CacheConstants.ALFRESCO_OWNERLOOKUP_CACHE, authorities, authorityOwnedDocs);
    }
    
    // TODO: Cache the final set? e.g. searcher.cacheInsert(authorities, authorityOwnedDocs)
    return new SolrOwnerSetScorer(weight, authorityOwnedDocs, context, searcher);
   
}
 
Example 17
Source Project: lucene-solr   Source File: ResponseLogComponent.java    License: Apache License 2.0 5 votes vote down vote up
protected void processIds(ResponseBuilder rb, DocList dl, IndexSchema schema,
    SolrIndexSearcher searcher) throws IOException {
  
  StringBuilder sb = new StringBuilder();

  Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
  for(DocIterator iter = dl.iterator(); iter.hasNext();) {

    sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields)))
      .append(',');
  }
  if (sb.length() > 0) {
    rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
  }  
}
 
Example 18
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScore) throws IOException
{
    if(!(searcher instanceof SolrIndexSearcher))
    {
        throw new IllegalStateException("Must have a SolrIndexSearcher");
    }
    return new SolrReaderSetQueryWeight((SolrIndexSearcher)searcher, this, authorities);
}
 
Example 19
Source Project: jate   Source File: AppWeirdness.java    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
public List<JATETerm> extract(SolrCore core, JATEProperties properties) throws JATEException {
		if (core.isClosed()) {
			core.open();
		}
		SolrIndexSearcher searcher = core.getSearcher().get();
//		try {
			this.freqFeatureBuilder = new FrequencyTermBasedFBMaster(searcher, properties, 0);
			this.freqFeature = (FrequencyTermBased) freqFeatureBuilder.build();

			FrequencyTermBasedFBMaster fwbb = new FrequencyTermBasedFBMaster(searcher, properties, 1);
			FrequencyTermBased fwb = (FrequencyTermBased) fwbb.build();

			TTFReferenceFeatureFileBuilder ftrb = new TTFReferenceFeatureFileBuilder(this.referenceFrequencyFilePath);
			FrequencyTermBased frb = ftrb.build();

			Weirdness weirdness = new Weirdness();
			weirdness.registerFeature(FrequencyTermBased.class.getName() + Weirdness.SUFFIX_WORD, fwb);
			weirdness.registerFeature(FrequencyTermBased.class.getName() + Weirdness.SUFFIX_REF, frb);

			List<String> candidates = new ArrayList<>(this.freqFeature.getMapTerm2TTF().keySet());

			filterByTTF(candidates);

			List<JATETerm> terms = weirdness.execute(candidates);
			terms = cutoff(terms);

			addAdditionalTermInfo(terms, searcher, properties.getSolrFieldNameJATENGramInfo(),
					properties.getSolrFieldNameID());
			return terms;
//		} finally {
//			try {
//				searcher.close();
//			} catch (IOException e) {
//				log.error(e.toString());
//			}
//		}
	}
 
Example 20
public AbstractAuthorityQueryWeight(SolrIndexSearcher searcher, boolean needsScores, Query query, String authTermName, String authTermText) throws IOException
{
	super(query);
    this.searcher = searcher;
    searcher.collectionStatistics(authTermName);
    final IndexReaderContext context = searcher.getTopReaderContext();
    final Term term = new Term(authTermName, authTermText);
    final TermContext termContext = TermContext.build(context, term);
    searcher.termStatistics(term, termContext);
    this.needsScores = needsScores;
}
 
Example 21
Source Project: BioSolr   Source File: AbstractFacetTreeBuilder.java    License: Apache License 2.0 5 votes vote down vote up
protected void checkFieldsInSchema(SolrIndexSearcher searcher, Collection<String> fields) throws SolrException {
	IndexSchema schema = searcher.getSchema();
	for (String field : fields) {
		SchemaField sField = schema.getField(field);
		if (sField == null) {
			throw new SolrException(ErrorCode.BAD_REQUEST, "\"" + field
					+ "\" is not in schema " + schema.getSchemaName());
		}
	}
}
 
Example 22
@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScore) throws IOException
{
    if(!(searcher instanceof SolrIndexSearcher))
    {
        throw new IllegalStateException("Must have a SolrIndexSearcher");
    }
    return new SolrOwnerSetQueryWeight((SolrIndexSearcher)searcher, this, authorities);
}
 
Example 23
Source Project: lucene-solr   Source File: SolrPluginUtils.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Pre-fetch documents into the index searcher's document cache.
 *
 * This is an entirely optional step which you might want to perform for
 * the following reasons:
 *
 * <ul>
 *     <li>Locates the document-retrieval costs in one spot, which helps
 *     detailed performance measurement</li>
 *
 *     <li>Determines a priori what fields will be needed to be fetched by
 *     various subtasks, like response writing and highlighting.  This
 *     minimizes the chance that many needed fields will be loaded lazily.
 *     (it is more efficient to load all the field we require normally).</li>
 * </ul>
 *
 * If lazy field loading is disabled, this method does nothing.
 */
public static void optimizePreFetchDocs(ResponseBuilder rb,
                                        DocList docs,
                                        Query query,
                                        SolrQueryRequest req,
                                        SolrQueryResponse res) throws IOException {
  SolrIndexSearcher searcher = req.getSearcher();
  if(!searcher.getDocFetcher().isLazyFieldLoadingEnabled()) {
    // nothing to do
    return;
  }

  ReturnFields returnFields = res.getReturnFields();
  if(returnFields.getLuceneFieldNames() != null) {
    Set<String> fieldFilter = returnFields.getLuceneFieldNames();

    if (rb.doHighlights) {
      // copy return fields list
      fieldFilter = new HashSet<>(fieldFilter);
      // add highlight fields

      SolrHighlighter highlighter = HighlightComponent.getHighlighter(req.getCore());
      for (String field: highlighter.getHighlightFields(query, req, null))
        fieldFilter.add(field);

      // fetch unique key if one exists.
      SchemaField keyField = searcher.getSchema().getUniqueKeyField();
      if(null != keyField)
        fieldFilter.add(keyField.getName());
    }

    // get documents
    DocIterator iter = docs.iterator();
    for (int i=0; i<docs.size(); i++) {
      searcher.doc(iter.nextDoc(), fieldFilter);
    }

  }

}
 
Example 24
Source Project: lucene-solr   Source File: AddBlockUpdateTest.java    License: Apache License 2.0 5 votes vote down vote up
protected void assertSingleParentOf(final SolrIndexSearcher searcher,
    final String childTerm, String parentExp) throws IOException {
  final TopDocs docs = searcher.search(join(childTerm), 10);
  assertEquals(1, docs.totalHits.value);
  final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent);
  assertEquals(parentExp, pAct);
}
 
Example 25
Source Project: SolRDF   Source File: Log.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Debugs the given query command.
 * 
 * @param cmd the query command.
 */
public void debugQuery(final QueryCommand cmd, final SolrIndexSearcher.QueryResult result) {
	if (logger.isDebugEnabled()) {
		final StringBuilder builder = new StringBuilder("*:*");
		for (final Query filter : cmd.getFilterList()) {
			builder.append(" & ").append(filter);
		}
		
		logger.debug(createMessage(
				MessageCatalog._00109_SOLR_QUERY, 
				builder.toString(), 
				result.getDocList().size(), 
				result.getDocList().matches()));
	}
}
 
Example 26
Source Project: lucene-solr   Source File: UnInvertedField.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Called for each term in the field being uninverted.
 * Collects {@link #maxTermCounts} for all bigTerms as well as storing them in {@link #bigTerms}.
 * @param te positioned at the current term.
 * @param termNum the ID/pointer/ordinal of the current term. Monotonically increasing between calls.
 */
@Override
protected void visitTerm(TermsEnum te, int termNum) throws IOException {

  if (termNum >= maxTermCounts.length) {
    // resize by doubling - for very large number of unique terms, expanding
    // by 4K and resultant GC will dominate uninvert times.  Resize at end if material
    int[] newMaxTermCounts = new int[ Math.min(Integer.MAX_VALUE-16, maxTermCounts.length*2) ];
    System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, termNum);
    maxTermCounts = newMaxTermCounts;
  }

  final BytesRef term = te.term();

  if (te.docFreq() > maxTermDocFreq) {
    Term t = new Term(field, term);  // this makes a deep copy of the term bytes
    TopTerm topTerm = new TopTerm();
    topTerm.term = t.bytes();
    topTerm.termNum = termNum;
    topTerm.termQuery = new TermQuery(t);

    bigTerms.put(topTerm.termNum, topTerm);

    if (deState == null) {
      deState = new SolrIndexSearcher.DocsEnumState();
      deState.fieldName = field;
      deState.liveDocs = searcher.getLiveDocsBits();
      deState.termsEnum = te;  // TODO: check for MultiTermsEnum in SolrIndexSearcher could now fail?
      deState.postingsEnum = postingsEnum;
      deState.minSetSizeCached = maxTermDocFreq;
    }

    postingsEnum = deState.postingsEnum;
    DocSet set = searcher.getDocSet(deState);
    maxTermCounts[termNum] = set.size();
  }
}
 
Example 27
Source Project: lucene-solr   Source File: FacetHeatmap.java    License: Apache License 2.0 5 votes vote down vote up
private Bits getTopAcceptDocs(DocSet docSet, SolrIndexSearcher searcher) throws IOException {
  if (docSet.size() == searcher.numDocs()) {
    return null; // means match everything (all live docs). This can speedup things a lot.
  } else if (docSet.size() == 0) {
    return new Bits.MatchNoBits(searcher.maxDoc()); // can speedup things a lot
  } else {
    return docSet.getBits();
  }
}
 
Example 28
private void buildSuggesterIndex(SolrSuggester suggester, SolrIndexSearcher newSearcher) {
  try {
    LOG.info("Building suggester index for: " + suggester.getName());
    final long startMillis = System.currentTimeMillis();
    suggester.build(core, newSearcher);
    final long timeTakenMillis = System.currentTimeMillis() - startMillis;
    LOG.info("Built suggester " + suggester.getName() + ", took " + timeTakenMillis + " ms");
  } catch (Exception e) {
    LOG.error("Exception in building suggester index for: " + suggester.getName(), e);
  }
}
 
Example 29
Source Project: lucene-solr   Source File: AddBlockUpdateTest.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void testExceptionThrown() throws Exception {
  final String abcD = getStringFromDocument(block("abcD"));
  log.info(abcD);
  assertBlockU(abcD);

  Document docToFail = getDocument();
  Element root = docToFail.createElement("add");
  docToFail.appendChild(root);
  Element doc1 = docToFail.createElement("doc");
  root.appendChild(doc1);
  attachField(docToFail, doc1, "id", id());
  attachField(docToFail, doc1, parent, "Y");
  attachField(docToFail, doc1, "sample_i", "notanumber/ignore_exception");
  Element subDoc1 = docToFail.createElement("doc");
  doc1.appendChild(subDoc1);
  attachField(docToFail, subDoc1, "id", id());
  attachField(docToFail, subDoc1, child, "x");
  Element doc2 = docToFail.createElement("doc");
  root.appendChild(doc2);
  attachField(docToFail, doc2, "id", id());
  attachField(docToFail, doc2, parent, "W");

  assertFailedBlockU(getStringFromDocument(docToFail));

  assertBlockU(getStringFromDocument(block("efgH")));
  assertBlockU(commit());

  final SolrIndexSearcher searcher = getSearcher();
  assertQ(req("q","*:*","indent","true", "fl","id,parent_s,child_s"), "//*[@numFound='" + "abcDefgH".length() + "']");
  assertSingleParentOf(searcher, one("abc"), "D");
  assertSingleParentOf(searcher, one("efg"), "H");

  assertQ(req(child + ":x"), "//*[@numFound='0']");
  assertQ(req(parent + ":Y"), "//*[@numFound='0']");
  assertQ(req(parent + ":W"), "//*[@numFound='0']");
}
 
Example 30
Source Project: jate   Source File: AppBasic.java    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
public List<JATETerm> extract(SolrCore core, JATEProperties properties) throws JATEException {
    SolrIndexSearcher searcher = core.getSearcher().get();
    try {

        this.freqFeatureBuilder = new FrequencyTermBasedFBMaster(searcher, properties, 0);
        this.freqFeature = (FrequencyTermBased) freqFeatureBuilder.build();

        Set<String> uniqueCandidateTerms = freqFeature.getMapTerm2TTF().keySet();
        TermComponentIndexFBMaster termCompIndexFeatureBuilder = new TermComponentIndexFBMaster(properties,
                new ArrayList<>(uniqueCandidateTerms));
        TermComponentIndex termComponentIndexFeature = (TermComponentIndex) termCompIndexFeatureBuilder.build();

        ContainmentFBMaster cb = new ContainmentFBMaster(searcher, properties, termComponentIndexFeature,
                uniqueCandidateTerms);
        Containment cf = (Containment) cb.build();

        Basic basic = new Basic();
        basic.registerFeature(FrequencyTermBased.class.getName(), this.freqFeature);
        basic.registerFeature(Containment.class.getName(), cf);

        List<String> candidates = new ArrayList<>(this.freqFeature.getMapTerm2TTF().keySet());

        filterByTTF(candidates);

        List<JATETerm> terms = basic.execute(candidates);
        terms = cutoff(terms);

        addAdditionalTermInfo(terms, searcher, properties.getSolrFieldNameJATENGramInfo(),
                properties.getSolrFieldNameID());
        LOG.info("Complete Basic term extraction.");
        return terms;
    } finally {
        try {
            searcher.close();
        } catch (IOException e) {
            LOG.error(e.toString());
        }
    }
}