org.apache.solr.search.SolrIndexSearcher Java Examples

The following examples show how to use org.apache.solr.search.SolrIndexSearcher. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: QueryAutoFilteringComponent.java    From query-autofiltering-component with Apache License 2.0 6 votes vote down vote up
private ArrayList<String> getStringFields( SolrIndexSearcher searcher ) {
  IndexSchema schema = searcher.getSchema();
  ArrayList<String> strFields = new ArrayList<String>( );
    
  Collection<String> fieldNames = searcher.getFieldNames();
  Iterator<String> fnIt = fieldNames.iterator();
  while ( fnIt.hasNext() ) {
    String fieldName = fnIt.next( );
    if (excludeFields == null || !excludeFields.contains( fieldName )) {
      SchemaField field = schema.getField(fieldName);
      if (field.stored() && field.getType() instanceof StrField ) {
        strFields.add( fieldName );
      }
    }
  }
    
  return strFields;
}
 
Example #2
Source File: SolrDenySetQuery.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
public DelegatingCollector getFilterCollector(IndexSearcher searcher)
{
    String[] auths = authorities.substring(1).split(authorities.substring(0, 1));
    try
    {
        HybridBitSet denySet = getACLSet(auths, QueryConstants.FIELD_DENIED, (SolrIndexSearcher) searcher);
        if(denySet instanceof EmptyHybridBitSet)
        {
            return new AllAccessCollector();
        }
        else
        {
            return new AccessControlCollector(denySet);
        }
    }
    catch(Exception e)
    {
        throw new RuntimeException(e);
    }
}
 
Example #3
Source File: SecureRealTimeGetComponent.java    From incubator-sentry with Apache License 2.0 6 votes vote down vote up
/**
 * @param doc SolrDocument to check
 * @param idField field where the id is stored
 * @param fieldType type of id field
 * @param filterQuery Query to filter by
 * @param searcher SolrIndexSearcher on which to apply the filter query
 * @returns the internal docid, or -1 if doc is not found or doesn't match filter
 */
private static int getFilteredInternalDocId(SolrDocument doc, SchemaField idField, FieldType fieldType,
      Query filterQuery, SolrIndexSearcher searcher) throws IOException {
  int docid = -1;
  Field f = (Field)doc.getFieldValue(idField.getName());
  String idStr = f.stringValue();
  BytesRef idBytes = new BytesRef();
  fieldType.readableToIndexed(idStr, idBytes);
  // get the internal document id
  long segAndId = searcher.lookupId(idBytes);

    // if docid is valid, run it through the filter
  if (segAndId >= 0) {
    int segid = (int) segAndId;
    AtomicReaderContext ctx = searcher.getTopReaderContext().leaves().get((int) (segAndId >> 32));
    docid = segid + ctx.docBase;
    Weight weight = filterQuery.createWeight(searcher);
    Scorer scorer = weight.scorer(ctx, null);
    if (scorer == null || segid != scorer.advance(segid)) {
      // filter doesn't match.
      docid = -1;
    }
  }
  return docid;
}
 
Example #4
Source File: FacetTreeGenerator.java    From BioSolr with Apache License 2.0 6 votes vote down vote up
/**
 * Get a reference to the searcher for the required collection. If the collection is
 * not the same as the search collection, we assume it is under the same Solr instance.
 * @param rb the response builder holding the facets.
 * @return a counted reference to the searcher.
 * @throws SolrException if the collection cannot be found.
 */
private RefCounted<SolrIndexSearcher> getSearcherReference(ResponseBuilder rb) throws SolrException {
	RefCounted<SolrIndexSearcher> searcherRef;
	
	SolrCore currentCore = rb.req.getCore();
	if (StringUtils.isBlank(collection)) {
		searcherRef = currentCore.getSearcher();
	} else {
		// Using an alternative core - find it
		SolrCore reqCore = currentCore.getCoreDescriptor().getCoreContainer().getCore(collection);
		if (reqCore == null) {
			throw new SolrException(ErrorCode.BAD_REQUEST, "Collection \"" + collection
					+ "\" cannot be found");
		}
		searcherRef = reqCore.getSearcher();
	}
	
	return searcherRef;
}
 
Example #5
Source File: SolrSuggester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Build the underlying Lucene Suggester */
public void build(SolrCore core, SolrIndexSearcher searcher) throws IOException {
  log.info("SolrSuggester.build({})", name);

  dictionary = dictionaryFactory.create(core, searcher);
  try {
    lookup.build(dictionary);
  } catch (AlreadyClosedException e) {
    RuntimeException e2 = new SolrCoreState.CoreIsClosedException
        ("Suggester build has been interrupted by a core reload or shutdown.");
    e2.initCause(e);
    throw e2;
  }
  if (storeDir != null) {
    File target = getStoreFile();
    if(!lookup.store(new FileOutputStream(target))) {
      log.error("Store Lookup build failed");
    } else {
      if (log.isInfoEnabled()) {
        log.info("Stored suggest data to: {}", target.getAbsolutePath());
      }
    }
  }
}
 
Example #6
Source File: AbstractAuthoritySetQuery.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
protected BitsFilter getACLFilter(String[] auths, String field, SolrIndexSearcher searcher) throws IOException
{
    HybridBitSet aclBits = getACLSet(auths, field, searcher);
    List<LeafReaderContext> leaves = searcher.getTopReaderContext().leaves();
    List<FixedBitSet> bitSets = new ArrayList<FixedBitSet>(leaves.size());

    for(LeafReaderContext readerContext :  leaves)
    {
    	LeafReader reader = readerContext.reader();
        int maxDoc = reader.maxDoc();
        FixedBitSet bits = new FixedBitSet(maxDoc);
        bitSets.add(bits);

        NumericDocValues fieldValues = DocValuesCache.getNumericDocValues(QueryConstants.FIELD_ACLID, reader);
        if (fieldValues != null) {
            for (int i = 0; i < maxDoc; i++) {
                long aclID = fieldValues.get(i);
                if (aclBits.get(aclID)) {
                    bits.set(i);
                }
            }
        }
    }

    return new BitsFilter(bitSets);
}
 
Example #7
Source File: RandomTestDictionaryFactory.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public RandomTestDictionary create(SolrCore core, SolrIndexSearcher searcher) {
  if(params == null) {
    // should not happen; implies setParams was not called
    throw new IllegalStateException("Value of params not set");
  }
  String name = (String)params.get(CommonParams.NAME);
  if (name == null) { // Shouldn't happen since this is the component name
    throw new IllegalArgumentException(CommonParams.NAME + " is a mandatory parameter");
  }
  long maxItems = DEFAULT_MAX_ITEMS;
  Object specifiedMaxItems = params.get(RAND_DICT_MAX_ITEMS);
  if (specifiedMaxItems != null) {
    maxItems = Long.parseLong(specifiedMaxItems.toString());
  }
  return new RandomTestDictionary(name, maxItems);
}
 
Example #8
Source File: FacetTreeGenerator.java    From BioSolr with Apache License 2.0 6 votes vote down vote up
public List<SimpleOrderedMap<Object>> generateTree(ResponseBuilder rb, NamedList<Integer> facetValues) throws IOException {
	List<SimpleOrderedMap<Object>> retVal = null;
	
	// First get the searcher for the required collection
	RefCounted<SolrIndexSearcher> searcherRef = getSearcherReference(rb);
	
	try {
		// Build the facet tree(s)
		Collection<TreeFacetField> fTrees = treeBuilder.processFacetTree(searcherRef.get(), extractFacetValues(facetValues));
		LOGGER.debug("Extracted {} facet trees", fTrees.size());
		
		if (pruner != null) {
			// Prune the trees
			fTrees = pruner.prune(fTrees);
		}

		// Convert the trees into a SimpleOrderedMap
		retVal = convertTreeFacetFields(fTrees);
	} finally {
		// Make sure the search ref count is decreased
		searcherRef.decref();
	}
	
	return retVal;
}
 
Example #9
Source File: HashTermStatistics.java    From liresolr with GNU General Public License v2.0 6 votes vote down vote up
public static void addToStatistics(SolrIndexSearcher searcher, String field) throws IOException {
        // check if this field is already in the stats.
//        synchronized (instance) {
            if (termstats.get(field)!=null) return;
//        }
        // else add it to the stats.
        Terms terms = searcher.getSlowAtomicReader().terms(field);
        HashMap<String, Integer> term2docFreq = new HashMap<String, Integer>(1000);
        termstats.put(field, term2docFreq);
        if (terms!=null) {
            TermsEnum termsEnum = terms.iterator();
            BytesRef term;
            while ((term = termsEnum.next()) != null) {
                term2docFreq.put(term.utf8ToString(), termsEnum.docFreq());
            }
        }
    }
 
Example #10
Source File: SolrPluginUtils.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Generates an NamedList of Explanations for each item in a list of docs.
 *
 * @param query The Query you want explanations in the context of
 * @param docs The Documents you want explained relative that query
 */
public static NamedList<Explanation> getExplanations
  (Query query,
   DocList docs,
   SolrIndexSearcher searcher,
   IndexSchema schema) throws IOException {

  NamedList<Explanation> explainList = new SimpleOrderedMap<>();
  DocIterator iterator = docs.iterator();
  for (int i=0; i<docs.size(); i++) {
    int id = iterator.nextDoc();

    Document doc = searcher.doc(id);
    String strid = schema.printableUniqueKey(doc);

    explainList.add(strid, searcher.explain(query, id) );
  }
  return explainList;
}
 
Example #11
Source File: LireRequestHandler.java    From liresolr with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Returns a random set of documents from the index. Mainly for testing purposes.
 *
 * @param req
 * @param rsp
 * @throws IOException
 */
private void handleRandomSearch(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
    SolrIndexSearcher searcher = req.getSearcher();
    Query query = new MatchAllDocsQuery();
    DocList docList = searcher.getDocList(query, getFilterQueries(req), Sort.RELEVANCE, 0, numberOfCandidateResults, 0);
    int paramRows = Math.min(req.getParams().getInt("rows", defaultNumberOfResults), docList.size());
    if (docList.size() < 1) {
        rsp.add("Error", "No documents in index");
    } else {
        LinkedList list = new LinkedList();
        while (list.size() < paramRows) {
            DocList auxList = docList.subset((int) (Math.random() * docList.size()), 1);
            Document doc = null;
            for (DocIterator it = auxList.iterator(); it.hasNext(); ) {
                doc = searcher.doc(it.nextDoc());
            }
            if (!list.contains(doc)) {
                list.add(doc);
            }
        }
        rsp.addResponse(list);
    }
}
 
Example #12
Source File: ChildNodeFacetTreeBuilder.java    From BioSolr with Apache License 2.0 5 votes vote down vote up
/**
 * Fetch facets for items containing a specific set of values.
 * @param searcher the searcher for the collection being used.
 * @param facetValues the incoming values to use as filters.
 * @param filterField the item field containing the child values, which will be used
 * to filter against.
 * @return a map of node value to child values for the items.
 * @throws IOException
 */
private Map<String, Set<String>> filterEntriesByField(SolrIndexSearcher searcher, Collection<String> facetValues,
		String filterField) throws IOException {
	Map<String, Set<String>> filteredEntries = new HashMap<>();

	LOGGER.debug("Looking up {} entries in field {}", facetValues.size(), filterField);
	Query filter = buildFilterQuery(filterField, facetValues);
	LOGGER.trace("Filter query: {}", filter);

	DocSet docs = searcher.getDocSet(filter);

	for (DocIterator it = docs.iterator(); it.hasNext(); ) {
		Document doc = searcher.doc(it.nextDoc(), docFields);
		String nodeId = doc.get(getNodeField());
		
		// Get the children for the node, if necessary
		Set<String> childIds;
		if (filterField.equals(getNodeField())) {
			// Filtering on the node field - child IDs are redundant
			childIds = Collections.emptySet();
		} else {
			childIds = new HashSet<>(Arrays.asList(doc.getValues(filterField)));
			LOGGER.trace("Got {} children for node {}", childIds.size(), nodeId);
		}
		filteredEntries.put(nodeId, childIds);
		
		// Record the label, if required
		if (isLabelRequired(nodeId)) {
			recordLabel(nodeId, doc.getValues(getLabelField()));
		}
	}

	return filteredEntries;
}
 
Example #13
Source File: SolrCore.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Computes fingerprint of a segment and caches it only if all the version in segment are included in the fingerprint.
 * We can't use computeIfAbsent as caching is conditional (as described above)
 * There is chance that two threads may compute fingerprint on the same segment. It might be OK to do so rather than locking entire map.
 *
 * @param searcher   searcher that includes specified LeaderReaderContext
 * @param ctx        LeafReaderContext of a segment to compute fingerprint of
 * @param maxVersion maximum version number to consider for fingerprint computation
 * @return IndexFingerprint of the segment
 * @throws IOException Can throw IOException
 */
public IndexFingerprint getIndexFingerprint(SolrIndexSearcher searcher, LeafReaderContext ctx, long maxVersion)
    throws IOException {
  IndexReader.CacheHelper cacheHelper = ctx.reader().getReaderCacheHelper();
  if (cacheHelper == null) {
    if (log.isDebugEnabled()) {
      log.debug("Cannot cache IndexFingerprint as reader does not support caching. searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
    return IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
  }

  IndexFingerprint f = null;
  f = perSegmentFingerprintCache.get(cacheHelper.getKey());
  // fingerprint is either not cached or
  // if we want fingerprint only up to a version less than maxVersionEncountered in the segment, or
  // documents were deleted from segment for which fingerprint was cached
  //
  if (f == null || (f.getMaxInHash() > maxVersion) || (f.getNumDocs() != ctx.reader().numDocs())) {
    if (log.isDebugEnabled()) {
      log.debug("IndexFingerprint cache miss for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
    f = IndexFingerprint.getFingerprint(searcher, ctx, maxVersion);
    // cache fingerprint for the segment only if all the versions in the segment are included in the fingerprint
    if (f.getMaxVersionEncountered() == f.getMaxInHash()) {
      log.debug("Caching fingerprint for searcher:{} leafReaderContext:{} mavVersion:{}", searcher, ctx, maxVersion);
      perSegmentFingerprintCache.put(cacheHelper.getKey(), f);
    }

  } else {
    if (log.isDebugEnabled()) {
      log.debug("IndexFingerprint cache hit for searcher:{} reader:{} readerHash:{} maxVersion:{}", searcher, ctx.reader(), ctx.reader().hashCode(), maxVersion);
    }
  }
  if (log.isDebugEnabled()) {
    log.debug("Cache Size: {}, Segments Size:{}", perSegmentFingerprintCache.size(), searcher.getTopReaderContext().leaves().size());
  }
  return f;
}
 
Example #14
Source File: LocalDatasetGraph.java    From SolRDF with Apache License 2.0 5 votes vote down vote up
@Override
public Iterator<Node> listGraphNodes() {
    final SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
    try {
		request.getSearcher().search(result, GET_GRAPH_NODES_QUERY);
		final SimpleFacets facets = new SimpleFacets(
	    		request, 
	    		result.getDocSet(), 
	    		GET_GRAPH_NODES_QUERY_PARAMS);
		
		final NamedList<Integer> list = facets.getFacetTermEnumCounts(
				request.getSearcher(), 
				result.getDocSet(),
				Field.C,
				0,
				-1,
				1,
				false,
				"count"
				,null,
				null,
				false,
				null);
		final List<Node> graphs = new ArrayList<Node>();
		for (final Entry<String, Integer> entry : list) {
			if (!SolRDFGraph.UNNAMED_GRAPH_PLACEHOLDER.equals(entry.getKey())) {
				graphs.add(NTriples.asURI(entry.getKey()));
			}
		}
		
		LOGGER.debug(MessageCatalog._00112_GRAPHS_TOTAL_COUNT, graphs.size());
		
		return graphs.iterator();
	} catch (final Exception exception) {
		LOGGER.error(MessageCatalog._00113_NWS_FAILURE, exception);
		throw new SolrException(ErrorCode.SERVER_ERROR, exception);
	}	    
}
 
Example #15
Source File: IntervalFacets.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Constructor that accepts an already constructed array of {@link FacetInterval} objects. This array needs to be sorted
 * by start value in weakly ascending order. null values are not allowed in the array.
 */
public IntervalFacets(SchemaField schemaField, SolrIndexSearcher searcher, DocSet docs, FacetInterval[] intervals) throws IOException {
  this.schemaField = schemaField;
  this.searcher = searcher;
  this.docs = docs;
  this.intervals = intervals;
  doCount();
}
 
Example #16
Source File: SimpleFacets.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 *  Works like {@link #getFacetTermEnumCounts(SolrIndexSearcher, DocSet, String, int, int, int, boolean, String, String, Predicate, boolean)}
 *  but takes a substring directly for the contains check rather than a {@link Predicate} instance.
 */
public NamedList<Integer> getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing,
                                                 String sort, String prefix, String contains, boolean ignoreCase, boolean intersectsCheck)
  throws IOException {

  final Predicate<BytesRef> termFilter = new SubstringBytesRefFilter(contains, ignoreCase);
  return getFacetTermEnumCounts(searcher, docs, field, offset, limit, mincount, missing, sort, prefix, termFilter, intersectsCheck);
}
 
Example #17
Source File: AddBlockUpdateTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testSolrNestedFieldsSingleVal() throws Exception {
  SolrInputDocument document1 = sdoc("id", id(), parent, "X",
      "child1_s", sdoc("id", id(), "child_s", "y"),
      "child2_s", sdoc("id", id(), "child_s", "z"));

  SolrInputDocument document2 = sdoc("id", id(), parent, "A",
      "child1_s", sdoc("id", id(), "child_s", "b"),
      "child2_s", sdoc("id", id(), "child_s", "c"));

  indexSolrInputDocumentsDirectly(document1, document2);

  final SolrIndexSearcher searcher = getSearcher();
  assertJQ(req("q","*:*",
      "fl","*",
      "sort","id asc",
      "wt","json"),
      "/response/numFound==" + "XyzAbc".length());
  assertJQ(req("q",parent+":" + document2.getFieldValue(parent),
      "fl","*",
      "sort","id asc",
      "wt","json"),
      "/response/docs/[0]/id=='" + document2.getFieldValue("id") + "'");
  assertQ(req("q",child+":(y z b c)", "sort","_docid_ asc"),
      "//*[@numFound='" + "yzbc".length() + "']", // assert physical order of children
      "//doc[1]/arr[@name='child_s']/str[text()='y']",
      "//doc[2]/arr[@name='child_s']/str[text()='z']",
      "//doc[3]/arr[@name='child_s']/str[text()='b']",
      "//doc[4]/arr[@name='child_s']/str[text()='c']");
  assertSingleParentOf(searcher, one("bc"), "A");
  assertSingleParentOf(searcher, one("yz"), "X");
}
 
Example #18
Source File: AppATTF.java    From jate with GNU Lesser General Public License v3.0 5 votes vote down vote up
/**
     * ranking and filtering
     *
     * @param core        solr core
     * @param properties  jate properties file
     * @return List<JATETerm> a list of JATETerm
     * @throws JATEException
     */
    public List<JATETerm> extract(SolrCore core, JATEProperties properties) throws JATEException {
        if (core.isClosed()) {
            core.open();
        }
        SolrIndexSearcher searcher = core.getSearcher().get();
//        try {
	        this.freqFeatureBuilder = new FrequencyTermBasedFBMaster(searcher, properties, FrequencyTermBasedFBMaster.FEATURE_TYPE_TERM);
	        this.freqFeature = (FrequencyTermBased) freqFeatureBuilder.build();
	
	        Algorithm attf = new ATTF();
	        attf.registerFeature(FrequencyTermBased.class.getName(), freqFeature);
	
	        List<String> candidates = new ArrayList<>(freqFeature.getMapTerm2TTF().keySet());
	
	        filterByTTF(candidates);
	
	        List<JATETerm> terms = attf.execute(candidates);
	
	        terms = cutoff(terms);
	        LOG.info("Complete ATTF term extraction.");
	
	        addAdditionalTermInfo(terms, searcher, properties.getSolrFieldNameJATENGramInfo(),
	                properties.getSolrFieldNameID());
	
	        return terms;
//        } finally {
//        	try {
//				searcher.close();
//			} catch (IOException e) {
//				LOG.error(e.toString());
//			}
//        }
    }
 
Example #19
Source File: UnInvertedField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public UnInvertedField(String field, SolrIndexSearcher searcher) throws IOException {
  super(field,
      // threshold, over which we use set intersections instead of counting
      // to (1) save memory, and (2) speed up faceting.
      // Add 2 for testing purposes so that there will always be some terms under
      // the threshold even when the index is very
      // small.
      searcher.maxDoc()/20 + 2,
      DEFAULT_INDEX_INTERVAL_BITS);

  assert TestInjection.injectUIFOutOfMemoryError();

  final String prefix = TrieField.getMainValuePrefix(searcher.getSchema().getFieldType(field));
  this.searcher = searcher;
  try {
    // TODO: it's wasteful to create one of these each time
    // but DocTermOrds will throw an exception if it thinks the field has doc values (which is faked by UnInvertingReader)
    LeafReader r = SlowCompositeReaderWrapper.wrap(searcher.getRawReader());
    uninvert(r, r.getLiveDocs(), prefix == null ? null : new BytesRef(prefix));
  } catch (IllegalStateException ise) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, ise);
  }
  if (tnums != null) {
    for(byte[] target : tnums) {
      if (target != null && target.length > (1<<24)*.9) {
        log.warn("Approaching too many values for UnInvertedField faceting on field '{}' : bucket size={}", field, target.length);
      }
    }
  }

  // free space if outrageously wasteful (tradeoff memory/cpu) 
  if ((maxTermCounts.length - numTermsInField) > 1024) { // too much waste!
    int[] newMaxTermCounts = new int[numTermsInField];
    System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, numTermsInField);
    maxTermCounts = newMaxTermCounts;
  }

  log.info("UnInverted multi-valued field {}", this);
  //System.out.println("CREATED: " + toString() + " ti.index=" + ti.index);
}
 
Example #20
Source File: ClassificationUpdateProcessorIntegrationTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private Document getDoc(String id) throws IOException {
  try (SolrQueryRequest req = req()) {
    SolrIndexSearcher searcher = req.getSearcher();
    TermQuery query = new TermQuery(new Term(ID, id));
    TopDocs doc1 = searcher.search(query, 1);
    ScoreDoc scoreDoc = doc1.scoreDocs[0];
    return searcher.doc(scoreDoc.doc);
  }
}
 
Example #21
Source File: UninvertDocValuesMergePolicyTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static void withNewRawReader(TestHarness h, DirectoryReaderConsumer consumer) {
  try (SolrCore core = h.getCoreInc()) {
    final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
    final SolrIndexSearcher searcher = searcherRef.get();
    try {
      try {
        consumer.accept(searcher.getRawReader());
      } catch (Exception e) {
        fail(e.toString());
      }
    } finally {
      searcherRef.decref();
    }
  }
}
 
Example #22
Source File: FrequencyCtxSentenceBasedFBWorker.java    From jate with GNU Lesser General Public License v3.0 5 votes vote down vote up
public FrequencyCtxSentenceBasedFBWorker(FrequencyCtxBased feature, JATEProperties properties,
                                         List<Integer> docIds,
                                         Set<String> allCandidates,
                                         SolrIndexSearcher solrIndexSearcher,
                                         int maxTasksPerWorker) {
    super(docIds, maxTasksPerWorker);
    this.properties = properties;
    this.solrIndexSearcher = solrIndexSearcher;
    this.allCandidates=allCandidates;
    this.feature=feature;
}
 
Example #23
Source File: LocalGraph.java    From SolRDF with Apache License 2.0 5 votes vote down vote up
@Override
protected int graphBaseSize() {
	final SolrIndexSearcher.QueryResult result = new SolrIndexSearcher.QueryResult();
    try {
	    return searcher.search(result, graphSizeQueryCommand()).getDocListAndSet().docList.matches();
	} catch (final Exception exception) {
		LOGGER.error(MessageCatalog._00113_NWS_FAILURE, exception);
		throw new SolrException(ErrorCode.SERVER_ERROR, exception);
	}	    
}
 
Example #24
Source File: AppBasic.java    From jate with GNU Lesser General Public License v3.0 5 votes vote down vote up
public List<JATETerm> extract(SolrCore core, JATEProperties properties) throws JATEException {
    SolrIndexSearcher searcher = core.getSearcher().get();
    try {

        this.freqFeatureBuilder = new FrequencyTermBasedFBMaster(searcher, properties, 0);
        this.freqFeature = (FrequencyTermBased) freqFeatureBuilder.build();

        Set<String> uniqueCandidateTerms = freqFeature.getMapTerm2TTF().keySet();
        TermComponentIndexFBMaster termCompIndexFeatureBuilder = new TermComponentIndexFBMaster(properties,
                new ArrayList<>(uniqueCandidateTerms));
        TermComponentIndex termComponentIndexFeature = (TermComponentIndex) termCompIndexFeatureBuilder.build();

        ContainmentFBMaster cb = new ContainmentFBMaster(searcher, properties, termComponentIndexFeature,
                uniqueCandidateTerms);
        Containment cf = (Containment) cb.build();

        Basic basic = new Basic();
        basic.registerFeature(FrequencyTermBased.class.getName(), this.freqFeature);
        basic.registerFeature(Containment.class.getName(), cf);

        List<String> candidates = new ArrayList<>(this.freqFeature.getMapTerm2TTF().keySet());

        filterByTTF(candidates);

        List<JATETerm> terms = basic.execute(candidates);
        terms = cutoff(terms);

        addAdditionalTermInfo(terms, searcher, properties.getSolrFieldNameJATENGramInfo(),
                properties.getSolrFieldNameID());
        LOG.info("Complete Basic term extraction.");
        return terms;
    } finally {
        try {
            searcher.close();
        } catch (IOException e) {
            LOG.error(e.toString());
        }
    }
}
 
Example #25
Source File: AddBlockUpdateTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Test
public void testExceptionThrown() throws Exception {
  final String abcD = getStringFromDocument(block("abcD"));
  log.info(abcD);
  assertBlockU(abcD);

  Document docToFail = getDocument();
  Element root = docToFail.createElement("add");
  docToFail.appendChild(root);
  Element doc1 = docToFail.createElement("doc");
  root.appendChild(doc1);
  attachField(docToFail, doc1, "id", id());
  attachField(docToFail, doc1, parent, "Y");
  attachField(docToFail, doc1, "sample_i", "notanumber/ignore_exception");
  Element subDoc1 = docToFail.createElement("doc");
  doc1.appendChild(subDoc1);
  attachField(docToFail, subDoc1, "id", id());
  attachField(docToFail, subDoc1, child, "x");
  Element doc2 = docToFail.createElement("doc");
  root.appendChild(doc2);
  attachField(docToFail, doc2, "id", id());
  attachField(docToFail, doc2, parent, "W");

  assertFailedBlockU(getStringFromDocument(docToFail));

  assertBlockU(getStringFromDocument(block("efgH")));
  assertBlockU(commit());

  final SolrIndexSearcher searcher = getSearcher();
  assertQ(req("q","*:*","indent","true", "fl","id,parent_s,child_s"), "//*[@numFound='" + "abcDefgH".length() + "']");
  assertSingleParentOf(searcher, one("abc"), "D");
  assertSingleParentOf(searcher, one("efg"), "H");

  assertQ(req(child + ":x"), "//*[@numFound='0']");
  assertQ(req(parent + ":Y"), "//*[@numFound='0']");
  assertQ(req(parent + ":W"), "//*[@numFound='0']");
}
 
Example #26
Source File: AsyncBuildSuggestComponent.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
private void buildSuggesterIndex(SolrSuggester suggester, SolrIndexSearcher newSearcher) {
  try {
    LOG.info("Building suggester index for: " + suggester.getName());
    final long startMillis = System.currentTimeMillis();
    suggester.build(core, newSearcher);
    final long timeTakenMillis = System.currentTimeMillis() - startMillis;
    LOG.info("Built suggester " + suggester.getName() + ", took " + timeTakenMillis + " ms");
  } catch (Exception e) {
    LOG.error("Exception in building suggester index for: " + suggester.getName(), e);
  }
}
 
Example #27
Source File: FacetHeatmap.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private Bits getTopAcceptDocs(DocSet docSet, SolrIndexSearcher searcher) throws IOException {
  if (docSet.size() == searcher.numDocs()) {
    return null; // means match everything (all live docs). This can speedup things a lot.
  } else if (docSet.size() == 0) {
    return new Bits.MatchNoBits(searcher.maxDoc()); // can speedup things a lot
  } else {
    return docSet.getBits();
  }
}
 
Example #28
Source File: UnInvertedField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Called for each term in the field being uninverted.
 * Collects {@link #maxTermCounts} for all bigTerms as well as storing them in {@link #bigTerms}.
 * @param te positioned at the current term.
 * @param termNum the ID/pointer/ordinal of the current term. Monotonically increasing between calls.
 */
@Override
protected void visitTerm(TermsEnum te, int termNum) throws IOException {

  if (termNum >= maxTermCounts.length) {
    // resize by doubling - for very large number of unique terms, expanding
    // by 4K and resultant GC will dominate uninvert times.  Resize at end if material
    int[] newMaxTermCounts = new int[ Math.min(Integer.MAX_VALUE-16, maxTermCounts.length*2) ];
    System.arraycopy(maxTermCounts, 0, newMaxTermCounts, 0, termNum);
    maxTermCounts = newMaxTermCounts;
  }

  final BytesRef term = te.term();

  if (te.docFreq() > maxTermDocFreq) {
    Term t = new Term(field, term);  // this makes a deep copy of the term bytes
    TopTerm topTerm = new TopTerm();
    topTerm.term = t.bytes();
    topTerm.termNum = termNum;
    topTerm.termQuery = new TermQuery(t);

    bigTerms.put(topTerm.termNum, topTerm);

    if (deState == null) {
      deState = new SolrIndexSearcher.DocsEnumState();
      deState.fieldName = field;
      deState.liveDocs = searcher.getLiveDocsBits();
      deState.termsEnum = te;  // TODO: check for MultiTermsEnum in SolrIndexSearcher could now fail?
      deState.postingsEnum = postingsEnum;
      deState.minSetSizeCached = maxTermDocFreq;
    }

    postingsEnum = deState.postingsEnum;
    DocSet set = searcher.getDocSet(deState);
    maxTermCounts[termNum] = set.size();
  }
}
 
Example #29
Source File: Log.java    From SolRDF with Apache License 2.0 5 votes vote down vote up
/**
 * Debugs the given query command.
 * 
 * @param cmd the query command.
 */
public void debugQuery(final QueryCommand cmd, final SolrIndexSearcher.QueryResult result) {
	if (logger.isDebugEnabled()) {
		final StringBuilder builder = new StringBuilder("*:*");
		for (final Query filter : cmd.getFilterList()) {
			builder.append(" & ").append(filter);
		}
		
		logger.debug(createMessage(
				MessageCatalog._00109_SOLR_QUERY, 
				builder.toString(), 
				result.getDocList().size(), 
				result.getDocList().matches()));
	}
}
 
Example #30
Source File: AddBlockUpdateTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected void assertSingleParentOf(final SolrIndexSearcher searcher,
    final String childTerm, String parentExp) throws IOException {
  final TopDocs docs = searcher.search(join(childTerm), 10);
  assertEquals(1, docs.totalHits.value);
  final String pAct = searcher.doc(docs.scoreDocs[0].doc).get(parent);
  assertEquals(parentExp, pAct);
}