Java Code Examples for org.apache.lucene.index.IndexReader

The following examples show how to use org.apache.lucene.index.IndexReader. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestFieldCacheSort.java    License: Apache License 2.0 6 votes vote down vote up
/** test that we throw exception on multi-valued field, creates corrupt reader, use SORTED_SET instead */
public void testMultiValuedField() throws IOException {
  Directory indexStore = newDirectory();
  IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(new MockAnalyzer(random())));
  for(int i=0; i<5; i++) {
      Document doc = new Document();
      doc.add(new StringField("string", "a"+i, Field.Store.NO));
      doc.add(new StringField("string", "b"+i, Field.Store.NO));
      writer.addDocument(doc);
  }
  writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
  writer.close();
  Sort sort = new Sort(
      new SortField("string", SortField.Type.STRING),
      SortField.FIELD_DOC);
  IndexReader reader = UninvertingReader.wrap(DirectoryReader.open(indexStore),
                       Collections.singletonMap("string", Type.SORTED));
  IndexSearcher searcher = new IndexSearcher(reader);
  expectThrows(IllegalStateException.class, () -> {
    searcher.search(new MatchAllDocsQuery(), 500, sort);
  });
  reader.close();
  indexStore.close();
}
 
Example 2
Source Project: incubator-retired-blur   Source File: BlurUtilsTest.java    License: Apache License 2.0 6 votes vote down vote up
private IndexReader getReader() throws CorruptIndexException, LockObtainFailedException, IOException {
  RAMDirectory directory = new RAMDirectory();
  IndexWriterConfig conf = new IndexWriterConfig(LUCENE_VERSION, new KeywordAnalyzer());
  IndexWriter writer = new IndexWriter(directory, conf);
  Document doc = new Document();
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  doc.add(new StringField("a", "b", Store.YES));
  doc.add(new StringField("family", "f1", Store.YES));

  Document doc1 = new Document();
  doc.add(new StringField("a", "b", Store.YES));
  writer.addDocument(doc);
  writer.addDocument(doc1);
  writer.close();
  return DirectoryReader.open(directory);
}
 
Example 3
Source Project: lucene-solr   Source File: TestLRUQueryCache.java    License: Apache License 2.0 6 votes vote down vote up
public void testPropagateBulkScorer() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  w.addDocument(new Document());
  IndexReader reader = w.getReader();
  w.close();
  IndexSearcher searcher = newSearcher(reader);
  LeafReaderContext leaf = searcher.getIndexReader().leaves().get(0);
  AtomicBoolean scorerCalled = new AtomicBoolean();
  AtomicBoolean bulkScorerCalled = new AtomicBoolean();
  LRUQueryCache cache = new LRUQueryCache(1, Long.MAX_VALUE, context -> true, Float.POSITIVE_INFINITY);

  // test that the bulk scorer is propagated when a scorer should not be cached
  Weight weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1);
  weight = new WeightWrapper(weight, scorerCalled, bulkScorerCalled);
  weight = cache.doCache(weight, NEVER_CACHE);
  weight.bulkScorer(leaf);
  assertEquals(true, bulkScorerCalled.get());
  assertEquals(false, scorerCalled.get());
  assertEquals(0, cache.getCacheCount());

  searcher.getIndexReader().close();
  dir.close();
}
 
Example 4
Source Project: Easy-Cassandra-samples   Source File: MusicSearch.java    License: Apache License 2.0 6 votes vote down vote up
private List<String> returnMusics(Query query) throws IOException {
	int hitsPerPage = 10;
	IndexReader reader = DirectoryReader.open(LuceneUtil.INSTANCE.getDirectory());
	IndexSearcher searcher = new IndexSearcher(reader);
	TopScoreDocCollector collector = TopScoreDocCollector.create(
			hitsPerPage, true);
	searcher.search(query, collector);
	ScoreDoc[] hits = collector.topDocs().scoreDocs;
	
	   
	   List<String> musics = new LinkedList<>();
	    for(int i=0;i<hits.length;++i) {
	      int docId = hits[i].doc;
	      Document d = searcher.doc(docId);
	      musics.add(d.get(COLUMN_NAME));
	    }
	return musics;
}
 
Example 5
private IndexReader indexSomeFields() throws IOException {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
  FieldType ft = new FieldType();
  ft.setIndexOptions(IndexOptions.NONE);
  ft.setTokenized(false);
  ft.setStored(true);
  ft.freeze();

  Field title = new Field("title", "", fieldType);
  Field text = new Field("text", "", fieldType);
  Field category = new Field("category", "", fieldType);

  Document doc = new Document();
  doc.add(title);
  doc.add(text);
  doc.add(category);
  title.setStringValue("This is the title field.");
  text.setStringValue("This is the text field. You can put some text if you want.");
  category.setStringValue("This is the category field.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();
  return ir;
}
 
Example 6
Source Project: lucene-solr   Source File: TestFunctionScoreQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testTruncateNegativeScores() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
  Document doc = new Document();
  doc.add(new NumericDocValuesField("foo", -2));
  w.addDocument(doc);
  IndexReader reader = DirectoryReader.open(w);
  w.close();
  IndexSearcher searcher = newSearcher(reader);
  Query q = new FunctionScoreQuery(new MatchAllDocsQuery(), DoubleValuesSource.fromLongField("foo"));
  QueryUtils.check(random(), q, searcher);
  Explanation expl = searcher.explain(q, 0);
  assertEquals(0, expl.getValue().doubleValue(), 0f);
  assertTrue(expl.toString(), expl.getDetails()[0].getDescription().contains("truncated score"));
  reader.close();
  dir.close();
}
 
Example 7
Source Project: lucene-solr   Source File: FuzzyLikeThisQuery.java    License: Apache License 2.0 6 votes vote down vote up
private Query newTermQuery(IndexReader reader, Term term) throws IOException {
  if (ignoreTF) {
    return new ConstantScoreQuery(new TermQuery(term));
  } else {
    // we build an artificial TermStates that will give an overall df and ttf
    // equal to 1
    TermStates context = new TermStates(reader.getContext());
    for (LeafReaderContext leafContext : reader.leaves()) {
      Terms terms = leafContext.reader().terms(term.field());
      if (terms != null) {
        TermsEnum termsEnum = terms.iterator();
        if (termsEnum.seekExact(term.bytes())) {
          int freq = 1 - context.docFreq(); // we want the total df and ttf to be 1
          context.register(termsEnum.termState(), leafContext.ord, freq, freq);
        }
      }
    }
    return new TermQuery(term, context);
  }
}
 
Example 8
Source Project: dexter   Source File: LuceneHelper.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Builds the TFIDF vector and its norm2
 * 
 * @param tfidf
 *            - the vector containing for each term its TFIDF score, it will
 *            be populated by this method
 * @param freq
 *            - the vector containing for each term its frequency
 * @param field
 *            - the field on which to compute the inverse document frequency
 * 
 * @return the norm of the TFIDF vector
 * 
 */
private double tfidfVector(Map<String, Double> tfidf,
		Map<String, Integer> freq, String field) {
	IndexReader reader = getReader();

	double norm = 0;
	for (Map.Entry<String, Integer> entry : freq.entrySet()) {
		Term t = new Term(field, entry.getKey());
		int df = 0;
		try {
			df = reader.docFreq(t);
		} catch (IOException e) {
			logger.error("computing tfidfVector ({}) ", e.toString());
			System.exit(-1);
		}
		double idf = Math.log(collectionSize / (double) df + 1)
				/ Math.log(2) + 1;
		double tfidfValue = entry.getValue() * idf;
		norm += tfidfValue * tfidfValue;
		tfidf.put(entry.getKey(), tfidfValue);
	}
	return Math.sqrt(norm);

}
 
Example 9
Source Project: lucene-solr   Source File: TestMultiPhraseQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Ignore //LUCENE-3821 fixes sloppy phrase scoring, except for this known problem 
public void testMultiSloppyWithRepeats() throws IOException {
  Directory indexStore = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexStore);
  add("a b c d e f g h i k", writer);
  IndexReader r = writer.getReader();
  writer.close();
  
  IndexSearcher searcher = newSearcher(r);
  
  MultiPhraseQuery.Builder qb = new MultiPhraseQuery.Builder();
  // this will fail, when the scorer would propagate [a] rather than [a,b],
  qb.add(new Term[] {new Term("body", "a"), new Term("body", "b")});
  qb.add(new Term[] {new Term("body", "a")});
  qb.setSlop(6);
  assertEquals(1, searcher.count(qb.build())); // should match on "a b"
  
  r.close();
  indexStore.close();
}
 
Example 10
Source Project: lucene-solr   Source File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public synchronized CacheEntry[] getCacheEntries() {
  List<CacheEntry> result = new ArrayList<>(17);
  for(final Map.Entry<Class<?>,Cache> cacheEntry: caches.entrySet()) {
    final Cache cache = cacheEntry.getValue();
    final Class<?> cacheType = cacheEntry.getKey();
    synchronized(cache.readerCache) {
      for (final Map.Entry<IndexReader.CacheKey,Map<CacheKey, Accountable>> readerCacheEntry : cache.readerCache.entrySet()) {
        final IndexReader.CacheKey readerKey = readerCacheEntry.getKey();
        if (readerKey == null) continue;
        final Map<CacheKey, Accountable> innerCache = readerCacheEntry.getValue();
        for (final Map.Entry<CacheKey, Accountable> mapEntry : innerCache.entrySet()) {
          CacheKey entry = mapEntry.getKey();
          result.add(new CacheEntry(readerKey, entry.field,
                                    cacheType, entry.custom,
                                    mapEntry.getValue()));
        }
      }
    }
  }
  return result.toArray(new CacheEntry[result.size()]);
}
 
Example 11
final Scorer scorer(IndexReader reader) throws IOException {
   if (terms.size() == 0)			  // optimize zero-term case
     return null;
   if (terms.size() == 1) {			  // optimize one-term case
     Term term = (Term)terms.elementAt(0);
     TermDocs docs = reader.termDocs(term);
     if (docs == null)
return null;
     return new TermScorer(docs, reader.norms(term.field()), weight);
   }

   TermPositions[] tps = new TermPositions[terms.size()];
   for (int i = 0; i < terms.size(); i++) {
     TermPositions p = reader.termPositions((Term)terms.elementAt(i));
     if (p == null)
return null;
     tps[i] = p;
   }

   if (slop == 0)				  // optimize exact case
     return new ExactPhraseScorer(tps, reader.norms(field), weight);
   else
     return
new SloppyPhraseScorer(tps, slop, reader.norms(field), weight);

 }
 
Example 12
Source Project: lucene-solr   Source File: TestSpanNearQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoPositions() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new StringField("foo", "bar", Field.Store.NO));
  iw.addDocument(doc);
  
  IndexReader ir = iw.getReader();
  iw.close();
  
  IndexSearcher is = new IndexSearcher(ir);
  SpanTermQuery query = new SpanTermQuery(new Term("foo", "bar"));
  SpanTermQuery query2 = new SpanTermQuery(new Term("foo", "baz"));

  IllegalStateException expected = expectThrows(IllegalStateException.class, () -> {
    is.search(new SpanNearQuery(new SpanQuery[] { query, query2 }, 10, true), 5);
  });
  assertTrue(expected.getMessage().contains("was indexed without position data"));

  ir.close();
  dir.close();
}
 
Example 13
Source Project: lucene-solr   Source File: TestWildcard.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Tests if a WildcardQuery that has only a trailing * in the term is
 * rewritten to a single PrefixQuery. The boost and rewriteMethod should be
 * preserved.
 */
public void testPrefixTerm() throws IOException {
  Directory indexStore = getIndexStore("field", new String[]{"prefix", "prefixx"});
  IndexReader reader = DirectoryReader.open(indexStore);
  IndexSearcher searcher = newSearcher(reader);

  MultiTermQuery wq = new WildcardQuery(new Term("field", "prefix*"));
  assertMatches(searcher, wq, 2);
  
  wq = new WildcardQuery(new Term("field", "*"));
  assertMatches(searcher, wq, 2);
  Terms terms = MultiTerms.getTerms(searcher.getIndexReader(), "field");
  assertFalse(wq.getTermsEnum(terms).getClass().getSimpleName().contains("AutomatonTermsEnum"));
  reader.close();
  indexStore.close();
}
 
Example 14
Source Project: lucene-solr   Source File: TestFieldCacheSort.java    License: Apache License 2.0 6 votes vote down vote up
/** Tests sorting a single document with scores */
public void testSortOneDocumentWithScores() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(newStringField("value", "foo", Field.Store.YES));
  writer.addDocument(doc);
  IndexReader ir = UninvertingReader.wrap(writer.getReader(),
                   Collections.singletonMap("value", Type.SORTED));
  writer.close();
  
  IndexSearcher searcher = newSearcher(ir);
  Sort sort = new Sort(new SortField("value", SortField.Type.STRING));

  TopDocs expected = searcher.search(new TermQuery(new Term("value", "foo")), 10);
  assertEquals(1, expected.totalHits.value);
  TopDocs actual = searcher.search(new TermQuery(new Term("value", "foo")), 10, sort, true);
  
  assertEquals(expected.totalHits.value, actual.totalHits.value);
  assertEquals(expected.scoreDocs[0].score, actual.scoreDocs[0].score, 0F);
  TestUtil.checkReader(ir);
  ir.close();
  dir.close();
}
 
Example 15
Source Project: lucene-solr   Source File: TestLegacyTerms.java    License: Apache License 2.0 6 votes vote down vote up
public void testIntFieldMinMax() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(100);
  int minValue = Integer.MAX_VALUE;
  int maxValue = Integer.MIN_VALUE;
  for(int i=0;i<numDocs;i++ ){
    Document doc = new Document();
    int num = random().nextInt();
    minValue = Math.min(num, minValue);
    maxValue = Math.max(num, maxValue);
    doc.add(new LegacyIntField("field", num, Field.Store.NO));
    w.addDocument(doc);
  }
  
  IndexReader r = w.getReader();
  Terms terms = MultiTerms.getTerms(r, "field");
  assertEquals(Integer.valueOf(minValue), LegacyNumericUtils.getMinInt(terms));
  assertEquals(Integer.valueOf(maxValue), LegacyNumericUtils.getMaxInt(terms));

  r.close();
  w.close();
  dir.close();
}
 
Example 16
public void testBasics() throws Exception {
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);

  Field body = new Field("body", "", fieldType);
  Document doc = new Document();
  doc.add(body);

  body.setStringValue("This is a test. Just a test highlighting from postings. Feel free to ignore.");
  iw.addDocument(doc);
  body.setStringValue("Highlighting the first term. Hope it works.");
  iw.addDocument(doc);

  IndexReader ir = iw.getReader();
  iw.close();

  IndexSearcher searcher = newSearcher(ir);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  Query query = new IntervalQuery("body", Intervals.term("highlighting"));
  TopDocs topDocs = searcher.search(query, 10, Sort.INDEXORDER);
  assertEquals(2, topDocs.totalHits.value);
  String snippets[] = highlighter.highlight("body", query, topDocs);
  assertEquals(2, snippets.length);
  assertEquals("Just a test <b>highlighting</b> from postings. ", snippets[0]);
  assertEquals("<b>Highlighting</b> the first term. ", snippets[1]);
  ir.close();
}
 
Example 17
Source Project: Java-Data-Science-Cookbook   Source File: SearchFiles.java    License: MIT License 6 votes vote down vote up
public static void main(String[] args) throws Exception {
	IndexReader reader = DirectoryReader.open(FSDirectory.open(Paths.get(INDEX_DIRECTORY)));
	IndexSearcher indexSearcher = new IndexSearcher(reader);

	Analyzer analyzer = new StandardAnalyzer();
	QueryParser queryParser = new QueryParser(FIELD_CONTENTS, analyzer);
	String searchString = "shakespeare";
	Query query = queryParser.parse(searchString);

	TopDocs results = indexSearcher.search(query, 5);
	ScoreDoc[] hits = results.scoreDocs;

	int numTotalHits = results.totalHits;
	System.out.println(numTotalHits + " total matching documents");

	for(int i=0;i<hits.length;++i) {
		int docId = hits[i].doc;
		Document d = indexSearcher.doc(docId);
		System.out.println((i + 1) + ". " + d.get("path") + " score=" + hits[i].score);
	}
}
 
Example 18
private void gatherIndexInfo(Map<String, Object> map) {
    SortedMap<String, IndexStatistics> indexStats = new TreeMap<>();
    for (IndexType indexType : IndexType.values()) {
        IndexStatistics stat = new IndexStatistics();
        IndexSearcher searcher = indexManager.getSearcher(indexType);
        stat.setName(indexType.name());
        indexStats.put(indexType.name(), stat);
        if (searcher != null) {
            IndexReader reader = searcher.getIndexReader();
            stat.setCount(reader.numDocs());
            stat.setDeletedCount(reader.numDeletedDocs());
            indexManager.release(indexType, searcher);
        } else {
            stat.setCount(0);
            stat.setDeletedCount(0);
        }
    }
    map.put("indexStatistics", indexStats);

    try (Analyzer analyzer = analyzerFactory.getAnalyzer()) {
        map.put("indexLuceneVersion", analyzer.getVersion().toString());
    } catch (IOException e) {
        LOG.debug("Unable to gather information", e);
    }
}
 
Example 19
Source Project: lucene-solr   Source File: KNearestNeighborClassifier.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Creates a {@link KNearestNeighborClassifier}.
 *
 * @param indexReader     the reader on the index to be used for classification
 * @param analyzer       an {@link Analyzer} used to analyze unseen text
 * @param similarity     the {@link Similarity} to be used by the underlying {@link IndexSearcher} or {@code null}
 *                       (defaults to {@link org.apache.lucene.search.similarities.BM25Similarity})
 * @param query          a {@link Query} to eventually filter the docs used for training the classifier, or {@code null}
 *                       if all the indexed docs should be used
 * @param k              the no. of docs to select in the MLT results to find the nearest neighbor
 * @param minDocsFreq    {@link MoreLikeThis#minDocFreq} parameter
 * @param minTermFreq    {@link MoreLikeThis#minTermFreq} parameter
 * @param classFieldName the name of the field used as the output for the classifier
 * @param textFieldNames the name of the fields used as the inputs for the classifier, they can contain boosting indication e.g. title^10
 */
public KNearestNeighborClassifier(IndexReader indexReader, Similarity similarity, Analyzer analyzer, Query query, int k, int minDocsFreq,
                                  int minTermFreq, String classFieldName, String... textFieldNames) {
  this.textFieldNames = textFieldNames;
  this.classFieldName = classFieldName;
  this.mlt = new MoreLikeThis(indexReader);
  this.mlt.setAnalyzer(analyzer);
  this.mlt.setFieldNames(textFieldNames);
  this.indexSearcher = new IndexSearcher(indexReader);
  if (similarity != null) {
    this.indexSearcher.setSimilarity(similarity);
  } else {
    this.indexSearcher.setSimilarity(new BM25Similarity());
  }
  if (minDocsFreq > 0) {
    mlt.setMinDocFreq(minDocsFreq);
  }
  if (minTermFreq > 0) {
    mlt.setMinTermFreq(minTermFreq);
  }
  this.query = query;
  this.k = k;
}
 
Example 20
public void testCambridgeMA() throws Exception {
  BufferedReader r = new BufferedReader(new InputStreamReader(
      this.getClass().getResourceAsStream("CambridgeMA.utf8"), StandardCharsets.UTF_8));
  String text = r.readLine();
  r.close();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, indexAnalyzer);
  Field body = new Field("body", text, fieldType);
  Document document = new Document();
  document.add(body);
  iw.addDocument(document);
  IndexReader ir = iw.getReader();
  try {
  iw.close();
  IndexSearcher searcher = newSearcher(ir);
  Query query =  new IntervalQuery("body",
           Intervals.unordered(Intervals.term("porter"),
               Intervals.term("square"),
               Intervals.term("massachusetts")));
  TopDocs topDocs = searcher.search(query, 10);
  assertEquals(1, topDocs.totalHits.value);
  UnifiedHighlighter highlighter = randomUnifiedHighlighter(searcher, indexAnalyzer);
  highlighter.setMaxLength(Integer.MAX_VALUE - 1);
  String snippets[] = highlighter.highlight("body", query, topDocs, 2);
  assertEquals(1, snippets.length);
  assertTrue(snippets[0].contains("<b>Square</b>"));
  assertTrue(snippets[0].contains("<b>Porter</b>"));
  } finally{ir.close();}
}
 
Example 21
Source Project: webdsl   Source File: AbstractEntitySearcher.java    License: Apache License 2.0 5 votes vote down vote up
public IndexReader getReader( ) {
    SearchFactory searchFactory = getFullTextSession( ).getSearchFactory( );
    DirectoryProvider<?>[] providers = searchFactory
            .getDirectoryProviders( entityClass );

    return searchFactory.getReaderProvider( ).openReader( providers );
}
 
Example 22
Source Project: mtas   Source File: MtasSearchTestConsistency.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Basic search preceded by 1.
 *
 * @throws IOException Signals that an I/O exception has occurred.
 */
@org.junit.Test
public void basicSearchPrecededBy1() throws IOException {
  String cql1 = "[pos=\"ADJ\"] precededby [pos=\"LID\"][]?";
  String cql2 = "[pos=\"LID\"][]?[pos=\"ADJ\"]";
  String cql3 = "[pos=\"LID\"][pos=\"LID\"][pos=\"ADJ\"]";
  // get total number
  IndexReader indexReader = DirectoryReader.open(directory);
  QueryResult queryResult1 = doQuery(indexReader, FIELD_CONTENT, cql1, null,
      null, null, false);
  QueryResult queryResult1disabled = doQuery(indexReader, FIELD_CONTENT, cql1,
      null, null, null, true);
  QueryResult queryResult2 = doQuery(indexReader, FIELD_CONTENT, cql2, null,
      null, null, false);
  QueryResult queryResult2disabled = doQuery(indexReader, FIELD_CONTENT, cql2,
      null, null, null, true);
  QueryResult queryResult3 = doQuery(indexReader, FIELD_CONTENT, cql3, null,
      null, null, false);
  QueryResult queryResult3disabled = doQuery(indexReader, FIELD_CONTENT, cql3,
      null, null, null, true);
  assertEquals("Adjective preceded by Article", queryResult1.hits,
      (long) queryResult2.hits - queryResult3.hits);
  assertEquals("Adjective preceded by Article - disabled twoPhaseIterator",
      queryResult1disabled.hits,
      (long) queryResult2disabled.hits - queryResult3disabled.hits);
  assertEquals("PrecededBy: twoPhaseIterator 1", queryResult1.hits,
      queryResult1disabled.hits);
  assertEquals("PrecededBy: twoPhaseIterator 2", queryResult2.hits,
      queryResult2disabled.hits);
  assertEquals("PrecededBy: twoPhaseIterator 3", queryResult3.hits,
      queryResult3disabled.hits);
  indexReader.close();
}
 
Example 23
Source Project: lucene-solr   Source File: TestHighFrequencyDictionary.java    License: Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  writer.commit();
  writer.close();
  IndexReader ir = DirectoryReader.open(dir);
  Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
  BytesRefIterator tf = dictionary.getEntryIterator();
  assertNull(tf.next());
  dir.close();
}
 
Example 24
Source Project: lucene-solr   Source File: TestTopDocsCollector.java    License: Apache License 2.0 5 votes vote down vote up
public void testRealisticConcurrentMinimumScore() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  try (LineFileDocs docs = new LineFileDocs(random())) {
    int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; i++) {
      writer.addDocument(docs.nextDoc());
    }
  }

  IndexReader reader = writer.getReader();
  writer.close();

  Terms terms = MultiTerms.getTerms(reader, "body");
  int termCount = 0;
  TermsEnum termsEnum = terms.iterator();
  while(termsEnum.next() != null) {
    termCount++;
  }
  assertTrue(termCount > 0);

  // Target ~10 terms to search:
  double chance = 10.0 / termCount;
  termsEnum = terms.iterator();
  while(termsEnum.next() != null) {
    if (random().nextDouble() <= chance) {
      BytesRef term = BytesRef.deepCopyOf(termsEnum.term());
      Query query = new TermQuery(new Term("body", term));

      TopDocsCollector<ScoreDoc> collector = doSearchWithThreshold(5, 0, query, reader);
      TopDocs tdc = doConcurrentSearchWithThreshold(5, 0, query, reader);
      TopDocs tdc2 = collector.topDocs();

      CheckHits.checkEqual(query, tdc.scoreDocs, tdc2.scoreDocs);
    }
  }

  reader.close();
  dir.close();
}
 
Example 25
Source Project: lucene-solr   Source File: TestReplicationHandlerBackup.java    License: Apache License 2.0 5 votes vote down vote up
private void verify(Path backup, int nDocs) throws IOException {
  log.info("Verifying ndocs={} in {}", nDocs, backup);
  try (Directory dir = new NIOFSDirectory(backup);
       IndexReader reader = DirectoryReader.open(dir)) {
    IndexSearcher searcher = new IndexSearcher(reader);
    TopDocs hits = searcher.search(new MatchAllDocsQuery(), 1);
    assertEquals(nDocs, hits.totalHits.value);
  }
}
 
Example 26
/**
 * test that even if a field is not indexed with points, optimized sort still works as expected,
 * although no optimization will be run
 */
public void testLongSortOptimizationOnFieldNotIndexedWithPoints() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig());
  final int numDocs = atLeast(100);
  // my_field is not indexed with points
  for (int i = 0; i < numDocs; ++i) {
    final Document doc = new Document();
    doc.add(new NumericDocValuesField("my_field", i));
    writer.addDocument(doc);
  }
  final IndexReader reader = DirectoryReader.open(writer);
  IndexSearcher searcher = new IndexSearcher(reader);
  final SortField sortField = new SortField("my_field", SortField.Type.LONG);
  final Sort sort = new Sort(sortField);
  final int numHits = 3;
  final int totalHitsThreshold = 3;

  final TopFieldCollector collector = TopFieldCollector.create(sort, numHits, null, totalHitsThreshold);
  searcher.search(new MatchAllDocsQuery(), collector);
  TopDocs topDocs = collector.topDocs();
  assertEquals(topDocs.scoreDocs.length, numHits);  // sort still works and returns expected number of docs
  for (int i = 0; i < numHits; i++) {
    FieldDoc fieldDoc = (FieldDoc) topDocs.scoreDocs[i];
    assertEquals(i, ((Long) fieldDoc.fields[0]).intValue()); // returns expected values
  }
  assertEquals(topDocs.totalHits.value, numDocs); // assert that all documents were collected => optimization was not run

  writer.close();
  reader.close();
  dir.close();
}
 
Example 27
Source Project: exhibitor   Source File: LogSearch.java    License: Apache License 2.0 5 votes vote down vote up
public LogSearch(File file) throws Exception
{
    this.file = file;
    directory = new NIOFSDirectory(file, new NativeFSLockFactory());
    reader = IndexReader.open(directory);
    searcher = new IndexSearcher(reader);
}
 
Example 28
Source Project: lucene-solr   Source File: TestCheckJoinIndex.java    License: Apache License 2.0 5 votes vote down vote up
public void testInconsistentDeletes() throws IOException {
  final Directory dir = newDirectory();
  final IndexWriterConfig iwc = newIndexWriterConfig();
  iwc.setMergePolicy(NoMergePolicy.INSTANCE); // so that deletions don't trigger merges
  final RandomIndexWriter w = new RandomIndexWriter(random(), dir, iwc);

  List<Document> block = new ArrayList<>();
  final int numChildren = TestUtil.nextInt(random(), 1, 3);
  for (int i = 0; i < numChildren; ++i) {
    Document doc = new Document();
    doc.add(new StringField("child", Integer.toString(i), Store.NO));
    block.add(doc);
  }
  Document parent = new Document();
  parent.add(new StringField("parent", "true", Store.NO));
  block.add(parent);
  w.addDocuments(block);

  if (random().nextBoolean()) {
    w.deleteDocuments(new Term("parent", "true"));
  } else {
    // delete any of the children
    w.deleteDocuments(new Term("child", Integer.toString(random().nextInt(numChildren))));
  }

  final IndexReader reader = w.getReader();
  w.close();

  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("parent", "true")));
  try {
    expectThrows(IllegalStateException.class, () -> CheckJoinIndex.check(reader, parentsFilter));
  } finally {
    reader.close();
    dir.close();
  }
}
 
Example 29
Source Project: netbeans   Source File: Queries.java    License: Apache License 2.0 5 votes vote down vote up
public RegexpTermEnum(
        final IndexReader in,
        final String  fieldName,
        final Pattern pattern,
        final String  startPrefix) throws IOException {
    final Term term = new Term(fieldName,startPrefix);
    this.fieldName = term.field();
    this.pattern = pattern;
    this.startPrefix = startPrefix;
    setEnum(in.terms(term));
}
 
Example 30
Source Project: lucene-solr   Source File: TestRangeFacetCounts.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Query rewrite(IndexReader reader) throws IOException {
  final Query inRewritten = in.rewrite(reader);
  if (in != inRewritten) {
    return new UsedQuery(inRewritten, used);
  }
  return super.rewrite(reader);
}