Java Code Examples for org.apache.lucene.document.Field.Store

The following examples show how to use org.apache.lucene.document.Field.Store. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateAllDeletedSegment() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.commit();
  writer.deleteDocuments(new Term("id", "doc")); // delete all docs in the first segment
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("id", "doc"), "f1", toBytes(2L));
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(2L, getValue(bdv));
  reader.close();
  
  dir.close();
}
 
Example 2
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example 3
Source Project: bluima   Source File: Txt2PubmedIdIndexer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public void process(JCas jCas) throws AnalysisEngineProcessException {
    int pmid = BlueCasUtil.getHeaderIntDocId(jCas);
    if (!BlueCasUtil.isEmptyText(jCas)) {
        // System.out.println("indexing:: " + pmid);
        Document doc = new Document();
        doc.add(new IntField(PMID_FIELD, pmid, Store.YES));
        doc.add(new TextField(CONTENT_FIELD, jCas.getDocumentText(),
                Store.YES));
        doc.add(new TextField(TITLE_FIELD, getTitle(jCas), Store.YES));
        try {
            indexWriter.addDocument(doc);
        } catch (IOException e) {
            throw new AnalysisEngineProcessException(e);
        }
    }
}
 
Example 4
Source Project: lucene-solr   Source File: TestNorms.java    License: Apache License 2.0 6 votes vote down vote up
public void testEmptyValueVsNoValue() throws IOException {
  Directory dir = newDirectory();
  IndexWriterConfig cfg = newIndexWriterConfig().setMergePolicy(newLogMergePolicy());
  IndexWriter w = new IndexWriter(dir, cfg);
  Document doc = new Document();
  w.addDocument(doc);
  doc.add(newTextField("foo", "", Store.NO));
  w.addDocument(doc);
  w.forceMerge(1);
  IndexReader reader = DirectoryReader.open(w);
  w.close();
  LeafReader leafReader = getOnlyLeafReader(reader);
  NumericDocValues normValues = leafReader.getNormValues("foo");
  assertNotNull(normValues);
  assertEquals(1, normValues.nextDoc()); // doc 0 does not have norms
  assertEquals(0, normValues.longValue());
  reader.close();
  dir.close();
}
 
Example 5
Source Project: dremio-oss   Source File: TestLuceneIndexer.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = StaleSearcherException.class)
public void testSearcherCacheTTL() throws Exception {
  try (LuceneSearchIndex index = new LuceneSearchIndex(null, "multithreaded-search", true, CommitWrapper.NO_OP, 500)) {
    for (int i = 0; i < 10; ++i) {
      final Document doc = new Document();
      doc.add(
          new StringField(CoreIndexedStore.ID_FIELD_NAME, new BytesRef(Integer.toString(i).getBytes()), Store.YES));
      doc.add(new StringField("user", "u1", Field.Store.YES));
      index.add(doc);
    }

    Query query = new TermQuery(new Term("user", "u1"));
    LuceneSearchIndex.SearchHandle searchHandle = index.createSearchHandle();
    List<Doc> docs = index.search(searchHandle, query, 4, new Sort(), 0);
    assertEquals(4, docs.size());

    // sleep to force cache expiry.
    Thread.sleep(1000);

    docs = index.searchAfter(searchHandle, query, 6, new Sort(), docs.get(3));
    assertEquals(6, docs.size());

    searchHandle.close();
  }
}
 
Example 6
Source Project: lucene-solr   Source File: TestMixedDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateNotExistingFieldDV() throws IOException {
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
    Document doc = new Document();
    doc.add(new StringField("id", "1", Store.YES));
    doc.add(new NumericDocValuesField("test", 1));
    writer.addDocument(doc);
    if (random().nextBoolean()) {
      writer.commit();
    }
    writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));

    Document doc1 = new Document();
    doc1.add(new StringField("id", "2", Store.YES));
    doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
        writer.addDocument(doc1)
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());

    iae = expectThrows(IllegalArgumentException.class, () ->
        writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
  }
}
 
Example 7
Source Project: lucene-solr   Source File: TestDocValuesStatsCollector.java    License: Apache License 2.0 6 votes vote down vote up
public void testOneDoc() throws IOException {
  try (Directory dir = newDirectory();
      IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig())) {
    String field = "numeric";
    Document doc = new Document();
    doc.add(new NumericDocValuesField(field, 1));
    doc.add(new StringField("id", "doc1", Store.NO));
    indexWriter.addDocument(doc);

    try (DirectoryReader reader = DirectoryReader.open(indexWriter)) {
      IndexSearcher searcher = new IndexSearcher(reader);
      LongDocValuesStats stats = new LongDocValuesStats(field);
      searcher.search(new MatchAllDocsQuery(), new DocValuesStatsCollector(stats));

      assertEquals(1, stats.count());
      assertEquals(0, stats.missing());
      assertEquals(1, stats.max().longValue());
      assertEquals(1, stats.min().longValue());
      assertEquals(1, stats.sum().longValue());
      assertEquals(1, stats.mean(), 0.0001);
      assertEquals(0, stats.variance(), 0.0001);
      assertEquals(0, stats.stdev(), 0.0001);
    }
  }
}
 
Example 8
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateBinaryDVFieldWithSameNameAsPostingField() throws Exception {
  // this used to fail because FieldInfos.Builder neglected to update
  // globalFieldMaps.docValuesTypes map
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new BinaryDocValuesField("f", toBytes(5L)));
  writer.addDocument(doc);
  writer.commit();
  writer.updateBinaryDocValue(new Term("f", "mock-value"), "f", toBytes(17L));
  writer.close();
  
  DirectoryReader r = DirectoryReader.open(dir);
  BinaryDocValues bdv = r.leaves().get(0).reader().getBinaryDocValues("f");
  assertEquals(0, bdv.nextDoc());
  assertEquals(17, getValue(bdv));
  r.close();
  
  dir.close();
}
 
Example 9
Source Project: lucene-solr   Source File: TestNormsFieldExistsQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testFieldExistsButNoDocsHaveField() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  // 1st segment has the field, but 2nd one does not
  Document doc = new Document();
  doc.add(new TextField("f", "value", Store.NO));
  iw.addDocument(doc);
  iw.commit();
  iw.addDocument(new Document());
  iw.commit();
  final IndexReader reader = iw.getReader();
  final IndexSearcher searcher = newSearcher(reader);
  iw.close();
  assertEquals(1, searcher.count(new NormsFieldExistsQuery("f")));
  reader.close();
  dir.close();
}
 
Example 10
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testSameFieldNameForPostingAndDocValue() throws Exception {
  // LUCENE-5192: FieldInfos.Builder neglected to update
  // globalFieldNumbers.docValuesType map if the field existed, resulting in
  // potentially adding the same field with different DV types.
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new NumericDocValuesField("f", 5));
  writer.addDocument(doc);
  writer.commit();
  
  Document doc2 = new Document();
  doc2.add(new BinaryDocValuesField("f", new BytesRef("mock")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(doc2);
  });
  writer.rollback();
  
  dir.close();
}
 
Example 11
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateTwoNonexistingTerms() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new NumericDocValuesField("f1", 1L));
  writer.addDocument(doc);
  // update w/ multiple nonexisting terms in same field
  writer.updateNumericDocValue(new Term("c", "foo"), "f1", 2L);
  writer.updateNumericDocValue(new Term("c", "bar"), "f1", 2L);
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
  assertEquals(0, dvs.nextDoc());
  assertEquals(1, dvs.longValue());
  reader.close();
  
  dir.close();
}
 
Example 12
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateNumericDVFieldWithSameNameAsPostingField() throws Exception {
  // this used to fail because FieldInfos.Builder neglected to update
  // globalFieldMaps.docValuesTypes map
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new NumericDocValuesField("f", 5));
  writer.addDocument(doc);
  writer.commit();
  writer.updateNumericDocValue(new Term("f", "mock-value"), "f", 17L);
  writer.close();
  
  DirectoryReader r = DirectoryReader.open(dir);
  NumericDocValues ndv = r.leaves().get(0).reader().getNumericDocValues("f");
  assertEquals(0, ndv.nextDoc());
  assertEquals(17, ndv.longValue());
  r.close();
  
  dir.close();
}
 
Example 13
Source Project: incubator-retired-blur   Source File: RowDocumentUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static List<List<Field>> getDocs(Row row, FieldManager fieldManager) throws IOException {
  List<Record> records = row.records;
  if (records == null) {
    return null;
  }
  int size = records.size();
  if (size == 0) {
    return null;
  }
  final String rowId = row.id;
  List<List<Field>> docs = new ArrayList<List<Field>>(size);
  for (int i = 0; i < size; i++) {
    Record record = records.get(i);
    List<Field> fields = getDoc(fieldManager, rowId, record);
    docs.add(fields);
  }
  List<Field> doc = docs.get(0);
  doc.add(new StringField(BlurConstants.PRIME_DOC, BlurConstants.PRIME_DOC_VALUE, Store.NO));
  return docs;
}
 
Example 14
private void addField(final Document doc, final SearchableField field, final String value) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO));
}
 
Example 15
Source Project: localization_nifi   Source File: IndexingAction.java    License: Apache License 2.0 5 votes vote down vote up
private void addField(final Document doc, final SearchableField field, final String value, final Store store) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), store));
}
 
Example 16
Source Project: lucene-solr   Source File: TestPhraseQuery.java    License: Apache License 2.0 5 votes vote down vote up
public void testSlopScoring() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory, 
      newIndexWriterConfig(new MockAnalyzer(random()))
        .setMergePolicy(newLogMergePolicy())
        .setSimilarity(new BM25Similarity()));

  Document doc = new Document();
  doc.add(newTextField("field", "foo firstname lastname foo", Field.Store.YES));
  writer.addDocument(doc);
  
  Document doc2 = new Document();
  doc2.add(newTextField("field", "foo firstname zzz lastname foo", Field.Store.YES));
  writer.addDocument(doc2);
  
  Document doc3 = new Document();
  doc3.add(newTextField("field", "foo firstname zzz yyy lastname foo", Field.Store.YES));
  writer.addDocument(doc3);
  
  IndexReader reader = writer.getReader();
  writer.close();

  IndexSearcher searcher = newSearcher(reader);
  searcher.setSimilarity(new ClassicSimilarity());
  PhraseQuery query = new PhraseQuery(Integer.MAX_VALUE, "field", "firstname", "lastname");
  ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
  assertEquals(3, hits.length);
  // Make sure that those matches where the terms appear closer to
  // each other get a higher score:
  assertEquals(1.0, hits[0].score, 0.01);
  assertEquals(0, hits[0].doc);
  assertEquals(0.63, hits[1].score, 0.01);
  assertEquals(1, hits[1].doc);
  assertEquals(0.47, hits[2].score, 0.01);
  assertEquals(2, hits[2].doc);
  QueryUtils.check(random(), query,searcher);
  reader.close();
  directory.close();
}
 
Example 17
Source Project: sdudoc   Source File: LuceneIndex.java    License: MIT License 5 votes vote down vote up
/**
 * 批量创创建document
 * @param list
 * @return
 */
public List  createDocumentAll(List list){
	List documents = new ArrayList();
	for(int i=0; i<list.size(); i++){
		Post post = (Post)list.get(i);
		Document document = new Document();
		document.add(new StringField("bookID",post.getBookID(),Store.YES));
		document.add(new TextField("bookTitle",post.getBookTitle(),Store.YES));
		document.add(new TextField("summary",post.getSummary(),Store.YES));
		document.add(new StringField("bookStyle",post.getBookStyle(),Store.YES));
		document.add(new StringField("authors",post.getAuthors(),Store.YES));
		documents.add(document);
	}
	return documents;
}
 
Example 18
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdatesOrder() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("upd", "t1", Store.NO));
  doc.add(new StringField("upd", "t2", Store.NO));
  doc.add(new NumericDocValuesField("f1", 1L));
  doc.add(new NumericDocValuesField("f2", 1L));
  writer.addDocument(doc);
  writer.updateNumericDocValue(new Term("upd", "t1"), "f1", 2L); // update f1 to 2
  writer.updateNumericDocValue(new Term("upd", "t1"), "f2", 2L); // update f2 to 2
  writer.updateNumericDocValue(new Term("upd", "t2"), "f1", 3L); // update f1 to 3
  writer.updateNumericDocValue(new Term("upd", "t2"), "f2", 3L); // update f2 to 3
  writer.updateNumericDocValue(new Term("upd", "t1"), "f1", 4L); // update f1 to 4 (but not f2)
  if (VERBOSE) {
    System.out.println("TEST: now close");
  }
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
  assertEquals(0, dvs.nextDoc());
  assertEquals(4, dvs.longValue());
  dvs = reader.leaves().get(0).reader().getNumericDocValues("f2");
  assertEquals(0, dvs.nextDoc());
  assertEquals(3, dvs.longValue());
  reader.close();
  
  dir.close();
}
 
Example 19
@Override
public boolean indexWord(String word) {
	Document doc = new Document();
	doc.add(new TextField(WORD_FIELD, word, Store.YES));
	try {
		writer.addDocument(doc);
	} catch (IOException e) {
		log.error("failed indexing word '" + word + "'", e);
		return false;
	}
	return true;
}
 
Example 20
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdateDocumentByMultipleTerms() throws Exception {
  // make sure the order of updates is respected, even when multiple terms affect same document
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("k1", "v1", Store.NO));
  doc.add(new StringField("k2", "v2", Store.NO));
  doc.add(new NumericDocValuesField("ndv", 5));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateNumericDocValue(new Term("k1", "v1"), "ndv", 17L);
  writer.updateNumericDocValue(new Term("k2", "v2"), "ndv", 3L);
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, ndv.nextDoc());
    assertEquals(3, ndv.longValue());
  }
  reader.close();
  dir.close();
}
 
Example 21
Source Project: lucene-solr   Source File: TestTopFieldCollector.java    License: Apache License 2.0 5 votes vote down vote up
public void testRelationVsTopDocsCount() throws Exception {
  Sort sort = new Sort(SortField.FIELD_SCORE, SortField.FIELD_DOC);
  try (Directory dir = newDirectory();
      IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE))) {
    Document doc = new Document();
    doc.add(new TextField("f", "foo bar", Store.NO));
    w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc));
    w.flush();
    w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc));
    w.flush();
    
    try (IndexReader reader = DirectoryReader.open(w)) {
      IndexSearcher searcher = new IndexSearcher(reader);
      TopFieldCollector collector = TopFieldCollector.create(sort, 2, 10);
      searcher.search(new TermQuery(new Term("f", "foo")), collector);
      assertEquals(10, collector.totalHits);
      assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation);
      
      collector = TopFieldCollector.create(sort, 2, 2);
      searcher.search(new TermQuery(new Term("f", "foo")), collector);
      assertTrue(10 >= collector.totalHits);
      assertEquals(TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO, collector.totalHitsRelation);
      
      collector = TopFieldCollector.create(sort, 10, 2);
      searcher.search(new TermQuery(new Term("f", "foo")), collector);
      assertEquals(10, collector.totalHits);
      assertEquals(TotalHits.Relation.EQUAL_TO, collector.totalHitsRelation);
    }
  }
}
 
Example 22
Source Project: NLIWOD   Source File: DBpediaIndex.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
private void addDocumentToIndex(final Resource resource, final String predicate, final String object) throws IOException {
	Document doc = new Document();
	doc.add(new StringField(FIELD_NAME_SUBJECT, resource.getURI(), Store.YES));
	doc.add(new StringField(FIELD_NAME_PREDICATE, predicate, Store.YES));
	doc.add(new TextField(FIELD_NAME_OBJECT, object, Store.YES));
	iwriter.addDocument(doc);
}
 
Example 23
private void addDocumentToIndex(final Resource resource, final String predicate, final RDFNode next) throws IOException {
	Document doc = new Document();
	doc.add(new StringField(FIELD_NAME_SUBJECT, resource.getURI(), Store.YES));
	doc.add(new StringField(FIELD_NAME_PREDICATE, predicate, Store.YES));
	doc.add(new TextField(FIELD_NAME_OBJECT, next.asLiteral().getString(), Store.YES));
	iwriter.addDocument(doc);
}
 
Example 24
Source Project: lucene-solr   Source File: TestPhraseQuery.java    License: Apache License 2.0 5 votes vote down vote up
public void testTopPhrases() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
  String[] docs = ArrayUtil.copyOfSubArray(DOCS, 0, DOCS.length);
  Collections.shuffle(Arrays.asList(docs), random());
  for (String value : DOCS) {
    Document doc = new Document();
    doc.add(new TextField("f", value, Store.NO));
    w.addDocument(doc);
  }
  IndexReader r = DirectoryReader.open(w);
  w.close();
  IndexSearcher searcher = newSearcher(r);
  for (Query query : Arrays.asList(
      new PhraseQuery("f", "b", "c"), // common phrase
      new PhraseQuery("f", "e", "f"), // always appear next to each other
      new PhraseQuery("f", "d", "d")  // repeated term
      )) {
    for (int topN = 1; topN <= 2; ++topN) {
      TopScoreDocCollector collector1 = TopScoreDocCollector.create(topN, null, Integer.MAX_VALUE);
      searcher.search(query, collector1);
      ScoreDoc[] hits1 = collector1.topDocs().scoreDocs;
      TopScoreDocCollector collector2 = TopScoreDocCollector.create(topN, null, 1);
      searcher.search(query, collector2);
      ScoreDoc[] hits2 = collector2.topDocs().scoreDocs;
      assertTrue("" + query, hits1.length > 0);
      CheckHits.checkEqual(query, hits1, hits2);
    }
  }
  r.close();
  dir.close();
}
 
Example 25
Source Project: lucene-solr   Source File: HighlighterPhraseTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testInOrderWithStopWords() throws IOException, InvalidTokenOffsetsException {
  MockAnalyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true,
      MockTokenFilter.ENGLISH_STOPSET);        
  final String TEXT = "the cd the ab the the the the the the the ab the cd the";
  final Directory directory = newDirectory();
  try (IndexWriter indexWriter = new IndexWriter(directory,
      newIndexWriterConfig(stopAnalyzer))) {
    final Document document = new Document();
    document.add(newTextField(FIELD, TEXT, Store.YES));
    indexWriter.addDocument(document);
  }
  try (IndexReader indexReader = DirectoryReader.open(directory)) {
    assertEquals(1, indexReader.numDocs());
    final IndexSearcher indexSearcher = newSearcher(indexReader);
    //equivalent of "ab the cd"
    final PhraseQuery phraseQuery = new PhraseQuery.Builder()
        .add(new Term(FIELD, "ab"), 0)
        .add(new Term(FIELD, "cd"), 2).build();

    TopDocs hits = indexSearcher.search(phraseQuery, 100);
    assertEquals(1, hits.totalHits.value);

    final Highlighter highlighter = new Highlighter(
        new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
        new QueryScorer(phraseQuery));
    String[] frags = highlighter.getBestFragments(stopAnalyzer, FIELD, TEXT, 10);
    assertEquals(1, frags.length);
    assertTrue("contains <B>ab</B> the <B>cd</B>",
        (frags[0].contains("<B>ab</B> the <B>cd</B>")));
    assertTrue("does not contain <B>cd</B> the <B>ab</B>",
        (!frags[0].contains("<B>cd</B> the <B>ab</B>")));
  } finally {
    directory.close();
  }
}
 
Example 26
Source Project: lucene-solr   Source File: SpatialPrefixTreeTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * A PrefixTree pruning optimization gone bad, applicable when optimize=true.
 * See <a href="https://issues.apache.org/jira/browse/LUCENE-4770">LUCENE-4770</a>.
 */
@Test
public void testBadPrefixTreePrune() throws Exception {

  trie = new QuadPrefixTree(ctx, 12);
  TermQueryPrefixTreeStrategy strategy = new TermQueryPrefixTreeStrategy(trie, "geo");
  Document doc = new Document();
  doc.add(new TextField("id", "1", Store.YES));

  Shape area = ctx.makeRectangle(-122.82, -122.78, 48.54, 48.56);

  Field[] fields = strategy.createIndexableFields(area, 0.025);
  for (Field field : fields) {
    doc.add(field);
  }
  addDocument(doc);

  Point upperleft = ctx.makePoint(-122.88, 48.54);
  Point lowerright = ctx.makePoint(-122.82, 48.62);

  Query query = strategy.makeQuery(new SpatialArgs(SpatialOperation.Intersects, ctx.makeRectangle(upperleft, lowerright)));

  commit();

  TopDocs search = indexSearcher.search(query, 10);
  ScoreDoc[] scoreDocs = search.scoreDocs;
  for (ScoreDoc scoreDoc : scoreDocs) {
    System.out.println(indexSearcher.doc(scoreDoc.doc));
  }

  assertEquals(1, search.totalHits.value);
}
 
Example 27
@Override
public Iterable<? extends Field> getFieldsForColumn(String family, Column column) {
  String name = getName(family, column.getName());
  List<Field> fields = new ArrayList<Field>();
  fields.add(new StoredField(name, column.getValue()));
  fields.add(new StringField(INTERNAL_FIELDNAME, family + "." + column.getValue(), Store.YES));
  return fields;
}
 
Example 28
Source Project: lucene-solr   Source File: TestOrdValues.java    License: Apache License 2.0 5 votes vote down vote up
private static void addDoc(RandomIndexWriter iw, int i) throws Exception {
  Document d = new Document();
  Field f;
  int scoreAndID = i + 1;

  FieldType customType = new FieldType(TextField.TYPE_STORED);
  customType.setTokenized(false);
  customType.setOmitNorms(true);
  
  f = newField(ID_FIELD, id2String(scoreAndID), customType); // for debug purposes
  d.add(f);
  d.add(new SortedDocValuesField(ID_FIELD, new BytesRef(id2String(scoreAndID))));

  FieldType customType2 = new FieldType(TextField.TYPE_NOT_STORED);
  customType2.setOmitNorms(true);
  f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
  d.add(f);

  f = new LegacyIntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
  d.add(f);
  d.add(new NumericDocValuesField(INT_FIELD, scoreAndID));

  f = new LegacyFloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
  d.add(f);
  d.add(new NumericDocValuesField(FLOAT_FIELD, Float.floatToRawIntBits(scoreAndID)));

  iw.addDocument(d);
  log("added: " + d);
}
 
Example 29
Source Project: lucene-solr   Source File: TestBlockJoin.java    License: Apache License 2.0 5 votes vote down vote up
private Document makeJob(String skill, int year) {
  Document job = new Document();
  job.add(newStringField("skill", skill, Field.Store.YES));
  job.add(new IntPoint("year", year));
  job.add(new StoredField("year", year));
  return job;
}
 
Example 30
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdatesOrder() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("upd", "t1", Store.NO));
  doc.add(new StringField("upd", "t2", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(2L)); // update f1 to 2
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f2", toBytes(2L)); // update f2 to 2
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f1", toBytes(3L)); // update f1 to 3
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f2", toBytes(3L)); // update f2 to 3
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(4L)); // update f1 to 4 (but not f2)
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(4, getValue(bdv));
  bdv = reader.leaves().get(0).reader().getBinaryDocValues("f2");
  assertEquals(0, bdv.nextDoc());
  assertEquals(3, getValue(bdv));
  reader.close();
  
  dir.close();
}