Java Code Examples for org.apache.lucene.index.IndexWriter#addDocuments()

The following examples show how to use org.apache.lucene.index.IndexWriter#addDocuments() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: TestBlockJoinValidation.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
  final IndexWriter indexWriter = new IndexWriter(directory, config);
  for (int i = 0; i < AMOUNT_OF_SEGMENTS; i++) {
    List<Document> segmentDocs = createDocsForSegment(i);
    indexWriter.addDocuments(segmentDocs);
    indexWriter.commit();
  }
  indexReader = DirectoryReader.open(indexWriter);
  indexWriter.close();
  indexSearcher = new IndexSearcher(indexReader);
  parentsFilter = new QueryBitSetProducer(new WildcardQuery(new Term("parent", "*")));
}
 
Example 2
Source File: TestTopDocsCollector.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSharedCountCollectorManager() throws Exception {
  Query q = new MatchAllDocsQuery();
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  TopDocsCollector<ScoreDoc> collector = doSearchWithThreshold( 5, 10, q, reader);
  TopDocs tdc = doConcurrentSearchWithThreshold(5, 10, q, reader);
  TopDocs tdc2 = collector.topDocs();

  CheckHits.checkEqual(q, tdc.scoreDocs, tdc2.scoreDocs);

  reader.close();
  dir.close();
}
 
Example 3
Source File: DirectUpdateHandler2.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void allowDuplicateUpdate(AddUpdateCommand cmd) throws IOException {
  RefCounted<IndexWriter> iw = solrCoreState.getIndexWriter(core);
  try {
    IndexWriter writer = iw.get();
    Iterable<Document> nestedDocs = cmd.getLuceneDocsIfNested();
    if (nestedDocs != null) {
      writer.addDocuments(nestedDocs);
    } else {
      writer.addDocument(cmd.getLuceneDocument());
    }
    if (ulog != null) ulog.add(cmd);

  } finally {
    iw.decref();
  }

}
 
Example 4
Source File: test.java    From vscode-extension with MIT License 5 votes vote down vote up
private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
    numDocAppends.inc(docs.size());
}
 
Example 5
Source File: SuperQueryTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
public static Directory createIndex() throws CorruptIndexException, LockObtainFailedException, IOException {
  Directory directory = new RAMDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)),
      newDocument(newStringField(ROW_ID, "1"), newStringField(PERSON_NAME, NAME1)), newDocument(newStringField(ROW_ID, "1"), newStringField(ADDRESS_STREET, STREET1)))));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "2"), newStringField(PERSON_NAME, NAME2)),
      newDocument(newStringField(ROW_ID, "2"), newStringField(ADDRESS_STREET, STREET1)))));
  writer.addDocuments(addPrime(Arrays.asList(newDocument(newStringField(ROW_ID, "3"), newStringField(PERSON_NAME, NAME1)),
      newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET1)), newDocument(newStringField(ROW_ID, "3"), newStringField(ADDRESS_STREET, STREET2)))));
  writer.close();
  return directory;
}
 
Example 6
Source File: RandomSuperQueryTest.java    From incubator-retired-blur with Apache License 2.0 5 votes vote down vote up
private Directory createIndex(Random random, Collection<Query> sampler) throws CorruptIndexException, LockObtainFailedException, IOException {
  Directory directory = new RAMDirectory();
  String[] columnFamilies = genWords(random, MIN_NUM_COL_FAM, MAX_NUM_COL_FAM, "colfam");
  Map<String, String[]> columns = new HashMap<String, String[]>();
  for (int i = 0; i < columnFamilies.length; i++) {
    columns.put(columnFamilies[i], genWords(random, MIN_NUM_COLS, MAX_NUM_COLS, "col"));
  }
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(LUCENE_VERSION, new StandardAnalyzer(LUCENE_VERSION)));
  int numberOfDocs = random.nextInt(MAX_NUM_OF_DOCS) + 1;
  for (int i = 0; i < numberOfDocs; i++) {
    writer.addDocuments(generatSuperDoc(random, columns, sampler));
  }
  writer.close();
  return directory;
}
 
Example 7
Source File: InternalEngine.java    From crate with Apache License 2.0 5 votes vote down vote up
private void addDocs(final List<ParseContext.Document> docs, final IndexWriter indexWriter) throws IOException {
    if (docs.size() > 1) {
        indexWriter.addDocuments(docs);
    } else {
        indexWriter.addDocument(docs.get(0));
    }
    numDocAppends.inc(docs.size());
}
 
Example 8
Source File: TestBlockJoin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testEmptyChildFilter() throws Exception {
  final Directory dir = newDirectory();
  final IndexWriterConfig config = new IndexWriterConfig(new MockAnalyzer(random()));
  config.setMergePolicy(NoMergePolicy.INSTANCE);
  // we don't want to merge - since we rely on certain segment setup
  final IndexWriter w = new IndexWriter(dir, config);

  final List<Document> docs = new ArrayList<>();

  docs.add(makeJob("java", 2007));
  docs.add(makeJob("python", 2010));
  docs.add(makeResume("Lisa", "United Kingdom"));
  w.addDocuments(docs);

  docs.clear();
  docs.add(makeJob("ruby", 2005));
  docs.add(makeJob("java", 2006));
  docs.add(makeResume("Frank", "United States"));
  w.addDocuments(docs);
  w.commit();

  IndexReader r = DirectoryReader.open(w);
  w.close();
  IndexSearcher s = newSearcher(r);
  BitSetProducer parentsFilter = new QueryBitSetProducer(new TermQuery(new Term("docType", "resume")));
  CheckJoinIndex.check(r, parentsFilter);

  BooleanQuery.Builder childQuery = new BooleanQuery.Builder();
  childQuery.add(new BooleanClause(new TermQuery(new Term("skill", "java")), Occur.MUST));
  childQuery.add(new BooleanClause(IntPoint.newRangeQuery("year", 2006, 2011), Occur.MUST));

  ToParentBlockJoinQuery childJoinQuery = new ToParentBlockJoinQuery(childQuery.build(), parentsFilter, ScoreMode.Avg);

  BooleanQuery.Builder fullQuery = new BooleanQuery.Builder();
  fullQuery.add(new BooleanClause(childJoinQuery, Occur.MUST));
  fullQuery.add(new BooleanClause(new MatchAllDocsQuery(), Occur.MUST));
  TopDocs topDocs = s.search(fullQuery.build(), 2);
  assertEquals(2, topDocs.totalHits.value);
  assertEquals(asSet("Lisa", "Frank"),
      asSet(s.doc(topDocs.scoreDocs[0].doc).get("name"), s.doc(topDocs.scoreDocs[1].doc).get("name")));

  ParentChildrenBlockJoinQuery childrenQuery =
      new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[0].doc);
  TopDocs matchingChildren = s.search(childrenQuery, 1);
  assertEquals(1, matchingChildren.totalHits.value);
  assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));

  childrenQuery = new ParentChildrenBlockJoinQuery(parentsFilter, childQuery.build(), topDocs.scoreDocs[1].doc);
  matchingChildren = s.search(childrenQuery, 1);
  assertEquals(1, matchingChildren.totalHits.value);
  assertEquals("java", s.doc(matchingChildren.scoreDocs[0].doc).get("skill"));

  r.close();
  dir.close();
}
 
Example 9
Source File: TestTopDocsCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testSetMinCompetitiveScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, 2);
  ScoreAndDoc scorer = new ScoreAndDoc();

  LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
  leafCollector.setScorer(scorer);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 1;
  scorer.score = 2;
  leafCollector.collect(1);
  assertNull(scorer.minCompetitiveScore);
  
  scorer.doc = 2;
  scorer.score = 3;
  leafCollector.collect(2);
  assertEquals(Math.nextUp(2f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 3;
  scorer.score = 0.5f;
  // Make sure we do not call setMinCompetitiveScore for non-competitive hits
  scorer.minCompetitiveScore = Float.NaN;
  leafCollector.collect(3);
  assertTrue(Float.isNaN(scorer.minCompetitiveScore));

  scorer.doc = 4;
  scorer.score = 4;
  leafCollector.collect(4);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  // Make sure the min score is set on scorers on new segments
  scorer = new ScoreAndDoc();
  leafCollector = collector.getLeafCollector(reader.leaves().get(1));
  leafCollector.setScorer(scorer);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertEquals(Math.nextUp(3f), scorer.minCompetitiveScore, 0f);

  scorer.doc = 1;
  scorer.score = 4;
  leafCollector.collect(1);
  assertEquals(Math.nextUp(4f), scorer.minCompetitiveScore, 0f);

  reader.close();
  dir.close();
}
 
Example 10
Source File: TestTopDocsCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testTotalHits() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
    TopScoreDocCollector collector = TopScoreDocCollector.create(2, null, totalHitsThreshold);
    ScoreAndDoc scorer = new ScoreAndDoc();

    LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
    leafCollector.setScorer(scorer);

    scorer.doc = 0;
    scorer.score = 3;
    leafCollector.collect(0);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    leafCollector = collector.getLeafCollector(reader.leaves().get(1));
    leafCollector.setScorer(scorer);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    scorer.doc = 5;
    scorer.score = 4;
    leafCollector.collect(1);

    TopDocs topDocs = collector.topDocs();
    assertEquals(4, topDocs.totalHits.value);
    assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
    assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
  }

  reader.close();
  dir.close();
}
 
Example 11
Source File: TestTopFieldCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testSetMinCompetitiveScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
  TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, 2);
  ScoreAndDoc scorer = new ScoreAndDoc();

  LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
  leafCollector.setScorer(scorer);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertNull(scorer.minCompetitiveScore);

  scorer.doc = 1;
  scorer.score = 2;
  leafCollector.collect(1);
  assertNull(scorer.minCompetitiveScore);
  
  scorer.doc = 2;
  scorer.score = 3;
  leafCollector.collect(2);
  assertEquals(2f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 3;
  scorer.score = 0.5f;
  // Make sure we do not call setMinCompetitiveScore for non-competitive hits
  scorer.minCompetitiveScore = Float.NaN;
  leafCollector.collect(3);
  assertTrue(Float.isNaN(scorer.minCompetitiveScore));

  scorer.doc = 4;
  scorer.score = 4;
  leafCollector.collect(4);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  // Make sure the min score is set on scorers on new segments
  scorer = new ScoreAndDoc();
  leafCollector = collector.getLeafCollector(reader.leaves().get(1));
  leafCollector.setScorer(scorer);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 0;
  scorer.score = 1;
  leafCollector.collect(0);
  assertEquals(3f, scorer.minCompetitiveScore, 0f);

  scorer.doc = 1;
  scorer.score = 4;
  leafCollector.collect(1);
  assertEquals(4f, scorer.minCompetitiveScore, 0f);

  reader.close();
  dir.close();
}
 
Example 12
Source File: TestTopFieldCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testTotalHitsWithScore() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig().setMergePolicy(NoMergePolicy.INSTANCE));
  Document doc = new Document();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc));
  w.flush();
  w.addDocuments(Arrays.asList(doc, doc, doc, doc, doc, doc));
  w.flush();
  IndexReader reader = DirectoryReader.open(w);
  assertEquals(2, reader.leaves().size());
  w.close();

  for (int totalHitsThreshold = 0; totalHitsThreshold < 20; ++ totalHitsThreshold) {
    Sort sort = new Sort(FIELD_SCORE, new SortField("foo", SortField.Type.LONG));
    TopFieldCollector collector = TopFieldCollector.create(sort, 2, null, totalHitsThreshold);
    ScoreAndDoc scorer = new ScoreAndDoc();

    LeafCollector leafCollector = collector.getLeafCollector(reader.leaves().get(0));
    leafCollector.setScorer(scorer);

    scorer.doc = 0;
    scorer.score = 3;
    leafCollector.collect(0);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    leafCollector = collector.getLeafCollector(reader.leaves().get(1));
    leafCollector.setScorer(scorer);

    scorer.doc = 1;
    scorer.score = 3;
    leafCollector.collect(1);

    scorer.doc = 5;
    scorer.score = 4;
    leafCollector.collect(1);

    TopDocs topDocs = collector.topDocs();
    assertEquals(totalHitsThreshold < 4, scorer.minCompetitiveScore != null);
    assertEquals(new TotalHits(4, totalHitsThreshold < 4 ? TotalHits.Relation.GREATER_THAN_OR_EQUAL_TO : TotalHits.Relation.EQUAL_TO), topDocs.totalHits);
  }

  reader.close();
  dir.close();
}