Java Code Examples for org.apache.lucene.document.StringField

The following examples show how to use org.apache.lucene.document.StringField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   Source File: TestEmptyTokenStream.java    License: Apache License 2.0 6 votes vote down vote up
public void testIndexWriter_LUCENE4656() throws IOException {
  Directory directory = newDirectory();
  IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(null));

  TokenStream ts = new EmptyTokenStream();
  assertFalse(ts.hasAttribute(TermToBytesRefAttribute.class));

  Document doc = new Document();
  doc.add(new StringField("id", "0", Field.Store.YES));
  doc.add(new TextField("description", ts));
  
  // this should not fail because we have no TermToBytesRefAttribute
  writer.addDocument(doc);
  
  assertEquals(1, writer.getDocStats().numDocs);

  writer.close();
  directory.close();
}
 
Example 2
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateNumericDVFieldWithSameNameAsPostingField() throws Exception {
  // this used to fail because FieldInfos.Builder neglected to update
  // globalFieldMaps.docValuesTypes map
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new NumericDocValuesField("f", 5));
  writer.addDocument(doc);
  writer.commit();
  writer.updateNumericDocValue(new Term("f", "mock-value"), "f", 17L);
  writer.close();
  
  DirectoryReader r = DirectoryReader.open(dir);
  NumericDocValues ndv = r.leaves().get(0).reader().getNumericDocValues("f");
  assertEquals(0, ndv.nextDoc());
  assertEquals(17, ndv.longValue());
  r.close();
  
  dir.close();
}
 
Example 3
Source Project: dremio-oss   Source File: SimpleDocumentWriter.java    License: Apache License 2.0 6 votes vote down vote up
private void addToDoc(IndexKey key, byte[]... values){
  Preconditions.checkArgument(key.getValueType() == String.class);
  final boolean sorted = key.isSorted();
  if (sorted) {
    checkIfSorted(key, (Object[]) values);
  }

  checkIfMultiValueField(key, (Object[]) values);

  final String indexFieldName = key.getIndexFieldName();
  final Store stored = key.isStored() ? Store.YES : Store.NO;
  for (final byte[] value : values) {
    if (value == null) {
      continue;
    }
    final BytesRef truncatedValue = new BytesRef(value,0, Math.min(value.length, MAX_STRING_LENGTH));
    doc.add(new StringField(indexFieldName, truncatedValue, stored));
  }

  if (sorted && values.length == 1 && values[0] != null) {
    Preconditions.checkArgument(key.getSortedValueType() == SearchFieldSorting.FieldType.STRING);
    doc.add(new SortedDocValuesField(indexFieldName, new BytesRef(values[0])));
  }
}
 
Example 4
Source Project: dremio-oss   Source File: TestLuceneIndexer.java    License: Apache License 2.0 6 votes vote down vote up
@Test(expected = StaleSearcherException.class)
public void testSearcherCacheTTL() throws Exception {
  try (LuceneSearchIndex index = new LuceneSearchIndex(null, "multithreaded-search", true, CommitWrapper.NO_OP, 500)) {
    for (int i = 0; i < 10; ++i) {
      final Document doc = new Document();
      doc.add(
          new StringField(CoreIndexedStore.ID_FIELD_NAME, new BytesRef(Integer.toString(i).getBytes()), Store.YES));
      doc.add(new StringField("user", "u1", Field.Store.YES));
      index.add(doc);
    }

    Query query = new TermQuery(new Term("user", "u1"));
    LuceneSearchIndex.SearchHandle searchHandle = index.createSearchHandle();
    List<Doc> docs = index.search(searchHandle, query, 4, new Sort(), 0);
    assertEquals(4, docs.size());

    // sleep to force cache expiry.
    Thread.sleep(1000);

    docs = index.searchAfter(searchHandle, query, 6, new Sort(), docs.get(3));
    assertEquals(6, docs.size());

    searchHandle.close();
  }
}
 
Example 5
public void testSoftDeleteWithTryUpdateDocValue() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_delete")
      .setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, newLogMergePolicy()));
  IndexWriter writer = new IndexWriter(dir, config);
  SearcherManager sm = new SearcherManager(writer, new SearcherFactory());
  Document d = new Document();
  d.add(new StringField("id", "0", Field.Store.YES));
  writer.addDocument(d);
  sm.maybeRefreshBlocking();
  doUpdate(new Term("id", "0"), writer,
      new NumericDocValuesField("soft_delete", 1), new NumericDocValuesField("other-field", 1));
  sm.maybeRefreshBlocking();
  assertEquals(1, writer.cloneSegmentInfos().size());
  SegmentCommitInfo si = writer.cloneSegmentInfos().info(0);
  assertEquals(1, si.getSoftDelCount());
  assertEquals(1, si.info.maxDoc());
  IOUtils.close(sm, writer, dir);
}
 
Example 6
Source Project: lucene-solr   Source File: TestFieldCacheSort.java    License: Apache License 2.0 6 votes vote down vote up
/** test that we throw exception on multi-valued field, creates corrupt reader, use SORTED_SET instead */
public void testMultiValuedField() throws IOException {
  Directory indexStore = newDirectory();
  IndexWriter writer = new IndexWriter(indexStore, newIndexWriterConfig(new MockAnalyzer(random())));
  for(int i=0; i<5; i++) {
      Document doc = new Document();
      doc.add(new StringField("string", "a"+i, Field.Store.NO));
      doc.add(new StringField("string", "b"+i, Field.Store.NO));
      writer.addDocument(doc);
  }
  writer.forceMerge(1); // enforce one segment to have a higher unique term count in all cases
  writer.close();
  Sort sort = new Sort(
      new SortField("string", SortField.Type.STRING),
      SortField.FIELD_DOC);
  IndexReader reader = UninvertingReader.wrap(DirectoryReader.open(indexStore),
                       Collections.singletonMap("string", Type.SORTED));
  IndexSearcher searcher = new IndexSearcher(reader);
  expectThrows(IllegalStateException.class, () -> {
    searcher.search(new MatchAllDocsQuery(), 500, sort);
  });
  reader.close();
  indexStore.close();
}
 
Example 7
Source Project: lucene-solr   Source File: TestSpanNotQuery.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoPositions() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  Document doc = new Document();
  doc.add(new StringField("foo", "bar", Field.Store.NO));
  iw.addDocument(doc);
  
  IndexReader ir = iw.getReader();
  iw.close();
  
  IndexSearcher is = new IndexSearcher(ir);
  SpanTermQuery query = new SpanTermQuery(new Term("foo", "bar"));
  SpanTermQuery query2 = new SpanTermQuery(new Term("foo", "baz"));

  IllegalStateException expected = expectThrows(IllegalStateException.class, () -> {
    is.search(new SpanNotQuery(query, query2), 5);
  });
  assertTrue(expected.getMessage().contains("was indexed without position data"));

  ir.close();
  dir.close();
}
 
Example 8
Source Project: modernmt   Source File: DocumentBuilder.java    License: Apache License 2.0 6 votes vote down vote up
public static Document newInstance(UUID owner, long memory, LanguageDirection direction, Reader contentReader) {
    Document document = new Document();
    document.add(new StringField(DOC_ID_FIELD, makeId(memory, direction), Field.Store.NO));
    document.add(new LongField(MEMORY_FIELD, memory, Field.Store.YES));

    if (owner != null) {
        document.add(new LongField(OWNER_MSB_FIELD, owner.getMostSignificantBits(), Field.Store.NO));
        document.add(new LongField(OWNER_LSB_FIELD, owner.getLeastSignificantBits(), Field.Store.NO));
    } else {
        document.add(new LongField(OWNER_MSB_FIELD, 0L, Field.Store.NO));
        document.add(new LongField(OWNER_LSB_FIELD, 0L, Field.Store.NO));
    }

    document.add(new CorpusContentField(makeContentFieldName(direction), contentReader));

    return document;
}
 
Example 9
Source Project: SnowGraph   Source File: InMemoryIndex.java    License: Apache License 2.0 6 votes vote down vote up
public InMemoryIndex(Map<String,String> id2Text){
    Analyzer analyzer = new EnglishAnalyzer();
    IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
    iwc.setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    try {
        IndexWriter writer = new IndexWriter(directory, iwc);
        for (String id:id2Text.keySet()) {
            Document doc=new Document();
            doc.add(new StringField("id", id, Field.Store.YES));
            doc.add(new TextField("content", id2Text.get(id), Field.Store.YES));
            writer.addDocument(doc);
        }
        writer.close();
    } catch (IOException e) {
        e.printStackTrace();
    }
}
 
Example 10
Source Project: lucene-solr   Source File: TestIndexSorting.java    License: Apache License 2.0 6 votes vote down vote up
public void testBadDVUpdate() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  Sort indexSort = new Sort(new SortField("foo", SortField.Type.LONG));
  iwc.setIndexSort(indexSort);
  IndexWriter w = new IndexWriter(dir, iwc);
  Document doc = new Document();
  doc.add(new StringField("id", new BytesRef("0"), Store.NO));
  doc.add(new NumericDocValuesField("foo", random().nextInt()));
  w.addDocument(doc);
  w.commit();
  IllegalArgumentException exc = expectThrows(IllegalArgumentException.class,
      () -> w.updateDocValues(new Term("id", "0"), new NumericDocValuesField("foo", -1)));
  assertEquals(exc.getMessage(), "cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");
  exc = expectThrows(IllegalArgumentException.class,
      () -> w.updateNumericDocValue(new Term("id", "0"), "foo", -1));
  assertEquals(exc.getMessage(), "cannot update docvalues field involved in the index sort, field=foo, sort=<long: \"foo\">");
  w.close();
  dir.close();
}
 
Example 11
Source Project: lucene-solr   Source File: TestTaxonomyFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
private void indexTwoDocs(TaxonomyWriter taxoWriter, IndexWriter indexWriter, FacetsConfig config, boolean withContent) throws Exception {
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    if (withContent) {
      doc.add(new StringField("f", "a", Field.Store.NO));
    }
    if (config != null) {
      doc.add(new FacetField("A", Integer.toString(i)));
      indexWriter.addDocument(config.build(taxoWriter, doc));
    } else {
      indexWriter.addDocument(doc);
    }
  }
  
  indexWriter.commit();
}
 
Example 12
Source Project: lucene-solr   Source File: TestMultiTermsEnum.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoTermsInField() throws Exception {
  Directory directory = new ByteBuffersDirectory();
  IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random())));
  Document document = new Document();
  document.add(new StringField("deleted", "0", Field.Store.YES));
  writer.addDocument(document);

  DirectoryReader reader = DirectoryReader.open(writer);
  writer.close();

  Directory directory2 = new ByteBuffersDirectory();
  writer = new IndexWriter(directory2, new IndexWriterConfig(new MockAnalyzer(random())));
  
  List<LeafReaderContext> leaves = reader.leaves();
  CodecReader[] codecReaders = new CodecReader[leaves.size()];
  for (int i = 0; i < leaves.size(); i++) {
    codecReaders[i] = new MigratingCodecReader((CodecReader) leaves.get(i).reader());
  }

  writer.addIndexes(codecReaders); // <- bang

  IOUtils.close(writer, reader, directory);
}
 
Example 13
Source Project: lucene-solr   Source File: TestMemoryIndex.java    License: Apache License 2.0 6 votes vote down vote up
@Test
public void testBuildFromDocument() {

  Document doc = new Document();
  doc.add(new TextField("field1", "some text", Field.Store.NO));
  doc.add(new TextField("field1", "some more text", Field.Store.NO));
  doc.add(new StringField("field2", "untokenized text", Field.Store.NO));

  analyzer.setPositionIncrementGap(100);

  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);

  assertThat(mi.search(new TermQuery(new Term("field1", "text"))), not(0.0f));
  assertThat(mi.search(new TermQuery(new Term("field2", "text"))), is(0.0f));
  assertThat(mi.search(new TermQuery(new Term("field2", "untokenized text"))), not(0.0f));

  assertThat(mi.search(new PhraseQuery("field1", "some", "more", "text")), not(0.0f));
  assertThat(mi.search(new PhraseQuery("field1", "some", "text")), not(0.0f));
  assertThat(mi.search(new PhraseQuery("field1", "text", "some")), is(0.0f));

}
 
Example 14
Source Project: lucene-solr   Source File: StrategyTestCase.java    License: Apache License 2.0 6 votes vote down vote up
protected List<Document> getDocuments(Iterator<SpatialTestData> sampleData) {
  List<Document> documents = new ArrayList<>();
  while (sampleData.hasNext()) {
    SpatialTestData data = sampleData.next();
    Document document = new Document();
    document.add(new StringField("id", data.id, Field.Store.YES));
    document.add(new StringField("name", data.name, Field.Store.YES));
    Shape shape = data.shape;
    shape = convertShapeFromGetDocuments(shape);
    if (shape != null) {
      for (Field f : strategy.createIndexableFields(shape)) {
        document.add(f);
      }
      if (storeShape)//just for diagnostics
        document.add(new StoredField(strategy.getFieldName(), shape.toString()));
    }

    documents.add(document);
  }
  return documents;
}
 
Example 15
Source Project: oodt   Source File: LuceneWorkflowInstanceRepository.java    License: Apache License 2.0 6 votes vote down vote up
private void addInstanceMetadataToDoc(Document doc, Metadata met) {
    if (met != null && met.getMap().keySet().size() > 0) {
        for (String metKey : met.getMap().keySet()) {
            List metVals = met.getAllMetadata(metKey);
            if (metVals != null && metVals.size() > 0) {
                for (Object metVal1 : metVals) {
                    String metVal = (String) metVal1;
                    doc.add(new Field(metKey, metVal, StringField.TYPE_STORED));
                }

                // now index the field name so that we can use it to
                // look it up when converting from doc to
                // WorkflowInstance
                doc.add(new Field("workflow_inst_met_flds", metKey,
                        StringField.TYPE_STORED));

            }
        }
    }
}
 
Example 16
Source Project: lucene-solr   Source File: AnalyzingInfixSuggester.java    License: Apache License 2.0 6 votes vote down vote up
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 
Example 17
Source Project: lucene-solr   Source File: TestDocValuesStatsCollector.java    License: Apache License 2.0 6 votes vote down vote up
public void testOneDoc() throws IOException {
  try (Directory dir = newDirectory();
      IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig())) {
    String field = "numeric";
    Document doc = new Document();
    doc.add(new NumericDocValuesField(field, 1));
    doc.add(new StringField("id", "doc1", Store.NO));
    indexWriter.addDocument(doc);

    try (DirectoryReader reader = DirectoryReader.open(indexWriter)) {
      IndexSearcher searcher = new IndexSearcher(reader);
      LongDocValuesStats stats = new LongDocValuesStats(field);
      searcher.search(new MatchAllDocsQuery(), new DocValuesStatsCollector(stats));

      assertEquals(1, stats.count());
      assertEquals(0, stats.missing());
      assertEquals(1, stats.max().longValue());
      assertEquals(1, stats.min().longValue());
      assertEquals(1, stats.sum().longValue());
      assertEquals(1, stats.mean(), 0.0001);
      assertEquals(0, stats.variance(), 0.0001);
      assertEquals(0, stats.stdev(), 0.0001);
    }
  }
}
 
Example 18
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateAllDeletedSegment() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new NumericDocValuesField("f1", 1L));
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.commit();
  writer.deleteDocuments(new Term("id", "doc")); // delete all docs in the first segment
  writer.addDocument(doc);
  writer.updateNumericDocValue(new Term("id", "doc"), "f1", 2L);
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
  assertEquals(0, dvs.nextDoc());
  assertEquals(2, dvs.longValue());
  
  reader.close();
  
  dir.close();
}
 
Example 19
Source Project: lucene-solr   Source File: FacetsConfig.java    License: Apache License 2.0 6 votes vote down vote up
private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));

      FacetsConfig.DimConfig ft = getDimConfig(facetField.dim);        
      if (ft.requireDimensionDrillDown) {
        doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
      }
    }
  }
}
 
Example 20
Source Project: lucene4ir   Source File: TRECNEWSDocumentIndexer.java    License: Apache License 2.0 5 votes vote down vote up
private void initFields() {
    docnumField = new StringField(Lucene4IRConstants.FIELD_DOCNUM, "", Field.Store.YES);
    if (indexPositions) {
        titleField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_TITLE, "", Field.Store.YES);
        textField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_CONTENT, "", Field.Store.YES);
        allField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_ALL, "", Field.Store.YES);
        authorField = new TermVectorEnabledTextField(Lucene4IRConstants.FIELD_AUTHOR, "", Field.Store.YES);
    } else {
        titleField = new TextField(Lucene4IRConstants.FIELD_TITLE, "", Field.Store.YES);
        textField = new TextField(Lucene4IRConstants.FIELD_CONTENT, "", Field.Store.YES);
        allField = new TextField(Lucene4IRConstants.FIELD_ALL, "", Field.Store.YES);
        authorField = new TextField(Lucene4IRConstants.FIELD_AUTHOR, "", Field.Store.YES);
    }
}
 
Example 21
Source Project: inception   Source File: MtasDocumentIndex.java    License: Apache License 2.0 5 votes vote down vote up
private void indexDocument(String aDocumentTitle, long aSourceDocumentId,
        long aAnnotationDocumentId, String aUser, byte[] aBinaryCas)
    throws IOException
{
    // Calculate timestamp that will be indexed
    String timestamp = DateTools.dateToString(new Date(), DateTools.Resolution.MILLISECOND);
    
    log.trace(
            "Indexing document in project [{}]({}). sourceId: {}, annotationId: {}, "
            + "user: {} timestamp: {}",
            project.getName(), project.getId(), aSourceDocumentId, aAnnotationDocumentId,
            aUser, timestamp);
    
    IndexWriter indexWriter = getIndexWriter();
    
    // Prepare bytearray with document content to be indexed
    String encodedCAS = new String(MtasUtils.bytesToChars(aBinaryCas));

    // Create new Lucene document
    Document doc = new Document();
    
    // Add indexed fields
    doc.add(new StringField(FIELD_ID, String.valueOf(aSourceDocumentId) + "/"
            + String.valueOf(aAnnotationDocumentId), Field.Store.YES));
    doc.add(new StringField(FIELD_SOURCE_DOCUMENT_ID, String.valueOf(aSourceDocumentId),
            Field.Store.YES));
    doc.add(new StringField(FIELD_ANNOTATION_DOCUMENT_ID,
            String.valueOf(aAnnotationDocumentId), Field.Store.YES));
    doc.add(new StringField(FIELD_TITLE, aDocumentTitle, Field.Store.YES));
    doc.add(new StringField(FIELD_USER, aUser, Field.Store.YES));
    doc.add(new StringField(FIELD_TIMESTAMP, timestamp, Field.Store.YES));
    doc.add(new TextField(FIELD_CONTENT, encodedCAS, Field.Store.NO));

    // Add document to the Lucene index
    indexWriter.addDocument(doc);

    scheduleCommit();
}
 
Example 22
private void addField(final Document doc, final SearchableField field, final String value) {
    if (value == null || (!field.isAttribute() && !searchableEventFields.contains(field))) {
        return;
    }

    doc.add(new StringField(field.getSearchableFieldName(), value.toLowerCase(), Store.NO));
}
 
Example 23
Source Project: localization_nifi   Source File: TestCachingIndexManager.java    License: Apache License 2.0 5 votes vote down vote up
@Test
public void test() throws IOException {
    // Create and IndexWriter and add a document to the index, then close the writer.
    // This gives us something that we can query.
    final EventIndexWriter writer = manager.borrowIndexWriter(indexDir);
    final Document doc = new Document();
    doc.add(new StringField("unit test", "true", Store.YES));
    writer.index(doc, 1000);
    manager.returnIndexWriter(writer);

    // Get an Index Searcher that we can use to query the index.
    final EventIndexSearcher cachedSearcher = manager.borrowIndexSearcher(indexDir);

    // Ensure that we get the expected results.
    assertCount(cachedSearcher, 1);

    // While we already have an Index Searcher, get a writer for the same index.
    // This will cause the Index Searcher to be marked as poisoned.
    final EventIndexWriter writer2 = manager.borrowIndexWriter(indexDir);

    // Obtain a new Index Searcher with the writer open. This Index Searcher should *NOT*
    // be the same as the previous searcher because the new one will be a Near-Real-Time Index Searcher
    // while the other is not.
    final EventIndexSearcher nrtSearcher = manager.borrowIndexSearcher(indexDir);
    assertNotSame(cachedSearcher, nrtSearcher);

    // Ensure that we get the expected query results.
    assertCount(nrtSearcher, 1);

    // Return the writer, so that there is no longer an active writer for the index.
    manager.returnIndexWriter(writer2);

    // Ensure that we still get the same result.
    assertCount(cachedSearcher, 1);
    manager.returnIndexSearcher(cachedSearcher);

    // Ensure that our near-real-time index searcher still gets the same result.
    assertCount(nrtSearcher, 1);
    manager.returnIndexSearcher(nrtSearcher);
}
 
Example 24
Source Project: everywhere   Source File: BaseIndex.java    License: Apache License 2.0 5 votes vote down vote up
private static void indexDoc(IndexWriter writer, FileBean t) throws Exception {
        Document doc = new Document();
        if (t.getContent() != null) {
            doc.add(new TextField(LuceneConstants.PATH, t.getFilepath(), Field.Store.YES));
            doc.add(new StringField(LuceneConstants.MODIFIED, UtilsTool.getDateStrByLastModified(t.getLastModified()), Field.Store.YES));
            doc.add(new TextField(LuceneConstants.CONTENT, t.getContent(), CommonConstants.IS_OPEN_CONTEXT ? Field.Store.YES : Field.Store.NO));
//            System.out.println("added to document:" + t.getFilepath());
            if (writer.getConfig().getOpenMode() == IndexWriterConfig.OpenMode.CREATE){
                writer.addDocument(doc);
            } else{
                writer.updateDocument(new Term(LuceneConstants.PATH, t.getFilepath()), doc);
            }
        }
    }
 
Example 25
Source Project: lucene-solr   Source File: TestNumericDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdatesOrder() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("upd", "t1", Store.NO));
  doc.add(new StringField("upd", "t2", Store.NO));
  doc.add(new NumericDocValuesField("f1", 1L));
  doc.add(new NumericDocValuesField("f2", 1L));
  writer.addDocument(doc);
  writer.updateNumericDocValue(new Term("upd", "t1"), "f1", 2L); // update f1 to 2
  writer.updateNumericDocValue(new Term("upd", "t1"), "f2", 2L); // update f2 to 2
  writer.updateNumericDocValue(new Term("upd", "t2"), "f1", 3L); // update f1 to 3
  writer.updateNumericDocValue(new Term("upd", "t2"), "f2", 3L); // update f2 to 3
  writer.updateNumericDocValue(new Term("upd", "t1"), "f1", 4L); // update f1 to 4 (but not f2)
  if (VERBOSE) {
    System.out.println("TEST: now close");
  }
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  NumericDocValues dvs = reader.leaves().get(0).reader().getNumericDocValues("f1");
  assertEquals(0, dvs.nextDoc());
  assertEquals(4, dvs.longValue());
  dvs = reader.leaves().get(0).reader().getNumericDocValues("f2");
  assertEquals(0, dvs.nextDoc());
  assertEquals(3, dvs.longValue());
  reader.close();
  
  dir.close();
}
 
Example 26
@Override
public Iterable<? extends Field> getFieldsForSubColumn(String family, Column column, String subName) {
  String name = getName(family, column.getName(), subName);
  Field field = new Field(name, column.getValue(), StringField.TYPE_NOT_STORED);
  if (isSortEnable()) {
    return addSort(column, name, field);
  }
  return makeIterable(field);
}
 
Example 27
Source Project: sql-layer   Source File: RowIndexer.java    License: GNU Affero General Public License v3.0 5 votes vote down vote up
protected void getKeyBytes(Row row) {
    
    byte[] bytes = row.hKey().hKeyBytes();
    keyEncodedString = encodeBytes(bytes, 0, bytes.length);
    Field field = new StringField(IndexedField.KEY_FIELD, keyEncodedString, Store.YES);
    currentDocument.add(field);
}
 
Example 28
Source Project: crate   Source File: LongColumnReferenceTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void insertValues(IndexWriter writer) throws Exception {
    for (long l = Long.MIN_VALUE; l < Long.MIN_VALUE + 10; l++) {
        Document doc = new Document();
        doc.add(new StringField("_id", Long.toString(l), Field.Store.NO));
        doc.add(new NumericDocValuesField(column, l));
        writer.addDocument(doc);
    }
}
 
Example 29
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedNumericMergeAwayAllValuesLargeSegment() throws IOException {
  Directory directory = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
  iwconfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);

  Document doc = new Document();
  doc.add(new StringField("id", "1", Field.Store.NO));
  doc.add(new SortedNumericDocValuesField("field", 42L));
  iwriter.addDocument(doc);
  final int numEmptyDocs = atLeast(1024);
  for (int i = 0; i < numEmptyDocs; ++i) {
    iwriter.addDocument(new Document());
  }
  iwriter.commit();
  iwriter.deleteDocuments(new Term("id", "1"));
  iwriter.forceMerge(1);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedNumericDocValues dv = getOnlyLeafReader(ireader).getSortedNumericDocValues("field");
  assertEquals(NO_MORE_DOCS, dv.nextDoc());

  ireader.close();
  directory.close();
}
 
Example 30
Source Project: dremio-oss   Source File: AuxiliaryIndexImpl.java    License: Apache License 2.0 5 votes vote down vote up
private Document toDoc(KVStoreTuple<K> key, T value){
  final Document doc = new Document();
  final SimpleDocumentWriter documentWriter = new SimpleDocumentWriter(doc);
  converter.convert(documentWriter, key.getObject(), value);

  doc.add(new StringField(CoreIndexedStore.ID_FIELD_NAME, new BytesRef(key.getSerializedBytes()), Field.Store.YES));

  return doc;
}