org.apache.lucene.document.NumericDocValuesField Java Examples

The following examples show how to use org.apache.lucene.document.NumericDocValuesField. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIllegalTypeChangeAcrossSegments() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  Document doc = new Document();
  doc.add(new NumericDocValuesField("dv", 0L));
  writer.addDocument(doc);
  writer.close();

  conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer2 = new IndexWriter(dir, conf);
  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("dv", new BytesRef("foo")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer2.addDocument(doc2);
  });

  writer2.close();
  dir.close();
}
 
Example #2
Source File: TestMixedDocValuesUpdates.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testUpdateNotExistingFieldDV() throws IOException {
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
    Document doc = new Document();
    doc.add(new StringField("id", "1", Store.YES));
    doc.add(new NumericDocValuesField("test", 1));
    writer.addDocument(doc);
    if (random().nextBoolean()) {
      writer.commit();
    }
    writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));

    Document doc1 = new Document();
    doc1.add(new StringField("id", "2", Store.YES));
    doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
        writer.addDocument(doc1)
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());

    iae = expectThrows(IllegalArgumentException.class, () ->
        writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
  }
}
 
Example #3
Source File: DocValuesAdapterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random()));

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv_binary", new BytesRef("lucene")));
  doc.add(new SortedDocValuesField("dv_sorted", new BytesRef("abc")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("python")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("java")));
  doc.add(new NumericDocValuesField("dv_numeric", 42L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 22L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 11L));
  doc.add(newStringField("no_dv", "aaa", Field.Store.NO));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 
Example #4
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testIllegalTypeChange() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  Document doc = new Document();
  doc.add(new NumericDocValuesField("dv", 0L));
  writer.addDocument(doc);
  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("dv", new BytesRef("foo")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(doc2);
  });

  IndexReader ir = writer.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  writer.close();
  dir.close();
}
 
Example #5
Source File: TestParallelLeafReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testWithDocValuesUpdates() throws Exception {
  Directory dir1 = newDirectory();
  IndexWriterConfig iwc1 = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter w1 = new IndexWriter(dir1, iwc1);
  Document d = new Document();
  d.add(newTextField("name", "billy", Field.Store.NO));
  d.add(new NumericDocValuesField("age", 21));
  w1.addDocument(d);
  w1.commit();
  w1.updateNumericDocValue(new Term("name", "billy"), "age", 22);
  w1.close();

  IndexReader r1 = DirectoryReader.open(dir1);
  LeafReader lr = new ParallelLeafReader(false, getOnlyLeafReader(r1));

  NumericDocValues dv = lr.getNumericDocValues("age");
  assertEquals(0, dv.nextDoc());
  assertEquals(22, dv.longValue());

  assertEquals(1, lr.getFieldInfos().fieldInfo("age").getDocValuesGen());

  IOUtils.close(lr, r1, dir1);

}
 
Example #6
Source File: RangeFacetsExample.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Build the example index. */
public void index() throws IOException {
  IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig(
      new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE));

  // Add documents with a fake timestamp, 1000 sec before
  // "now", 2000 sec before "now", ...:
  for(int i=0;i<100;i++) {
    Document doc = new Document();
    long then = nowSec - i * 1000;
    // Add as doc values field, so we can compute range facets:
    doc.add(new NumericDocValuesField("timestamp", then));
    // Add as numeric field so we can drill-down:
    doc.add(new LongPoint("timestamp", then));
    indexWriter.addDocument(doc);
  }

  // Open near-real-time searcher
  searcher = new IndexSearcher(DirectoryReader.open(indexWriter));
  indexWriter.close();
}
 
Example #7
Source File: TestLongValuesSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = TestUtil.nextInt(random(), 2049, 4000);
  int leastValue = 45;
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    if (i == 545)
      document.add(new NumericDocValuesField("onefield", LEAST_LONG_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example #8
Source File: SimpleDocumentWriter.java    From dremio-oss with Apache License 2.0 6 votes vote down vote up
void addToDoc(IndexKey key, Integer value){
  Preconditions.checkArgument(key.getValueType() == Integer.class);
  if(value == null){
    return;
  }

  checkIfMultiValueField(key);

  final String indexFieldName = key.getIndexFieldName();
  doc.add(new IntPoint(indexFieldName, value));
  if (key.isStored()) {
    doc.add(new StoredField(indexFieldName, value));
  }
  if (key.isSorted()) {
    Preconditions.checkArgument(key.getSortedValueType() == SearchFieldSorting.FieldType.INTEGER);
    doc.add(new NumericDocValuesField(indexFieldName, value));
  }
}
 
Example #9
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend1() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  doc.add(new NumericDocValuesField("foo", 0));
  w.addDocument(doc);
  w.close();

  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("foo", new BytesRef("hello")));
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  w2.close();
  dir.close();
}
 
Example #10
Source File: TestDoubleValuesSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  final int numDocs;
  if (TEST_NIGHTLY) {
    numDocs = TestUtil.nextInt(random(), 2049, 4000);
  } else {
    numDocs = atLeast(546);
  }
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    if (i == 545)
      document.add(new DoubleDocValuesField("onefield", LEAST_DOUBLE_VALUE));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example #11
Source File: TestFunctionScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNaN() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
  Document doc = new Document();
  doc.add(new NumericDocValuesField("foo", Double.doubleToLongBits(Double.NaN)));
  w.addDocument(doc);
  IndexReader reader = DirectoryReader.open(w);
  w.close();
  IndexSearcher searcher = newSearcher(reader);
  Query q = new FunctionScoreQuery(new MatchAllDocsQuery(), DoubleValuesSource.fromDoubleField("foo"));
  QueryUtils.check(random(), q, searcher);
  Explanation expl = searcher.explain(q, 0);
  assertEquals(0, expl.getValue().doubleValue(), 0f);
  assertTrue(expl.toString(), expl.getDetails()[0].getDescription().contains("NaN is an illegal score"));
  reader.close();
  dir.close();
}
 
Example #12
Source File: TestExpressionSorts.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(500);
  for (int i = 0; i < numDocs; i++) {
    Document document = new Document();
    document.add(newTextField("english", English.intToEnglish(i), Field.Store.NO));
    document.add(newTextField("oddeven", (i % 2 == 0) ? "even" : "odd", Field.Store.NO));
    document.add(new NumericDocValuesField("int", random().nextInt()));
    document.add(new NumericDocValuesField("long", random().nextLong()));
    document.add(new FloatDocValuesField("float", random().nextFloat()));
    document.add(new DoubleDocValuesField("double", random().nextDouble()));
    iw.addDocument(document);
  }
  reader = iw.getReader();
  iw.close();
  searcher = newSearcher(reader);
}
 
Example #13
Source File: BlockGroupingTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static List<Document> createRandomBlock(int book) {
  List<Document> block = new ArrayList<>();
  String bookName = "book" + book;
  int chapterCount = atLeast(10);
  for (int j = 0; j < chapterCount; j++) {
    Document doc = new Document();
    String chapterName = "chapter" + j;
    String chapterText = randomText();
    doc.add(new TextField("book", bookName, Field.Store.YES));
    doc.add(new TextField("chapter", chapterName, Field.Store.YES));
    doc.add(new TextField("text", chapterText, Field.Store.NO));
    doc.add(new NumericDocValuesField("length", chapterText.length()));
    doc.add(new SortedDocValuesField("book", new BytesRef(bookName)));
    if (j == chapterCount - 1) {
      doc.add(new TextField("blockEnd", "true", Field.Store.NO));
    }
    block.add(doc);
  }
  return block;
}
 
Example #14
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMixedTypesDifferentSegments() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  doc.add(new NumericDocValuesField("foo", 0));
  w.addDocument(doc);
  w.commit();

  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("foo", new BytesRef("hello")));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc2);
  });

  w.close();
  dir.close();
}
 
Example #15
Source File: BaseDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testRandomAdvanceNumeric() throws IOException {
  final long longRange;
  if (random().nextBoolean()) {
    longRange = TestUtil.nextInt(random(), 1, 1024);
  } else {
    longRange = TestUtil.nextLong(random(), 1, Long.MAX_VALUE);
  }
  doTestRandomAdvance(new FieldCreator() {
      @Override
      public Field next() {
        return new NumericDocValuesField("field", TestUtil.nextLong(random(), 0, longRange));
      }

      @Override
      public DocIdSetIterator iterator(IndexReader r) throws IOException {
        return MultiDocValues.getNumericValues(r, "field");
      }
    });
}
 
Example #16
Source File: SecureAtomicReaderTestBase.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example #17
Source File: TestTaxonomyFacetSumValueSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNoScore() throws Exception {
  Directory indexDir = newDirectory();
  Directory taxoDir = newDirectory();

  DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
  IndexWriter iw = new IndexWriter(indexDir, newIndexWriterConfig(new MockAnalyzer(random())));
  FacetsConfig config = new FacetsConfig();
  for (int i = 0; i < 4; i++) {
    Document doc = new Document();
    doc.add(new NumericDocValuesField("price", (i+1)));
    doc.add(new FacetField("a", Integer.toString(i % 2)));
    iw.addDocument(config.build(taxoWriter, doc));
  }
  
  DirectoryReader r = DirectoryReader.open(iw);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoWriter);

  FacetsCollector sfc = new FacetsCollector();
  newSearcher(r).search(new MatchAllDocsQuery(), sfc);
  Facets facets = new TaxonomyFacetSumValueSource(taxoReader, config, sfc, DoubleValuesSource.fromLongField("price"));
  assertEquals("dim=a path=[] value=10.0 childCount=2\n  1 (6.0)\n  0 (4.0)\n", facets.getTopChildren(10, "a").toString());

  iw.close();
  IOUtils.close(taxoWriter, taxoReader, taxoDir, r, indexDir);
}
 
Example #18
Source File: TestLongValueFacetCounts.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testOnlyBigLongs() throws Exception {
  Directory d = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), d);
  for (long l = 0; l < 3; l++) {
    Document doc = new Document();
    doc.add(new NumericDocValuesField("field", Long.MAX_VALUE - l));
    w.addDocument(doc);
  }

  IndexReader r = w.getReader();
  w.close();

  FacetsCollector fc = new FacetsCollector();
  IndexSearcher s = newSearcher(r);
  s.search(new MatchAllDocsQuery(), fc);

  LongValueFacetCounts facets = new LongValueFacetCounts("field", fc, false);

  FacetResult result = facets.getAllChildrenSortByValue();
  assertEquals("dim=field path=[] value=3 childCount=3\n  9223372036854775805 (1)\n  " +
               "9223372036854775806 (1)\n  9223372036854775807 (1)\n",
               result.toString());
  r.close();
  d.close();
}
 
Example #19
Source File: TestSoftDeletesRetentionMergePolicy.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSoftDeleteWithTryUpdateDocValue() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig config = newIndexWriterConfig().setSoftDeletesField("soft_delete")
      .setMergePolicy(new SoftDeletesRetentionMergePolicy("soft_delete", MatchAllDocsQuery::new, newLogMergePolicy()));
  IndexWriter writer = new IndexWriter(dir, config);
  SearcherManager sm = new SearcherManager(writer, new SearcherFactory());
  Document d = new Document();
  d.add(new StringField("id", "0", Field.Store.YES));
  writer.addDocument(d);
  sm.maybeRefreshBlocking();
  doUpdate(new Term("id", "0"), writer,
      new NumericDocValuesField("soft_delete", 1), new NumericDocValuesField("other-field", 1));
  sm.maybeRefreshBlocking();
  assertEquals(1, writer.cloneSegmentInfos().size());
  SegmentCommitInfo si = writer.cloneSegmentInfos().info(0);
  assertEquals(1, si.getSoftDelCount());
  assertEquals(1, si.info.maxDoc());
  IOUtils.close(sm, writer, dir);
}
 
Example #20
Source File: TestMatchesIterator.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();
  directory = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
      newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
  for (int i = 0; i < docFields.length; i++) {
    Document doc = new Document();
    doc.add(newField(FIELD_WITH_OFFSETS, docFields[i], OFFSETS));
    doc.add(newField(FIELD_NO_OFFSETS, docFields[i], TextField.TYPE_STORED));
    doc.add(newField(FIELD_DOCS_ONLY, docFields[i], DOCS));
    doc.add(newField(FIELD_FREQS, docFields[i], DOCS_AND_FREQS));
    doc.add(new IntPoint(FIELD_POINT, 10));
    doc.add(new NumericDocValuesField(FIELD_POINT, 10));
    doc.add(new NumericDocValuesField("id", i));
    doc.add(newField("id", Integer.toString(i), TextField.TYPE_STORED));
    writer.addDocument(doc);
  }
  writer.forceMerge(1);
  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(getOnlyLeafReader(reader));
}
 
Example #21
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMixedTypesDifferentDocuments() throws Exception {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  Document doc = new Document();
  doc.add(new NumericDocValuesField("foo", 0));
  w.addDocument(doc);

  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("foo", new BytesRef("hello")));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc2);
  });

  IndexReader ir = w.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  w.close();
  dir.close();
}
 
Example #22
Source File: TestDocValuesStatsCollector.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testOneDoc() throws IOException {
  try (Directory dir = newDirectory();
      IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig())) {
    String field = "numeric";
    Document doc = new Document();
    doc.add(new NumericDocValuesField(field, 1));
    doc.add(new StringField("id", "doc1", Store.NO));
    indexWriter.addDocument(doc);

    try (DirectoryReader reader = DirectoryReader.open(indexWriter)) {
      IndexSearcher searcher = new IndexSearcher(reader);
      LongDocValuesStats stats = new LongDocValuesStats(field);
      searcher.search(new MatchAllDocsQuery(), new DocValuesStatsCollector(stats));

      assertEquals(1, stats.count());
      assertEquals(0, stats.missing());
      assertEquals(1, stats.max().longValue());
      assertEquals(1, stats.min().longValue());
      assertEquals(1, stats.sum().longValue());
      assertEquals(1, stats.mean(), 0.0001);
      assertEquals(0, stats.variance(), 0.0001);
      assertEquals(0, stats.stdev(), 0.0001);
    }
  }
}
 
Example #23
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testTypeChangeViaAddIndexes2() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  Document doc = new Document();
  doc.add(new NumericDocValuesField("dv", 0L));
  writer.addDocument(doc);
  writer.close();

  Directory dir2 = newDirectory();
  conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer2 = new IndexWriter(dir2, conf);
  writer2.addIndexes(dir);
  Document doc2 = new Document();
  doc2.add(new SortedDocValuesField("dv", new BytesRef("foo")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer2.addDocument(doc2);
  });

  writer2.close();
  dir2.close();
  dir.close();
}
 
Example #24
Source File: BaseCompressingDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSingleBigValueCompression() throws IOException {
  try (final Directory dir = new ByteBuffersDirectory()) {
    final IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
    final IndexWriter iwriter = new IndexWriter(dir, iwc);

    final Document doc = new Document();
    final NumericDocValuesField dvf = new NumericDocValuesField("dv", 0);
    doc.add(dvf);
    for (int i = 0; i < 20000; ++i) {
      dvf.setLongValue(i & 1023);
      iwriter.addDocument(doc);
    }
    iwriter.forceMerge(1);
    final long size1 = dirSize(dir);
    dvf.setLongValue(Long.MAX_VALUE);
    iwriter.addDocument(doc);
    iwriter.forceMerge(1);
    final long size2 = dirSize(dir);
    // make sure the new value did not grow the bpv for every other value
    assertTrue(size2 < size1 + (20000 * (63 - 10)) / 8);
  }
}
 
Example #25
Source File: BaseDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected void addRandomFields(Document doc) {
  if (usually()) {
    doc.add(new NumericDocValuesField("ndv", random().nextInt(1 << 12)));
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
    doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  int numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
  }
}
 
Example #26
Source File: EnumFieldType.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
  if ( ! sf.hasDocValues()) {
    throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, 
        getClass().getSimpleName() + " requires docValues=\"true\".");
  }
  final IndexableField field = createField(sf, value);
  final List<IndexableField> fields = new ArrayList<>();
  fields.add(field);
  final long longValue = field.numericValue().longValue();
  if (sf.multiValued()) {
    fields.add(new SortedNumericDocValuesField(sf.getName(), longValue));
  } else {
    fields.add(new NumericDocValuesField(sf.getName(), longValue));
  }
  return fields;
}
 
Example #27
Source File: TestFieldValueQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testScore() throws IOException {
  final int iters = atLeast(10);
  for (int iter = 0; iter < iters; ++iter) {
    Directory dir = newDirectory();
    RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
    final int numDocs = atLeast(100);
    for (int i = 0; i < numDocs; ++i) {
      Document doc = new Document();
      final boolean hasValue = random().nextBoolean();
      if (hasValue) {
        doc.add(new NumericDocValuesField("dv1", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 1));
        doc.add(new SortedNumericDocValuesField("dv2", 2));
        doc.add(new StringField("has_value", "yes", Store.NO));
      }
      doc.add(new StringField("f", random().nextBoolean() ? "yes" : "no", Store.NO));
      iw.addDocument(doc);
    }
    if (random().nextBoolean()) {
      iw.deleteDocuments(new TermQuery(new Term("f", "no")));
    }
    iw.commit();
    final IndexReader reader = iw.getReader();
    final IndexSearcher searcher = newSearcher(reader);
    iw.close();

    final float boost = random().nextFloat() * 10;
    final Query ref = new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("has_value", "yes"))), boost);

    final Query q1 = new BoostQuery(new DocValuesFieldExistsQuery("dv1"), boost);
    assertSameMatches(searcher, ref, q1, true);

    final Query q2 = new BoostQuery(new DocValuesFieldExistsQuery("dv2"), boost);
    assertSameMatches(searcher, ref, q2, true);

    reader.close();
    dir.close();
  }
}
 
Example #28
Source File: TestNumericDocValuesUpdates.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testUpdateDocumentByMultipleTerms() throws Exception {
  // make sure the order of updates is respected, even when multiple terms affect same document
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("k1", "v1", Store.NO));
  doc.add(new StringField("k2", "v2", Store.NO));
  doc.add(new NumericDocValuesField("ndv", 5));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateNumericDocValue(new Term("k1", "v1"), "ndv", 17L);
  writer.updateNumericDocValue(new Term("k2", "v2"), "ndv", 3L);
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  NumericDocValues ndv = MultiDocValues.getNumericValues(reader, "ndv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, ndv.nextDoc());
    assertEquals(3, ndv.longValue());
  }
  reader.close();
  dir.close();
}
 
Example #29
Source File: LineFileDocs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public DocState() {
  doc = new Document();
  
  title = new StringField("title", "", Field.Store.NO);
  doc.add(title);

  FieldType ft = new FieldType(TextField.TYPE_STORED);
  ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
  ft.setStoreTermVectors(true);
  ft.setStoreTermVectorOffsets(true);
  ft.setStoreTermVectorPositions(true);
  
  titleTokenized = new Field("titleTokenized", "", ft);
  doc.add(titleTokenized);

  body = new Field("body", "", ft);
  doc.add(body);

  id = new StringField("docid", "", Field.Store.YES);
  doc.add(id);

  idNum = new IntPoint("docid_int", 0);
  doc.add(idNum);

  date = new StringField("date", "", Field.Store.YES);
  doc.add(date);

  titleDV = new SortedDocValuesField("titleDV", new BytesRef());
  idNumDV = new NumericDocValuesField("docid_intDV", 0);
  doc.add(titleDV);
  doc.add(idNumDV);
}
 
Example #30
Source File: TestAddIndexes.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testWithPendingDeletes() throws IOException {
  // main directory
  Directory dir = newDirectory();
  // auxiliary directory
  Directory aux = newDirectory();

  setUpDirs(dir, aux);
  IndexWriter writer = newWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())).setOpenMode(OpenMode.APPEND));
  writer.addIndexes(aux);

  // Adds 10 docs, then replaces them with another 10
  // docs, so 10 pending deletes:
  for (int i = 0; i < 20; i++) {
    Document doc = new Document();
    doc.add(newStringField("id", "" + (i % 10), Field.Store.NO));
    doc.add(newTextField("content", "bbb " + i, Field.Store.NO));
    doc.add(new IntPoint("doc", i));
    doc.add(new IntPoint("doc2d", i, i));
    doc.add(new NumericDocValuesField("dv", i));
    writer.updateDocument(new Term("id", "" + (i%10)), doc);
  }
  // Deletes one of the 10 added docs, leaving 9:
  PhraseQuery q = new PhraseQuery("content", "bbb", "14");
  writer.deleteDocuments(q);

  writer.forceMerge(1);
  writer.commit();

  verifyNumDocs(dir, 1039);
  verifyTermDocs(dir, new Term("content", "aaa"), 1030);
  verifyTermDocs(dir, new Term("content", "bbb"), 9);

  writer.close();
  dir.close();
  aux.close();
}