Java Code Examples for org.apache.lucene.document.BinaryDocValuesField

The following examples show how to use org.apache.lucene.document.BinaryDocValuesField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: IdFieldMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    XContentParser parser = context.parser();
    if (parser.currentName() != null && parser.currentName().equals(Defaults.NAME) && parser.currentToken().isValue()) {
        // we are in the parse Phase
        String id = parser.text();
        if (context.id() != null && !context.id().equals(id)) {
            throw new MapperParsingException("Provided id [" + context.id() + "] does not match the content one [" + id + "]");
        }
        context.id(id);
    } // else we are in the pre/post parse phase

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        fields.add(new Field(fieldType().names().indexName(), context.id(), fieldType()));
    }
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(fieldType().names().indexName(), new BytesRef(context.id())));
    }
}
 
Example 2
Source Project: lucene-solr   Source File: SerializedDVStrategy.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Field[] createIndexableFields(Shape shape) {
  int bufSize = Math.max(128, (int) (this.indexLastBufSize * 1.5));//50% headroom over last
  ByteArrayOutputStream byteStream = new ByteArrayOutputStream(bufSize);
  final BytesRef bytesRef = new BytesRef();//receiver of byteStream's bytes
  try {
    ctx.getBinaryCodec().writeShape(new DataOutputStream(byteStream), shape);
    //this is a hack to avoid redundant byte array copying by byteStream.toByteArray()
    byteStream.writeTo(new FilterOutputStream(null/*not used*/) {
      @Override
      public void write(byte[] b, int off, int len) throws IOException {
        bytesRef.bytes = b;
        bytesRef.offset = off;
        bytesRef.length = len;
      }
    });
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
  this.indexLastBufSize = bytesRef.length;//cache heuristic
  return new Field[]{new BinaryDocValuesField(getFieldName(), bytesRef)};
}
 
Example 3
Source Project: lucene-solr   Source File: DocValuesAdapterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random()));

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv_binary", new BytesRef("lucene")));
  doc.add(new SortedDocValuesField("dv_sorted", new BytesRef("abc")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("python")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("java")));
  doc.add(new NumericDocValuesField("dv_numeric", 42L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 22L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 11L));
  doc.add(newStringField("no_dv", "aaa", Field.Store.NO));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 
Example 4
Source Project: lucene-solr   Source File: AnalyzingInfixSuggester.java    License: Apache License 2.0 6 votes vote down vote up
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 
Example 5
Source Project: lucene-solr   Source File: TestOrdinalMappingLeafReader.java    License: Apache License 2.0 6 votes vote down vote up
private void buildIndexWithFacets(Directory indexDir, Directory taxoDir, boolean asc) throws IOException {
  IndexWriterConfig config = newIndexWriterConfig(null);
  RandomIndexWriter writer = new RandomIndexWriter(random(), indexDir, config);
  
  DirectoryTaxonomyWriter taxonomyWriter = new DirectoryTaxonomyWriter(taxoDir);
  for (int i = 1; i <= NUM_DOCS; i++) {
    Document doc = new Document();
    for (int j = i; j <= NUM_DOCS; j++) {
      int facetValue = asc ? j: NUM_DOCS - j;
      doc.add(new FacetField("tag", Integer.toString(facetValue)));
    }
    // add a facet under default dim config
    doc.add(new FacetField("id", Integer.toString(i)));
    
    // make sure OrdinalMappingLeafReader ignores non-facet BinaryDocValues fields
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(Integer.toString(i))));
    doc.add(new BinaryDocValuesField("cbdv", new BytesRef(Integer.toString(i*2))));
    writer.addDocument(facetConfig.build(taxonomyWriter, doc));
  }
  taxonomyWriter.commit();
  taxonomyWriter.close();
  writer.commit();
  writer.close();
}
 
Example 6
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void addRandomFields(Document doc) {
  if (usually()) {
    doc.add(new NumericDocValuesField("ndv", random().nextInt(1 << 12)));
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
    doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  int numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
  }
}
 
Example 7
Source Project: lucene-solr   Source File: QueryIndex.java    License: Apache License 2.0 6 votes vote down vote up
private List<Indexable> buildIndexables(List<MonitorQuery> updates) {
  List<Indexable> indexables = new ArrayList<>();
  for (MonitorQuery mq : updates) {
    if (serializer != null && mq.getQueryString() == null) {
      throw new IllegalArgumentException("Cannot add a MonitorQuery with a null string representation to a non-ephemeral Monitor");
    }
    BytesRef serialized = serializer == null ? EMPTY : serializer.serialize(mq);
    for (QueryCacheEntry qce : QueryCacheEntry.decompose(mq, decomposer)) {
      Document doc = presearcher.indexQuery(qce.matchQuery, mq.getMetadata());
      doc.add(new StringField(FIELDS.query_id, qce.queryId, Field.Store.NO));
      doc.add(new SortedDocValuesField(FIELDS.cache_id, new BytesRef(qce.cacheId)));
      doc.add(new SortedDocValuesField(FIELDS.query_id, new BytesRef(qce.queryId)));
      doc.add(new BinaryDocValuesField(FIELDS.mq, serialized));
      indexables.add(new Indexable(qce, doc));
    }
  }
  return indexables;
}
 
Example 8
Source Project: lucene-solr   Source File: TestMixedDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateNotExistingFieldDV() throws IOException {
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  try (Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, conf)) {
    Document doc = new Document();
    doc.add(new StringField("id", "1", Store.YES));
    doc.add(new NumericDocValuesField("test", 1));
    writer.addDocument(doc);
    if (random().nextBoolean()) {
      writer.commit();
    }
    writer.updateDocValues(new Term("id", "1"), new NumericDocValuesField("not_existing", 1));

    Document doc1 = new Document();
    doc1.add(new StringField("id", "2", Store.YES));
    doc1.add(new BinaryDocValuesField("not_existing", new BytesRef()));
    IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () ->
        writer.addDocument(doc1)
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());

    iae = expectThrows(IllegalArgumentException.class, () ->
        writer.updateDocValues(new Term("id", "1"), new BinaryDocValuesField("not_existing", new BytesRef()))
    );
    assertEquals("cannot change DocValues type from NUMERIC to BINARY for field \"not_existing\"", iae.getMessage());
  }
}
 
Example 9
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateSameDocMultipleTimes() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("key", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateBinaryDocValue(new Term("key", "doc"), "bdv", toBytes(17L)); // update existing field
  writer.updateBinaryDocValue(new Term("key", "doc"), "bdv", toBytes(3L)); // update existing field 2nd time in this commit
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(3, getValue(bdv));
  }
  reader.close();
  dir.close();
}
 
Example 10
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateBinaryDVFieldWithSameNameAsPostingField() throws Exception {
  // this used to fail because FieldInfos.Builder neglected to update
  // globalFieldMaps.docValuesTypes map
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new BinaryDocValuesField("f", toBytes(5L)));
  writer.addDocument(doc);
  writer.commit();
  writer.updateBinaryDocValue(new Term("f", "mock-value"), "f", toBytes(17L));
  writer.close();
  
  DirectoryReader r = DirectoryReader.open(dir);
  BinaryDocValues bdv = r.leaves().get(0).reader().getBinaryDocValues("f");
  assertEquals(0, bdv.nextDoc());
  assertEquals(17, getValue(bdv));
  r.close();
  
  dir.close();
}
 
Example 11
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateAllDeletedSegment() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  writer.addDocument(doc);
  writer.addDocument(doc);
  writer.commit();
  writer.deleteDocuments(new Term("id", "doc")); // delete all docs in the first segment
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("id", "doc"), "f1", toBytes(2L));
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(2L, getValue(bdv));
  reader.close();
  
  dir.close();
}
 
Example 12
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 6 votes vote down vote up
public void testUpdateTwoNonexistingTerms() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "doc", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  writer.addDocument(doc);
  // update w/ multiple nonexisting terms in same field
  writer.updateBinaryDocValue(new Term("c", "foo"), "f1", toBytes(2L));
  writer.updateBinaryDocValue(new Term("c", "bar"), "f1", toBytes(2L));
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  assertEquals(1, reader.leaves().size());
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(1L, getValue(bdv));
  reader.close();
  
  dir.close();
}
 
Example 13
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testDifferentTypedDocValuesField() throws Exception {
  Directory d = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), d);
  Document doc = new Document();
  doc.add(new NumericDocValuesField("field", 17));
  w.addDocument(doc);
  
  // Index doc values are single-valued so we should not
  // be able to add same field more than once:
  doc.add(new BinaryDocValuesField("field", new BytesRef("blah")));
  expectThrows(IllegalArgumentException.class, () -> {
    w.addDocument(doc);
  });

  DirectoryReader r = w.getReader();
  w.close();
  NumericDocValues values = DocValues.getNumeric(getOnlyLeafReader(r), "field");
  assertEquals(0, values.nextDoc());
  assertEquals(17, values.longValue());
  r.close();
  d.close();
}
 
Example 14
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testAddBinaryTwice() throws IOException {
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv", new BytesRef("foo!")));
  iwriter.addDocument(doc);
  
  doc.add(new BinaryDocValuesField("dv", new BytesRef("bar!")));
  expectThrows(IllegalArgumentException.class, () -> {
    iwriter.addDocument(doc);
  });
  
  IndexReader ir = iwriter.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  
  iwriter.close();
  directory.close();
}
 
Example 15
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend2() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example 16
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend3() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  // Also add another document so there is a segment to write here:
  w2.addDocument(new Document());
  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example 17
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testSameFieldNameForPostingAndDocValue() throws Exception {
  // LUCENE-5192: FieldInfos.Builder neglected to update
  // globalFieldNumbers.docValuesType map if the field existed, resulting in
  // potentially adding the same field with different DV types.
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("f", "mock-value", Store.NO));
  doc.add(new NumericDocValuesField("f", 5));
  writer.addDocument(doc);
  writer.commit();
  
  Document doc2 = new Document();
  doc2.add(new BinaryDocValuesField("f", new BytesRef("mock")));
  expectThrows(IllegalArgumentException.class, () -> {
    writer.addDocument(doc2);
  });
  writer.rollback();
  
  dir.close();
}
 
Example 18
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example 19
Source Project: Elasticsearch   Source File: UidFieldMapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    Field uid = new Field(NAME, Uid.createUid(context.stringBuilder(), context.type(), context.id()), Defaults.FIELD_TYPE);
    context.uid(uid);
    fields.add(uid);
    if (fieldType().hasDocValues()) {
        fields.add(new BinaryDocValuesField(NAME, new BytesRef(uid.stringValue())));
    }
}
 
Example 20
Source Project: lucene-solr   Source File: TestMemoryIndex.java    License: Apache License 2.0 5 votes vote down vote up
public void testDocValuesDoNotAffectBoostPositionsOrOffset() throws Exception {
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
  doc.add(new TextField("text", "quick brown fox", Field.Store.NO));
  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer, true, true);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();
  TermsEnum tenum = leafReader.terms("text").iterator();

  assertEquals("brown", tenum.next().utf8ToString());
  PostingsEnum penum = tenum.postings(null, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(1, penum.nextPosition());
  assertEquals(6, penum.startOffset());
  assertEquals(11, penum.endOffset());

  assertEquals("fox", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(2, penum.nextPosition());
  assertEquals(12, penum.startOffset());
  assertEquals(15, penum.endOffset());

  assertEquals("quick", tenum.next().utf8ToString());
  penum = tenum.postings(penum, PostingsEnum.OFFSETS);
  assertEquals(0, penum.nextDoc());
  assertEquals(1, penum.freq());
  assertEquals(0, penum.nextPosition());
  assertEquals(0, penum.startOffset());
  assertEquals(5, penum.endOffset());

  BinaryDocValues binaryDocValues = leafReader.getBinaryDocValues("text");
  assertEquals(0, binaryDocValues.nextDoc());
  assertEquals("quick brown fox", binaryDocValues.binaryValue().utf8ToString());
}
 
Example 21
public void testNormsWithDocValues() throws Exception {
  MemoryIndex mi = new MemoryIndex(true, true);
  MockAnalyzer mockAnalyzer = new MockAnalyzer(random());

  mi.addField(new BinaryDocValuesField("text", new BytesRef("quick brown fox")), mockAnalyzer);
  mi.addField(new TextField("text", "quick brown fox", Field.Store.NO), mockAnalyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("text", new BytesRef("quick brown fox")));
  Field field = new TextField("text", "quick brown fox", Field.Store.NO);
  doc.add(field);
  Directory dir = newDirectory();
  IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(random(), mockAnalyzer));
  writer.addDocument(doc);
  writer.close();

  IndexReader controlIndexReader = DirectoryReader.open(dir);
  LeafReader controlLeafReader =  controlIndexReader.leaves().get(0).reader();

  NumericDocValues norms = controlLeafReader.getNormValues("text");
  assertEquals(0, norms.nextDoc());
  NumericDocValues norms2 = leafReader.getNormValues("text");
  assertEquals(0, norms2.nextDoc());
  assertEquals(norms.longValue(), norms2.longValue());

  controlIndexReader.close();
  dir.close();
}
 
Example 22
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testBinaryMergeAwayAllValuesLargeSegment() throws IOException {
  Directory directory = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
  iwconfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);

  Document doc = new Document();
  doc.add(new StringField("id", "1", Field.Store.NO));
  doc.add(new BinaryDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  final int numEmptyDocs = atLeast(1024);
  for (int i = 0; i < numEmptyDocs; ++i) {
    iwriter.addDocument(new Document());
  }
  iwriter.commit();
  iwriter.deleteDocuments(new Term("id", "1"));
  iwriter.forceMerge(1);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  BinaryDocValues dv = getOnlyLeafReader(ireader).getBinaryDocValues("field");
  assertEquals(NO_MORE_DOCS, dv.nextDoc());

  ireader.close();
  directory.close();
}
 
Example 23
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testRandomAdvanceBinary() throws IOException {
  doTestRandomAdvance(new FieldCreator() {
      @Override
      public Field next() {
        byte[] bytes = new byte[random().nextInt(10)];
        random().nextBytes(bytes);
        return new BinaryDocValuesField("field", new BytesRef(bytes));
      }

      @Override
      public DocIdSetIterator iterator(IndexReader r) throws IOException {
        return MultiDocValues.getBinaryValues(r, "field");
      }
    });
}
 
Example 24
Source Project: lucene-solr   Source File: TestDocValues.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * field with binary docvalues
 */
public void testBinaryField() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("foo", new BytesRef("bar")));
  iw.addDocument(doc);
  DirectoryReader dr = DirectoryReader.open(iw);
  LeafReader r = getOnlyLeafReader(dr);
  
  // ok
  assertNotNull(DocValues.getBinary(r, "foo"));
  
  // errors
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getNumeric(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSorted(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSortedSet(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSortedNumeric(r, "foo");
  });
  
  dr.close();
  iw.close();
  dir.close();
}
 
Example 25
Source Project: lucene-solr   Source File: TestDocValues.java    License: Apache License 2.0 5 votes vote down vote up
public void testAddNullNumericDocValues() throws IOException {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  if (random().nextBoolean()) {
    doc.add(new NumericDocValuesField("foo", null));
  } else {
    doc.add(new BinaryDocValuesField("foo", null));
  }
  IllegalArgumentException iae = expectThrows(IllegalArgumentException.class, () -> iw.addDocument(doc));
  assertEquals("field=\"foo\": null value not allowed", iae.getMessage());
  IOUtils.close(iw, dir);
}
 
Example 26
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testMultipleBinaryDocValues() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setMaxBufferedDocs(10); // prevent merges
  IndexWriter writer = new IndexWriter(dir, conf);
  
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
    doc.add(new BinaryDocValuesField("bdv1", toBytes(i)));
    doc.add(new BinaryDocValuesField("bdv2", toBytes(i)));
    writer.addDocument(doc);
  }
  writer.commit();
  
  // update all docs' bdv1 field
  writer.updateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv1", toBytes(17L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  LeafReader r = reader.leaves().get(0).reader();
  
  BinaryDocValues bdv1 = r.getBinaryDocValues("bdv1");
  BinaryDocValues bdv2 = r.getBinaryDocValues("bdv2");
  for (int i = 0; i < r.maxDoc(); i++) {
    assertEquals(i, bdv1.nextDoc());
    assertEquals(17, getValue(bdv1));
    assertEquals(i, bdv2.nextDoc());
    assertEquals(i, getValue(bdv2));
  }
  
  reader.close();
  dir.close();
}
 
Example 27
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testDocumentWithNoValue() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  for (int i = 0; i < 2; i++) {
    Document doc = new Document();
    doc.add(new StringField("dvUpdateKey", "dv", Store.NO));
    if (i == 0) { // index only one document with value
      doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
    }
    writer.addDocument(doc);
  }
  writer.commit();
  
  // update all docs' bdv field
  writer.updateBinaryDocValue(new Term("dvUpdateKey", "dv"), "bdv", toBytes(17L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  LeafReader r = reader.leaves().get(0).reader();
  BinaryDocValues bdv = r.getBinaryDocValues("bdv");
  for (int i = 0; i < r.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(17, getValue(bdv));
  }
  
  reader.close();
  dir.close();
}
 
Example 28
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdateDocumentByMultipleTerms() throws Exception {
  // make sure the order of updates is respected, even when multiple terms affect same document
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("k1", "v1", Store.NO));
  doc.add(new StringField("k2", "v2", Store.NO));
  doc.add(new BinaryDocValuesField("bdv", toBytes(5L)));
  writer.addDocument(doc); // flushed document
  writer.commit();
  writer.addDocument(doc); // in-memory document
  
  writer.updateBinaryDocValue(new Term("k1", "v1"), "bdv", toBytes(17L));
  writer.updateBinaryDocValue(new Term("k2", "v2"), "bdv", toBytes(3L));
  writer.close();
  
  final DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(reader, "bdv");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(3, getValue(bdv));
  }
  reader.close();
  dir.close();
}
 
Example 29
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testDeleteUnusedUpdatesFiles() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("id", "d0", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
  writer.addDocument(doc);

  // update each field twice to make sure all unneeded files are deleted
  for (String f : new String[] { "f1", "f2" }) {
    writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(2L));
    writer.commit();
    int numFiles = dir.listAll().length;
    
    // update again, number of files shouldn't change (old field's gen is
    // removed) 
    writer.updateBinaryDocValue(new Term("id", "d0"), f, toBytes(3L));
    writer.commit();
    
    assertEquals(numFiles, dir.listAll().length);
  }

  writer.close();
  dir.close();
}
 
Example 30
Source Project: lucene-solr   Source File: TestBinaryDocValuesUpdates.java    License: Apache License 2.0 5 votes vote down vote up
public void testUpdatesOrder() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  Document doc = new Document();
  doc.add(new StringField("upd", "t1", Store.NO));
  doc.add(new StringField("upd", "t2", Store.NO));
  doc.add(new BinaryDocValuesField("f1", toBytes(1L)));
  doc.add(new BinaryDocValuesField("f2", toBytes(1L)));
  writer.addDocument(doc);
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(2L)); // update f1 to 2
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f2", toBytes(2L)); // update f2 to 2
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f1", toBytes(3L)); // update f1 to 3
  writer.updateBinaryDocValue(new Term("upd", "t2"), "f2", toBytes(3L)); // update f2 to 3
  writer.updateBinaryDocValue(new Term("upd", "t1"), "f1", toBytes(4L)); // update f1 to 4 (but not f2)
  writer.close();
  
  DirectoryReader reader = DirectoryReader.open(dir);
  BinaryDocValues bdv = reader.leaves().get(0).reader().getBinaryDocValues("f1");
  assertEquals(0, bdv.nextDoc());
  assertEquals(4, getValue(bdv));
  bdv = reader.leaves().get(0).reader().getBinaryDocValues("f2");
  assertEquals(0, bdv.nextDoc());
  assertEquals(3, getValue(bdv));
  reader.close();
  
  dir.close();
}