Java Code Examples for org.apache.lucene.document.SortedSetDocValuesField

The following examples show how to use org.apache.lucene.document.SortedSetDocValuesField. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: StringFieldMapper.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
    if (valueAndBoost.value() == null) {
        return;
    }
    if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) {
        return;
    }
    if (context.includeInAll(includeInAll, this)) {
        context.allEntries().addText(fieldType().names().fullName(), valueAndBoost.value(), valueAndBoost.boost());
    }

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().names().indexName(), valueAndBoost.value(), fieldType());
        field.setBoost(valueAndBoost.boost());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(valueAndBoost.value())));
    }
}
 
Example 2
Source Project: lucene-solr   Source File: DocValuesAdapterTest.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random()));

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv_binary", new BytesRef("lucene")));
  doc.add(new SortedDocValuesField("dv_sorted", new BytesRef("abc")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("python")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("java")));
  doc.add(new NumericDocValuesField("dv_numeric", 42L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 22L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 11L));
  doc.add(newStringField("no_dv", "aaa", Field.Store.NO));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 
Example 3
Source Project: lucene-solr   Source File: AnalyzingInfixSuggester.java    License: Apache License 2.0 6 votes vote down vote up
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 
Example 4
Source Project: lucene-solr   Source File: FacetsConfig.java    License: Apache License 2.0 6 votes vote down vote up
private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));

      FacetsConfig.DimConfig ft = getDimConfig(facetField.dim);        
      if (ft.requireDimensionDrillDown) {
        doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
      }
    }
  }
}
 
Example 5
public void testToString() {
  Query q1 = createQuery("foo", 3, 5);
  assertEquals("foo:[3 TO 5]", q1.toString());
  assertEquals("[3 TO 5]", q1.toString("foo"));
  assertEquals("foo:[3 TO 5]", q1.toString("bar"));

  Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
  assertEquals("foo:[[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, true);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, false);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), null, true, true);
  assertEquals("foo:[[62 61 72] TO *}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", null, new BytesRef("baz"), true, true);
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString());
  assertEquals("{* TO [62 61 7a]]", q2.toString("foo"));
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString("bar"));
}
 
Example 6
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 6 votes vote down vote up
@Override
protected void addRandomFields(Document doc) {
  if (usually()) {
    doc.add(new NumericDocValuesField("ndv", random().nextInt(1 << 12)));
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
    doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  int numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
  }
}
 
Example 7
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 6 votes vote down vote up
public void testSortedSetOneValue() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
  
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  
  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();
  
  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(0, dv.nextDoc());
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);

  ireader.close();
  directory.close();
}
 
Example 8
Source Project: lucene-solr   Source File: TestDocValuesQueries.java    License: Apache License 2.0 6 votes vote down vote up
public void testToString() {
  Query q1 = SortedNumericDocValuesField.newSlowRangeQuery("foo", 3, 5);
  assertEquals("foo:[3 TO 5]", q1.toString());
  assertEquals("[3 TO 5]", q1.toString("foo"));
  assertEquals("foo:[3 TO 5]", q1.toString("bar"));

  Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
  assertEquals("foo:[[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, true);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, false);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), null, true, true);
  assertEquals("foo:[[62 61 72] TO *}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", null, new BytesRef("baz"), true, true);
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString());
  assertEquals("{* TO [62 61 7a]]", q2.toString("foo"));
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString("bar"));
}
 
Example 9
Source Project: lucene-solr   Source File: TestDocValuesQueries.java    License: Apache License 2.0 6 votes vote down vote up
public void testMissingField() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  iw.addDocument(new Document());
  IndexReader reader = iw.getReader();
  iw.close();
  IndexSearcher searcher = newSearcher(reader);
  for (Query query : Arrays.asList(
      NumericDocValuesField.newSlowRangeQuery("foo", 2, 4),
      SortedNumericDocValuesField.newSlowRangeQuery("foo", 2, 4),
      SortedDocValuesField.newSlowRangeQuery("foo", new BytesRef("abc"), new BytesRef("bcd"), random().nextBoolean(), random().nextBoolean()),
      SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("abc"), new BytesRef("bcd"), random().nextBoolean(), random().nextBoolean()))) {
    Weight w = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1);
    assertNull(w.scorer(searcher.getIndexReader().leaves().get(0)));
  }
  reader.close();
  dir.close();
}
 
Example 10
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend2() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example 11
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend3() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  // Also add another document so there is a segment to write here:
  w2.addDocument(new Document());
  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example 12
Source Project: lucene-solr   Source File: ICUCollationField.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example 13
Source Project: lucene-solr   Source File: CollationField.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example 14
Source Project: lucene-solr   Source File: EnumField.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
  if (sf.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    final IndexableField field = createField(sf, value);
    fields.add(field);

    if (sf.multiValued()) {
      BytesRefBuilder bytes = new BytesRefBuilder();
      readableToIndexed(enumMapping.stringValueToIntValue(value.toString()).toString(), bytes);
      fields.add(new SortedSetDocValuesField(sf.getName(), bytes.toBytesRef()));
    } else {
      final long bits = field.numericValue().intValue();
      fields.add(new NumericDocValuesField(sf.getName(), bits));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(sf, value));
  }
}
 
Example 15
Source Project: lucene-solr   Source File: TestSortableTextField.java    License: Apache License 2.0 6 votes vote down vote up
public void testWhiteboxCreateFields() throws Exception {
  List<IndexableField> values = null;

  // common case...
  for (String field : Arrays.asList("keyword_stxt", "keyword_dv_stxt",
                                    "whitespace_stxt", "whitespace_f_stxt", "whitespace_l_stxt")) {
    values = createIndexableFields(field);
    assertEquals(field, 2, values.size());
    assertThat(field, values.get(0), instanceOf(Field.class));
    assertThat(field, values.get(1), instanceOf(SortedDocValuesField.class));
  }
  
  // special cases...
  values = createIndexableFields("whitespace_nois_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(SortedDocValuesField.class));
  //
  values = createIndexableFields("whitespace_nodv_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  //
  values = createIndexableFields("whitespace_m_stxt");
  assertEquals(2, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  assertThat(values.get(1), instanceOf(SortedSetDocValuesField.class));      
}
 
Example 16
Source Project: lucene-solr   Source File: SortableBinaryField.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    IndexableField storedField = createField(field, value);
    fields.add(storedField);
    ByteBuffer byteBuffer = toObject(storedField);
    BytesRef bytes = new BytesRef
        (byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example 17
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example 18
Source Project: crate   Source File: GroupByOptimizedIteratorTest.java    License: Apache License 2.0 6 votes vote down vote up
@Before
public void prepare() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    columnName = "x";
    expectedResult = new ArrayList<>(20);
    for (long i = 0; i < 20; i++) {
        Document doc = new Document();
        String val = "val_" + i;
        doc.add(new SortedSetDocValuesField(columnName, new BytesRef(val)));
        iw.addDocument(doc);
        expectedResult.add(new Object[] { val, 1L });
    }
    iw.commit();
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    inExpr = new InputCollectExpression(0);
    CountAggregation aggregation = (CountAggregation) getFunctions().getQualified(
        CountAggregation.COUNT_STAR_SIGNATURE,
        Collections.emptyList(),
        CountAggregation.COUNT_STAR_SIGNATURE.getReturnType().createType()
    );
    aggregationContexts = List.of(new AggregationContext(aggregation, () -> true, List.of()));
}
 
Example 19
Source Project: Elasticsearch   Source File: TypeFieldMapper.java    License: Apache License 2.0 5 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), context.type(), fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(context.type())));
    }
}
 
Example 20
Source Project: lucene-solr   Source File: TestMemoryIndex.java    License: Apache License 2.0 5 votes vote down vote up
public void testDocValues_resetIterator() throws Exception {
  Document doc = new Document();

  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));

  doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L));

  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
  assertEquals(3, sortedSetDocValues.getValueCount());
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedSetDocValues.advanceExact(0));
    assertEquals(0L, sortedSetDocValues.nextOrd());
    assertEquals(1L, sortedSetDocValues.nextOrd());
    assertEquals(2L, sortedSetDocValues.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
  }

  SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedNumericDocValues.advanceExact(0));
    assertEquals(5, sortedNumericDocValues.docValueCount());
    assertEquals(30L, sortedNumericDocValues.nextValue());
    assertEquals(31L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(33L, sortedNumericDocValues.nextValue());
  }

}
 
Example 21
Source Project: lucene-solr   Source File: TestJoinUtil.java    License: Apache License 2.0 5 votes vote down vote up
private void addLinkFields(final Random random, Document document, final String fieldName, String linkValue,
    boolean multipleValuesPerDocument, boolean globalOrdinalJoin) {
  document.add(newTextField(random, fieldName, linkValue, Field.Store.NO));

  final int linkInt = Integer.parseUnsignedInt(linkValue,16);
  document.add(new IntPoint(fieldName + "INT", linkInt));
  document.add(new FloatPoint(fieldName + "FLOAT", linkInt));

  final long linkLong = linkInt<<32 | linkInt;
  document.add(new LongPoint(fieldName + "LONG", linkLong));
  document.add(new DoublePoint(fieldName + "DOUBLE", linkLong));

  if (multipleValuesPerDocument) {
    document.add(new SortedSetDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new SortedNumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new SortedNumericDocValuesField(fieldName+ "FLOAT", Float.floatToRawIntBits(linkInt)));
    document.add(new SortedNumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new SortedNumericDocValuesField(fieldName+ "DOUBLE", Double.doubleToRawLongBits(linkLong)));
  } else {
    document.add(new SortedDocValuesField(fieldName, new BytesRef(linkValue)));
    document.add(new NumericDocValuesField(fieldName+ "INT", linkInt));
    document.add(new FloatDocValuesField(fieldName+ "FLOAT", linkInt));
    document.add(new NumericDocValuesField(fieldName+ "LONG", linkLong));
    document.add(new DoubleDocValuesField(fieldName+ "DOUBLE", linkLong));
  }
  if (globalOrdinalJoin) {
    document.add(new SortedDocValuesField("join_field", new BytesRef(linkValue)));
  }
}
 
Example 22
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedSetTwoFields() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
  
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  doc.add(new SortedSetDocValuesField("field2", new BytesRef("world")));
  iwriter.addDocument(doc);
  
  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();
  
  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(0, dv.nextDoc());
  
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);
  
  dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field2");

  assertEquals(0, dv.nextDoc());
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("world"), bytes);
  
  ireader.close();
  directory.close();
}
 
Example 23
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedSetTwoValues() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
  
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
  iwriter.addDocument(doc);
  
  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();
  
  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(0, dv.nextDoc());
  
  assertEquals(0, dv.nextOrd());
  assertEquals(1, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);
  
  bytes = dv.lookupOrd(1);
  assertEquals(new BytesRef("world"), bytes);

  ireader.close();
  directory.close();
}
 
Example 24
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedSetTwoValuesUnordered() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
  
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("world")));
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  
  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();
  
  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(0, dv.nextDoc());
  
  assertEquals(0, dv.nextOrd());
  assertEquals(1, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);
  
  bytes = dv.lookupOrd(1);
  assertEquals(new BytesRef("world"), bytes);

  ireader.close();
  directory.close();
}
 
Example 25
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedSetEnumAdvanceIndependently() throws IOException {
  Directory directory = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
  iwconfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);
  
  Document doc = new Document();
  SortedSetDocValuesField field1 = new SortedSetDocValuesField("field", new BytesRef("2"));
  SortedSetDocValuesField field2 = new SortedSetDocValuesField("field", new BytesRef("3"));
  doc.add(field1);
  doc.add(field2);
  iwriter.addDocument(doc);
  field1.setBytesValue(new BytesRef("1"));
  iwriter.addDocument(doc);
  field2.setBytesValue(new BytesRef("2"));
  iwriter.addDocument(doc);

  iwriter.commit();
  iwriter.forceMerge(1);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  doTestSortedSetEnumAdvanceIndependently(dv);

  ireader.close();
  directory.close();
}
 
Example 26
Source Project: lucene-solr   Source File: BaseDocValuesFormatTestCase.java    License: Apache License 2.0 5 votes vote down vote up
public void testSortedSetMergeAwayAllValuesLargeSegment() throws IOException {
  Directory directory = newDirectory();
  Analyzer analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwconfig = newIndexWriterConfig(analyzer);
  iwconfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory, iwconfig);

  Document doc = new Document();
  doc.add(new StringField("id", "1", Field.Store.NO));
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  final int numEmptyDocs = atLeast(1024);
  for (int i = 0; i < numEmptyDocs; ++i) {
    iwriter.addDocument(new Document());
  }
  iwriter.commit();
  iwriter.deleteDocuments(new Term("id", "1"));
  iwriter.forceMerge(1);

  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();

  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(NO_MORE_DOCS, dv.nextDoc());

  ireader.close();
  directory.close();
}
 
Example 27
Source Project: lucene-solr   Source File: TestMinShouldMatch2.java    License: Apache License 2.0 5 votes vote down vote up
private static void addSome(Document doc, String values[]) {
  List<String> list = Arrays.asList(values);
  Collections.shuffle(list, random());
  int howMany = TestUtil.nextInt(random(), 1, list.size());
  for (int i = 0; i < howMany; i++) {
    doc.add(new StringField("field", list.get(i), Field.Store.NO));
    doc.add(new SortedSetDocValuesField("dv", new BytesRef(list.get(i))));
  }
}
 
Example 28
Source Project: lucene-solr   Source File: TestDocValuesQueries.java    License: Apache License 2.0 5 votes vote down vote up
public void testEquals() {
  Query q1 = SortedNumericDocValuesField.newSlowRangeQuery("foo", 3, 5);
  QueryUtils.checkEqual(q1, SortedNumericDocValuesField.newSlowRangeQuery("foo", 3, 5));
  QueryUtils.checkUnequal(q1, SortedNumericDocValuesField.newSlowRangeQuery("foo", 3, 6));
  QueryUtils.checkUnequal(q1, SortedNumericDocValuesField.newSlowRangeQuery("foo", 4, 5));
  QueryUtils.checkUnequal(q1, SortedNumericDocValuesField.newSlowRangeQuery("bar", 3, 5));

  Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
  QueryUtils.checkEqual(q2, SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true));
  QueryUtils.checkUnequal(q2, SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("baz"), new BytesRef("baz"), true, true));
  QueryUtils.checkUnequal(q2, SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("bar"), true, true));
  QueryUtils.checkUnequal(q2, SortedSetDocValuesField.newSlowRangeQuery("quux", new BytesRef("bar"), new BytesRef("baz"), true, true));
}
 
Example 29
Source Project: lucene-solr   Source File: TestDocValues.java    License: Apache License 2.0 5 votes vote down vote up
/** 
 * field with sortedset docvalues
 */
public void testSortedSetField() throws Exception {
  Directory dir = newDirectory();
  IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(null));
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("bar")));
  iw.addDocument(doc);
  DirectoryReader dr = DirectoryReader.open(iw);
  LeafReader r = getOnlyLeafReader(dr);
  
  // ok
  assertNotNull(DocValues.getSortedSet(r, "foo"));
  
  // errors
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getBinary(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getNumeric(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSorted(r, "foo");
  });
  expectThrows(IllegalStateException.class, () -> {
    DocValues.getSortedNumeric(r, "foo");
  });
  
  dr.close();
  iw.close();
  dir.close();
}
 
Example 30
Source Project: lucene-solr   Source File: TestDocValuesIndexing.java    License: Apache License 2.0 5 votes vote down vote up
public void testTooLargeTermSortedSetBytes() throws IOException {
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("dv", new BytesRef("just fine")));
  iwriter.addDocument(doc);
  
  Document hugeDoc = new Document();
  byte bytes[] = new byte[100000];
  BytesRef b = new BytesRef(bytes);
  random().nextBytes(bytes);
  hugeDoc.add(new SortedSetDocValuesField("dv", b));
  expectThrows(IllegalArgumentException.class, () -> {
    iwriter.addDocument(hugeDoc);
  });

  IndexReader ir = iwriter.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  iwriter.close();
  directory.close();
}