org.apache.lucene.document.SortedSetDocValuesField Java Examples

The following examples show how to use org.apache.lucene.document.SortedSetDocValuesField. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: StringFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    ValueAndBoost valueAndBoost = parseCreateFieldForString(context, fieldType().nullValueAsString(), fieldType().boost());
    if (valueAndBoost.value() == null) {
        return;
    }
    if (ignoreAbove > 0 && valueAndBoost.value().length() > ignoreAbove) {
        return;
    }
    if (context.includeInAll(includeInAll, this)) {
        context.allEntries().addText(fieldType().names().fullName(), valueAndBoost.value(), valueAndBoost.boost());
    }

    if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) {
        Field field = new Field(fieldType().names().indexName(), valueAndBoost.value(), fieldType());
        field.setBoost(valueAndBoost.boost());
        fields.add(field);
    }
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(valueAndBoost.value())));
    }
}
 
Example #2
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend2() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example #3
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMixedTypesAfterReopenAppend3() throws IOException {
  Directory dir = newDirectory();
  IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random()))) ;
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("foo", new BytesRef("foo")));
  w.addDocument(doc);
  w.close();

  Document doc2 = new Document();
  IndexWriter w2 = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
  doc2.add(new StringField("foo", "bar", Field.Store.NO));
  doc2.add(new BinaryDocValuesField("foo", new BytesRef("foo")));
  // NOTE: this case follows a different code path inside
  // DefaultIndexingChain/FieldInfos, because the field (foo)
  // is first added without DocValues:
  expectThrows(IllegalArgumentException.class, () -> {
    w2.addDocument(doc2);
  });

  // Also add another document so there is a segment to write here:
  w2.addDocument(new Document());
  w2.forceMerge(1);
  w2.close();
  dir.close();
}
 
Example #4
Source File: TestDocValuesQueries.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMissingField() throws IOException {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir);
  iw.addDocument(new Document());
  IndexReader reader = iw.getReader();
  iw.close();
  IndexSearcher searcher = newSearcher(reader);
  for (Query query : Arrays.asList(
      NumericDocValuesField.newSlowRangeQuery("foo", 2, 4),
      SortedNumericDocValuesField.newSlowRangeQuery("foo", 2, 4),
      SortedDocValuesField.newSlowRangeQuery("foo", new BytesRef("abc"), new BytesRef("bcd"), random().nextBoolean(), random().nextBoolean()),
      SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("abc"), new BytesRef("bcd"), random().nextBoolean(), random().nextBoolean()))) {
    Weight w = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE, 1);
    assertNull(w.scorer(searcher.getIndexReader().leaves().get(0)));
  }
  reader.close();
  dir.close();
}
 
Example #5
Source File: TestDocValuesQueries.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testToString() {
  Query q1 = SortedNumericDocValuesField.newSlowRangeQuery("foo", 3, 5);
  assertEquals("foo:[3 TO 5]", q1.toString());
  assertEquals("[3 TO 5]", q1.toString("foo"));
  assertEquals("foo:[3 TO 5]", q1.toString("bar"));

  Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
  assertEquals("foo:[[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, true);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, false);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), null, true, true);
  assertEquals("foo:[[62 61 72] TO *}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", null, new BytesRef("baz"), true, true);
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString());
  assertEquals("{* TO [62 61 7a]]", q2.toString("foo"));
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString("bar"));
}
 
Example #6
Source File: ICUCollationField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example #7
Source File: CollationField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    fields.add(createField(field, value));
    final BytesRef bytes = getCollationKey(field.getName(), value.toString());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example #8
Source File: EnumField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField sf, Object value) {
  if (sf.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    final IndexableField field = createField(sf, value);
    fields.add(field);

    if (sf.multiValued()) {
      BytesRefBuilder bytes = new BytesRefBuilder();
      readableToIndexed(enumMapping.stringValueToIntValue(value.toString()).toString(), bytes);
      fields.add(new SortedSetDocValuesField(sf.getName(), bytes.toBytesRef()));
    } else {
      final long bits = field.numericValue().intValue();
      fields.add(new NumericDocValuesField(sf.getName(), bits));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(sf, value));
  }
}
 
Example #9
Source File: TestSortableTextField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testWhiteboxCreateFields() throws Exception {
  List<IndexableField> values = null;

  // common case...
  for (String field : Arrays.asList("keyword_stxt", "keyword_dv_stxt",
                                    "whitespace_stxt", "whitespace_f_stxt", "whitespace_l_stxt")) {
    values = createIndexableFields(field);
    assertEquals(field, 2, values.size());
    assertThat(field, values.get(0), instanceOf(Field.class));
    assertThat(field, values.get(1), instanceOf(SortedDocValuesField.class));
  }
  
  // special cases...
  values = createIndexableFields("whitespace_nois_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(SortedDocValuesField.class));
  //
  values = createIndexableFields("whitespace_nodv_stxt");
  assertEquals(1, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  //
  values = createIndexableFields("whitespace_m_stxt");
  assertEquals(2, values.size());
  assertThat(values.get(0), instanceOf(Field.class));
  assertThat(values.get(1), instanceOf(SortedSetDocValuesField.class));      
}
 
Example #10
Source File: BaseDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testSortedSetOneValue() throws IOException {
  Directory directory = newDirectory();
  RandomIndexWriter iwriter = new RandomIndexWriter(random(), directory);
  
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("field", new BytesRef("hello")));
  iwriter.addDocument(doc);
  
  DirectoryReader ireader = iwriter.getReader();
  iwriter.close();
  
  SortedSetDocValues dv = getOnlyLeafReader(ireader).getSortedSetDocValues("field");
  assertEquals(0, dv.nextDoc());
  assertEquals(0, dv.nextOrd());
  assertEquals(NO_MORE_ORDS, dv.nextOrd());
  
  BytesRef bytes = dv.lookupOrd(0);
  assertEquals(new BytesRef("hello"), bytes);

  ireader.close();
  directory.close();
}
 
Example #11
Source File: BaseDocValuesFormatTestCase.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected void addRandomFields(Document doc) {
  if (usually()) {
    doc.add(new NumericDocValuesField("ndv", random().nextInt(1 << 12)));
    doc.add(new BinaryDocValuesField("bdv", new BytesRef(TestUtil.randomSimpleString(random()))));
    doc.add(new SortedDocValuesField("sdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  int numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedSetDocValuesField("ssdv", new BytesRef(TestUtil.randomSimpleString(random(), 2))));
  }
  numValues = random().nextInt(5);
  for (int i = 0; i < numValues; ++i) {
    doc.add(new SortedNumericDocValuesField("sndv", TestUtil.nextLong(random(), Long.MIN_VALUE, Long.MAX_VALUE)));
  }
}
 
Example #12
Source File: TestIndexSortSortedNumericDocValuesRangeQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testToString() {
  Query q1 = createQuery("foo", 3, 5);
  assertEquals("foo:[3 TO 5]", q1.toString());
  assertEquals("[3 TO 5]", q1.toString("foo"));
  assertEquals("foo:[3 TO 5]", q1.toString("bar"));

  Query q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), true, true);
  assertEquals("foo:[[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, true);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]]", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), new BytesRef("baz"), false, false);
  assertEquals("foo:{[62 61 72] TO [62 61 7a]}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", new BytesRef("bar"), null, true, true);
  assertEquals("foo:[[62 61 72] TO *}", q2.toString());
  q2 = SortedSetDocValuesField.newSlowRangeQuery("foo", null, new BytesRef("baz"), true, true);
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString());
  assertEquals("{* TO [62 61 7a]]", q2.toString("foo"));
  assertEquals("foo:{* TO [62 61 7a]]", q2.toString("bar"));
}
 
Example #13
Source File: FacetsConfig.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void processSSDVFacetFields(Map<String,List<SortedSetDocValuesFacetField>> byField, Document doc) throws IOException {
  //System.out.println("process SSDV: " + byField);
  for(Map.Entry<String,List<SortedSetDocValuesFacetField>> ent : byField.entrySet()) {

    String indexFieldName = ent.getKey();
    //System.out.println("  field=" + indexFieldName);

    for(SortedSetDocValuesFacetField facetField : ent.getValue()) {
      FacetLabel cp = new FacetLabel(facetField.dim, facetField.label);
      String fullPath = pathToString(cp.components, cp.length);
      //System.out.println("add " + fullPath);

      // For facet counts:
      doc.add(new SortedSetDocValuesField(indexFieldName, new BytesRef(fullPath)));

      // For drill-down:
      doc.add(new StringField(indexFieldName, fullPath, Field.Store.NO));

      FacetsConfig.DimConfig ft = getDimConfig(facetField.dim);        
      if (ft.requireDimensionDrillDown) {
        doc.add(new StringField(indexFieldName, facetField.dim, Field.Store.NO));
      }
    }
  }
}
 
Example #14
Source File: SortableBinaryField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public List<IndexableField> createFields(SchemaField field, Object value) {
  if (field.hasDocValues()) {
    List<IndexableField> fields = new ArrayList<>();
    IndexableField storedField = createField(field, value);
    fields.add(storedField);
    ByteBuffer byteBuffer = toObject(storedField);
    BytesRef bytes = new BytesRef
        (byteBuffer.array(), byteBuffer.arrayOffset() + byteBuffer.position(), byteBuffer.remaining());
    if (field.multiValued()) {
      fields.add(new SortedSetDocValuesField(field.getName(), bytes));
    } else {
      fields.add(new SortedDocValuesField(field.getName(), bytes));
    }
    return fields;
  } else {
    return Collections.singletonList(createField(field, value));
  }
}
 
Example #15
Source File: AnalyzingInfixSuggester.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private Document buildDocument(BytesRef text, Set<BytesRef> contexts, long weight, BytesRef payload) throws IOException {
  String textString = text.utf8ToString();
  Document doc = new Document();
  FieldType ft = getTextFieldType();
  doc.add(new Field(TEXT_FIELD_NAME, textString, ft));
  if (minPrefixChars>0) {
    doc.add(new Field(TEXTGRAMS_FIELD_NAME, textString, ft));
  }
  doc.add(new StringField(EXACT_TEXT_FIELD_NAME, textString, Field.Store.NO));
  doc.add(new BinaryDocValuesField(TEXT_FIELD_NAME, text));
  doc.add(new NumericDocValuesField("weight", weight));
  if (payload != null) {
    doc.add(new BinaryDocValuesField("payloads", payload));
  }
  if (contexts != null) {
    for(BytesRef context : contexts) {
      doc.add(new StringField(CONTEXTS_FIELD_NAME, context, Field.Store.NO));
      doc.add(new SortedSetDocValuesField(CONTEXTS_FIELD_NAME, context));
    }
  }
  return doc;
}
 
Example #16
Source File: SecureAtomicReaderTestBase.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
private Iterable<IndexableField> getDoc(int i) {
  Document document = new Document();
  document.add(new StringField("test", "test", Store.YES));
  document.add(new StringField("info", "info", Store.YES));
  if (i == 3) {
    document.add(new StringField("shouldnotsee", "shouldnotsee", Store.YES));
  }
  if (i == 5) {
    document.add(new StringField("termmask", "term", Store.YES));
  }
  document.add(new NumericDocValuesField("number", i));
  document.add(new BinaryDocValuesField("bin", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedDocValuesField("sorted", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(Integer.toString(i).getBytes())));
  document.add(new SortedSetDocValuesField("sortedset", new BytesRef(("0" + Integer.toString(i)).getBytes())));
  return document;
}
 
Example #17
Source File: DocValuesAdapterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
protected void createIndex() throws IOException {
  indexDir = createTempDir("testIndex");

  Directory dir = newFSDirectory(indexDir);
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockAnalyzer(random()));

  Document doc = new Document();
  doc.add(new BinaryDocValuesField("dv_binary", new BytesRef("lucene")));
  doc.add(new SortedDocValuesField("dv_sorted", new BytesRef("abc")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("python")));
  doc.add(new SortedSetDocValuesField("dv_sortedset", new BytesRef("java")));
  doc.add(new NumericDocValuesField("dv_numeric", 42L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 22L));
  doc.add(new SortedNumericDocValuesField("dv_sortednumeric", 11L));
  doc.add(newStringField("no_dv", "aaa", Field.Store.NO));
  writer.addDocument(doc);

  writer.commit();
  writer.close();
  dir.close();
}
 
Example #18
Source File: GroupByOptimizedIteratorTest.java    From crate with Apache License 2.0 6 votes vote down vote up
@Before
public void prepare() throws Exception {
    IndexWriter iw = new IndexWriter(new ByteBuffersDirectory(), new IndexWriterConfig(new StandardAnalyzer()));
    columnName = "x";
    expectedResult = new ArrayList<>(20);
    for (long i = 0; i < 20; i++) {
        Document doc = new Document();
        String val = "val_" + i;
        doc.add(new SortedSetDocValuesField(columnName, new BytesRef(val)));
        iw.addDocument(doc);
        expectedResult.add(new Object[] { val, 1L });
    }
    iw.commit();
    indexSearcher = new IndexSearcher(DirectoryReader.open(iw));

    inExpr = new InputCollectExpression(0);
    CountAggregation aggregation = (CountAggregation) getFunctions().getQualified(
        CountAggregation.COUNT_STAR_SIGNATURE,
        Collections.emptyList(),
        CountAggregation.COUNT_STAR_SIGNATURE.getReturnType().createType()
    );
    aggregationContexts = List.of(new AggregationContext(aggregation, () -> true, List.of()));
}
 
Example #19
Source File: TestDocValuesIndexing.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testTooLargeTermSortedSetBytes() throws IOException {
  Analyzer analyzer = new MockAnalyzer(random());

  Directory directory = newDirectory();
  // we don't use RandomIndexWriter because it might add more docvalues than we expect !!!!1
  IndexWriterConfig iwc = newIndexWriterConfig(analyzer);
  iwc.setMergePolicy(newLogMergePolicy());
  IndexWriter iwriter = new IndexWriter(directory, iwc);
  Document doc = new Document();
  doc.add(new SortedSetDocValuesField("dv", new BytesRef("just fine")));
  iwriter.addDocument(doc);
  
  Document hugeDoc = new Document();
  byte bytes[] = new byte[100000];
  BytesRef b = new BytesRef(bytes);
  random().nextBytes(bytes);
  hugeDoc.add(new SortedSetDocValuesField("dv", b));
  expectThrows(IllegalArgumentException.class, () -> {
    iwriter.addDocument(hugeDoc);
  });

  IndexReader ir = iwriter.getReader();
  assertEquals(1, ir.numDocs());
  ir.close();
  iwriter.close();
  directory.close();
}
 
Example #20
Source File: IpColumnReferenceTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private static void addIPv4Values(IndexWriter writer) throws IOException {
    for (int i = 0; i < 10; i++) {
        Document doc = new Document();
        doc.add(new StringField("_id", Integer.toString(i), Field.Store.NO));
        InetAddress address = InetAddresses.forString("192.168.0." + i);
        doc.add(new SortedSetDocValuesField(IP_COLUMN, new BytesRef(InetAddressPoint.encode(address))));
        if (i == 0) {
            address = InetAddresses.forString("192.168.0.1");
            doc.add(new SortedSetDocValuesField(IP_ARRAY_COLUMN, new BytesRef(InetAddressPoint.encode(address))));
            address = InetAddresses.forString("192.168.0.2");
            doc.add(new SortedSetDocValuesField(IP_ARRAY_COLUMN, new BytesRef(InetAddressPoint.encode(address))));
        }
        writer.addDocument(doc);
    }
}
 
Example #21
Source File: TypeFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
protected void parseCreateField(ParseContext context, List<Field> fields) throws IOException {
    if (fieldType().indexOptions() == IndexOptions.NONE && !fieldType().stored()) {
        return;
    }
    fields.add(new Field(fieldType().names().indexName(), context.type(), fieldType()));
    if (fieldType().hasDocValues()) {
        fields.add(new SortedSetDocValuesField(fieldType().names().indexName(), new BytesRef(context.type())));
    }
}
 
Example #22
Source File: IpColumnReferenceTest.java    From crate with Apache License 2.0 5 votes vote down vote up
private static void addIPv6Values(IndexWriter writer) throws IOException {
    for (int i = 10; i < 20; i++) {
        Document doc = new Document();
        doc.add(new StringField("_id", Integer.toString(i), Field.Store.NO));
        InetAddress address = InetAddresses.forString("7bd0:8082:2df8:487e:e0df:e7b5:9362:" + Integer.toHexString(i));
        doc.add(new SortedSetDocValuesField(IP_COLUMN, new BytesRef(InetAddressPoint.encode(address))));
        writer.addDocument(doc);
    }
}
 
Example #23
Source File: TestExitableDirectoryReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void addDVs(Document d1, int i) {
  d1.add(new NumericDocValuesField("numeric", i));
  d1.add(new BinaryDocValuesField("binary", new BytesRef(""+i)));
  d1.add(new SortedDocValuesField("sorted", new BytesRef(""+i)));
  d1.add(new SortedNumericDocValuesField("sortednumeric", i));
  d1.add(new SortedSetDocValuesField("sortedset", new BytesRef(""+i)));
}
 
Example #24
Source File: TestIndexSorting.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testWrongSortFieldType() throws Exception {
  Directory dir = newDirectory();
  List<Field> dvs = new ArrayList<>();
  dvs.add(new SortedDocValuesField("field", new BytesRef("")));
  dvs.add(new SortedSetDocValuesField("field", new BytesRef("")));
  dvs.add(new NumericDocValuesField("field", 42));
  dvs.add(new SortedNumericDocValuesField("field", 42));

  List<SortField> sortFields = new ArrayList<>();
  sortFields.add(new SortField("field", SortField.Type.STRING));
  sortFields.add(new SortedSetSortField("field", false));
  sortFields.add(new SortField("field", SortField.Type.INT));
  sortFields.add(new SortedNumericSortField("field", SortField.Type.INT));

  for (int i = 0; i < sortFields.size(); i++) {
    for (int j = 0; j < dvs.size(); j++) {
      if (i == j) {
        continue;
      }
      Sort indexSort = new Sort(sortFields.get(i));
      IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
      iwc.setIndexSort(indexSort);
      IndexWriter w = new IndexWriter(dir, iwc);
      Document doc = new Document();
      doc.add(dvs.get(j));
      IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc));
      assertThat(exc.getMessage(), containsString("expected field [field] to be "));
      doc.clear();
      doc.add(dvs.get(i));
      w.addDocument(doc);
      doc.add(dvs.get(j));
      exc = expectThrows(IllegalArgumentException.class, () -> w.addDocument(doc));
      assertThat(exc.getMessage(), containsString("cannot change DocValues type"));
      w.rollback();
      IOUtils.close(w);
    }
  }
  IOUtils.close(dir);
}
 
Example #25
Source File: TestDuelingCodecs.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * populates a writer with random stuff. this must be fully reproducable with the seed!
 */
public static void createRandomIndex(int numdocs, RandomIndexWriter writer, long seed) throws IOException {
  Random random = new Random(seed);
  // primary source for our data is from linefiledocs, it's realistic.
  LineFileDocs lineFileDocs = new LineFileDocs(random);

  // TODO: we should add other fields that use things like docs&freqs but omit positions,
  // because linefiledocs doesn't cover all the possibilities.
  for (int i = 0; i < numdocs; i++) {
    Document document = lineFileDocs.nextDoc();
    // grab the title and add some SortedSet instances for fun
    String title = document.get("titleTokenized");
    String split[] = title.split("\\s+");
    document.removeFields("sortedset");
    for (String trash : split) {
      document.add(new SortedSetDocValuesField("sortedset", new BytesRef(trash)));
    }
    // add a numeric dv field sometimes
    document.removeFields("sparsenumeric");
    if (random.nextInt(4) == 2) {
      document.add(new NumericDocValuesField("sparsenumeric", random.nextInt()));
    }
    // add sortednumeric sometimes
    document.removeFields("sparsesortednum");
    if (random.nextInt(5) == 1) {
      document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
      if (random.nextBoolean()) {
        document.add(new SortedNumericDocValuesField("sparsesortednum", random.nextLong()));
      }
    }
    writer.addDocument(document);
  }
  
  lineFileDocs.close();
}
 
Example #26
Source File: ICUCollationField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected Query getSpecializedRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
  String f = field.getName();
  BytesRef low = part1 == null ? null : getCollationKey(f, part1);
  BytesRef high = part2 == null ? null : getCollationKey(f, part2);
  if (!field.indexed() && field.hasDocValues()) {
    return SortedSetDocValuesField.newSlowRangeQuery(
        field.getName(), low, high, minInclusive, maxInclusive);
  } else {
    return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive);
  }
}
 
Example #27
Source File: BasicStorageTest.java    From lumongo with Apache License 2.0 5 votes vote down vote up
private static void addDoc(IndexWriter w, String title, String uid) throws IOException {
	Document doc = new Document();
	doc.add(new TextField("title", title, Field.Store.YES));
	doc.add(new TextField("uid", uid, Field.Store.YES));
	doc.add(new StringField("uid", uid, Field.Store.YES));
	doc.add(new IntPoint("testIntField", 3));
	long date = System.currentTimeMillis();
	doc.add(new LongPoint("date", date));
	doc.add(new NumericDocValuesField("date", date));
	doc.add(new SortedSetDocValuesField("category", new BytesRef("Anything")));
	Term uidTerm = new Term("uid", uid);

	w.updateDocument(uidTerm, doc);
}
 
Example #28
Source File: SortableTextField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static List<IndexableField> getIndexableFields(SchemaField field, IndexableField f, BytesRef bytes) {
  final IndexableField docval = field.multiValued()
    ? new SortedSetDocValuesField(field.getName(), bytes)
    : new SortedDocValuesField(field.getName(), bytes);
  
  if (null == f) {
    return Collections.singletonList(docval);
  } 
  return Arrays.asList(f, docval);
}
 
Example #29
Source File: CollationField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
protected Query getSpecializedRangeQuery(QParser parser, SchemaField field, String part1, String part2, boolean minInclusive, boolean maxInclusive) {
  String f = field.getName();
  BytesRef low = part1 == null ? null : getCollationKey(f, part1);
  BytesRef high = part2 == null ? null : getCollationKey(f, part2);
  if (!field.indexed() && field.hasDocValues()) {
    return SortedSetDocValuesField.newSlowRangeQuery(
        field.getName(), low, high, minInclusive, maxInclusive);
  } else {
    return new TermRangeQuery(field.getName(), low, high, minInclusive, maxInclusive);
  }
}
 
Example #30
Source File: TestLegacyFieldCache.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testDocValuesIntegration() throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig iwc = newIndexWriterConfig(null);
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwc);
  Document doc = new Document();
  doc.add(new BinaryDocValuesField("binary", new BytesRef("binary value")));
  doc.add(new SortedDocValuesField("sorted", new BytesRef("sorted value")));
  doc.add(new NumericDocValuesField("numeric", 42));
  doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value1")));
  doc.add(new SortedSetDocValuesField("sortedset", new BytesRef("sortedset value2")));
  iw.addDocument(doc);
  DirectoryReader ir = iw.getReader();
  iw.close();
  LeafReader ar = getOnlyLeafReader(ir);
  
  // Binary type: can be retrieved via getTerms()
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "binary", FieldCache.LEGACY_INT_PARSER);
  });
  
  // Sorted type: can be retrieved via getTerms(), getTermsIndex(), getDocTermOrds()
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "sorted", FieldCache.LEGACY_INT_PARSER);
  });
  
  // Numeric type: can be retrieved via getInts() and so on
  NumericDocValues numeric = FieldCache.DEFAULT.getNumerics(ar, "numeric", FieldCache.LEGACY_INT_PARSER);
  assertEquals(0, numeric.nextDoc());
  assertEquals(42, numeric.longValue());
     
  // SortedSet type: can be retrieved via getDocTermOrds() 
  expectThrows(IllegalStateException.class, () -> {
    FieldCache.DEFAULT.getNumerics(ar, "sortedset", FieldCache.LEGACY_INT_PARSER);
  });
  
  ir.close();
  dir.close();
}