org.apache.lucene.index.DocValues Java Examples

The following examples show how to use org.apache.lucene.index.DocValues. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source Project: Elasticsearch   Author: baidu   File: AbstractAtomicOrdinalsFieldData.java    License: Apache License 2.0 6 votes vote down vote up
public static AtomicOrdinalsFieldData empty() {
    return new AbstractAtomicOrdinalsFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public RandomAccessOrds getOrdinalsValues() {
            return DocValues.emptySortedSet();
        }
    };
}
 
Example #2
Source Project: lucene-solr   Author: apache   File: FieldCacheImpl.java    License: Apache License 2.0 6 votes vote down vote up
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 
Example #3
Source Project: lucene-solr   Author: apache   File: SortedSetFieldSource.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public FunctionValues getValues(Map<Object, Object> context, LeafReaderContext readerContext) throws IOException {
  SortedSetDocValues sortedSet = DocValues.getSortedSet(readerContext.reader(), field);
  SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
  return new DocTermsIndexDocValues(this, view) {
    @Override
    protected String toTerm(String readableValue) {
      return readableValue;
    }

    @Override
    public Object objectVal(int doc) throws IOException {
      return strVal(doc);
    }
  };
}
 
Example #4
Source Project: lucene-solr   Author: apache   File: IntFieldWriter.java    License: Apache License 2.0 6 votes vote down vote up
public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  int val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (int) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = (int) vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(this.field, val);
  return true;
}
 
Example #5
Source Project: lucene-solr   Author: apache   File: TestPrefixCompletionQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 
Example #6
Source Project: crate   Author: crate   File: GroupingLongCollectorBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
@Benchmark
public LongObjectHashMap<Long> measureGroupingOnNumericDocValues() throws Exception {
    Weight weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    LeafReaderContext leaf = searcher.getTopReaderContext().leaves().get(0);
    Scorer scorer = weight.scorer(leaf);
    NumericDocValues docValues = DocValues.getNumeric(leaf.reader(), "x");
    DocIdSetIterator docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            long number = docValues.longValue();
            sumByKey.compute(number, (key, oldValue) -> {
                if (oldValue == null) {
                    return number;
                } else {
                    return oldValue + number;
                }
            });
        }
    }
    return sumByKey;
}
 
Example #7
Source Project: lucene-solr   Author: apache   File: LongValueFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 
Example #8
Source Project: lucene-solr   Author: apache   File: GlobalOrdinalsQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 
Example #9
Source Project: lucene-solr   Author: apache   File: FieldFacetStats.java    License: Apache License 2.0 6 votes vote down vote up
public void facetMissingNum(int docID) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }
  
  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  if (docID == topLevelSortedValues.docID()) {
    int ord = topLevelSortedValues.ordValue();
    Integer missingCount = missingStats.get(ord);
    if (missingCount == null) {
      missingStats.put(ord, 1);
    } else {
      missingStats.put(ord, missingCount + 1);
    }
  }
}
 
Example #10
Source Project: lucene-solr   Author: apache   File: GlobalOrdinalsWithScoreQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 
Example #11
Source Project: lucene-solr   Author: apache   File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getIntComparator(int numHits) {
  return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 
Example #12
Source Project: lucene-solr   Author: apache   File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 
Example #13
Source Project: lucene-solr   Author: apache   File: TopLevelJoinQuery.java    License: Apache License 2.0 6 votes vote down vote up
private SortedSetDocValues validateAndFetchDocValues(SolrIndexSearcher solrSearcher, String fieldName, String querySide) throws IOException {
  final IndexSchema schema = solrSearcher.getSchema();
  final SchemaField field = schema.getFieldOrNull(fieldName);
  if (field == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, querySide + " field '" + fieldName + "' does not exist");
  }

  if (!field.hasDocValues()) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "'top-level' join queries require both 'from' and 'to' fields to have docValues, but " + querySide +
            " field [" + fieldName +  "] does not.");
  }

  final LeafReader leafReader = solrSearcher.getSlowAtomicReader();
  if (field.multiValued()) {
    return DocValues.getSortedSet(leafReader, fieldName);
  }
  return DocValues.singleton(DocValues.getSorted(leafReader, fieldName));
}
 
Example #14
Source Project: crate   Author: crate   File: GroupingLongCollectorBenchmark.java    License: Apache License 2.0 6 votes vote down vote up
@Benchmark
public LongObjectHashMap<Long> measureGroupingOnSortedNumericDocValues() throws Exception {
    var weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    var leaf = searcher.getTopReaderContext().leaves().get(0);
    var scorer = weight.scorer(leaf);
    var docValues = DocValues.getSortedNumeric(leaf.reader(), "y");
    var docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            if (docValues.docValueCount() == 1) {
                long number = docValues.nextValue();
                sumByKey.compute(number, (key, oldValue) -> {
                    if (oldValue == null) {
                        return number;
                    } else {
                        return oldValue + number;
                    }
                });
            }
        }
    }
    return sumByKey;
}
 
Example #15
Source Project: Elasticsearch   Author: baidu   File: MultiOrdinals.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RandomAccessOrds ordinals(ValuesHolder values) {
    if (multiValued) {
        return new MultiDocs(this, values);
    } else {
        return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values));
    }
}
 
Example #16
Source Project: Elasticsearch   Author: baidu   File: SortedNumericDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDocValues getLongValues() {
    try {
        return DocValues.getSortedNumeric(reader, field);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #17
Source Project: Elasticsearch   Author: baidu   File: SortedNumericDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

        NumericDocValues single = DocValues.unwrapSingleton(raw);
        if (single != null) {
            return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw));
        } else {
            return new MultiFloatValues(raw);
        }
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #18
Source Project: Elasticsearch   Author: baidu   File: SortedNumericDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
        return FieldData.sortableLongBitsToDoubles(raw);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #19
Source Project: Elasticsearch   Author: baidu   File: AtomicLongFieldData.java    License: Apache License 2.0 5 votes vote down vote up
public static AtomicNumericFieldData empty(final int maxDoc) {
    return new AtomicLongFieldData(0) {

        @Override
        public SortedNumericDocValues getLongValues() {
            return DocValues.emptySortedNumeric(maxDoc);
        }

        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

    };
}
 
Example #20
Source Project: lucene-solr   Author: apache   File: DocTermOrds.java    License: Apache License 2.0 5 votes vote down vote up
/** Returns a SortedSetDocValues view of this instance */
public SortedSetDocValues iterator(LeafReader reader) throws IOException {
  if (isEmpty()) {
    return DocValues.emptySortedSet();
  } else {
    return new Iterator(reader);
  }
}
 
Example #21
Source Project: Elasticsearch   Author: baidu   File: AbstractAtomicParentChildFieldData.java    License: Apache License 2.0 5 votes vote down vote up
public static AtomicParentChildFieldData empty() {
    return new AbstractAtomicParentChildFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public SortedDocValues getOrdinalsValues(String type) {
            return DocValues.emptySorted();
        }

        @Override
        public Set<String> types() {
            return ImmutableSet.of();
        }
    };
}
 
Example #22
Source Project: Elasticsearch   Author: baidu   File: BytesBinaryDVIndexFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public BytesBinaryDVAtomicFieldData load(LeafReaderContext context) {
    try {
        return new BytesBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName()));
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #23
Source Project: Elasticsearch   Author: baidu   File: SortedSetDVBytesAtomicFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public RandomAccessOrds getOrdinalsValues() {
    try {
        return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field));
    } catch (IOException e) {
        throw new IllegalStateException("cannot load docvalues", e);
    }
}
 
Example #24
Source Project: Elasticsearch   Author: baidu   File: BinaryDVAtomicFieldData.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedBinaryDocValues getBytesValues() {
    try {
        final BinaryDocValues values = DocValues.getBinary(reader, field);
        final Bits docsWithField = DocValues.getDocsWithField(reader, field);
        return FieldData.singleton(values, docsWithField);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #25
Source Project: lucene-solr   Author: apache   File: TestSortRandom.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(context.docBase ^ seed);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        assertEquals(docID, idSource.nextDoc());
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #26
Source Project: lucene-solr   Author: apache   File: TestFieldCacheSortRandom.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(seed ^ context.docBase);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          assertEquals(docID, idSource.advance(docID));
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }
  };
}
 
Example #27
Source Project: lucene-solr   Author: apache   File: Geo3DPointDistanceComparator.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
  LeafReader reader = context.reader();
  FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info != null) {
    Geo3DDocValuesField.checkCompatible(info);
  }
  currentDocs = DocValues.getSortedNumeric(reader, field);
  return this;
}
 
Example #28
Source Project: lucene-solr   Author: apache   File: DistanceValueSource.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * Returns the FunctionValues used by the function query.
 */
@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
  LeafReader reader = readerContext.reader();

  final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
  final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());

  return DoubleValues.withDefault(new DoubleValues() {

    private final Point from = DistanceValueSource.this.from;
    private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();

    @Override
    public double doubleValue() throws IOException {
      double x = Double.longBitsToDouble(ptX.longValue());
      double y = Double.longBitsToDouble(ptY.longValue());
      return calculator.distance(from, x, y) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return ptX.advanceExact(doc) && ptY.advanceExact(doc);
    }

  }, nullValue);
}
 
Example #29
Source Project: lucene-solr   Author: apache   File: DocTermsIndexDocValues.java    License: Apache License 2.0 5 votes vote down vote up
static SortedDocValues open(LeafReaderContext context, String field) throws IOException {
  try {
    return DocValues.getSorted(context.reader(), field);
  } catch (RuntimeException e) {
    throw new DocTermsIndexException(field, e);
  }
}
 
Example #30
Source Project: lucene-solr   Author: apache   File: GlobalOrdinalsWithScoreCollector.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}