org.apache.lucene.index.DocValues Java Examples

The following examples show how to use org.apache.lucene.index.DocValues. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: FieldFacetStats.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void facetMissingNum(int docID) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }
  
  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  if (docID == topLevelSortedValues.docID()) {
    int ord = topLevelSortedValues.ordValue();
    Integer missingCount = missingStats.get(ord);
    if (missingCount == null) {
      missingStats.put(ord, 1);
    } else {
      missingStats.put(ord, missingCount + 1);
    }
  }
}
 
Example #2
Source File: FieldCacheImpl.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public SortedDocValues getTermsIndex(LeafReader reader, String field, float acceptableOverheadRatio) throws IOException {
  SortedDocValues valuesIn = reader.getSortedDocValues(field);
  if (valuesIn != null) {
    // Not cached here by FieldCacheImpl (cached instead
    // per-thread by SegmentReader):
    return valuesIn;
  } else {
    final FieldInfo info = reader.getFieldInfos().fieldInfo(field);
    if (info == null) {
      return DocValues.emptySorted();
    } else if (info.getDocValuesType() != DocValuesType.NONE) {
      // we don't try to build a sorted instance from numeric/binary doc
      // values because dedup can be very costly
      throw new IllegalStateException("Type mismatch: " + field + " was indexed as " + info.getDocValuesType());
    } else if (info.getIndexOptions() == IndexOptions.NONE) {
      return DocValues.emptySorted();
    }
    SortedDocValuesImpl impl = (SortedDocValuesImpl) caches.get(SortedDocValues.class).get(reader, new CacheKey(field, acceptableOverheadRatio));
    return impl.iterator();
  }
}
 
Example #3
Source File: SortedSetFieldSource.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public FunctionValues getValues(Map<Object, Object> context, LeafReaderContext readerContext) throws IOException {
  SortedSetDocValues sortedSet = DocValues.getSortedSet(readerContext.reader(), field);
  SortedDocValues view = SortedSetSelector.wrap(sortedSet, selector);
  return new DocTermsIndexDocValues(this, view) {
    @Override
    protected String toTerm(String readableValue) {
      return readableValue;
    }

    @Override
    public Object objectVal(int doc) throws IOException {
      return strVal(doc);
    }
  };
}
 
Example #4
Source File: IntFieldWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public boolean write(SortDoc sortDoc, LeafReader reader, MapWriter.EntryWriter ew, int fieldIndex) throws IOException {
  int val;
  SortValue sortValue = sortDoc.getSortValue(this.field);
  if (sortValue != null) {
    if (sortValue.isPresent()) {
      val = (int) sortValue.getCurrentValue();
    } else { //empty-value
      return false;
    }
  } else {
    // field is not part of 'sort' param, but part of 'fl' param
    NumericDocValues vals = DocValues.getNumeric(reader, this.field);
    if (vals.advance(sortDoc.docId) == sortDoc.docId) {
      val = (int) vals.longValue();
    } else {
      return false;
    }
  }
  ew.put(this.field, val);
  return true;
}
 
Example #5
Source File: TestPrefixCompletionQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 
Example #6
Source File: GroupingLongCollectorBenchmark.java    From crate with Apache License 2.0 6 votes vote down vote up
@Benchmark
public LongObjectHashMap<Long> measureGroupingOnNumericDocValues() throws Exception {
    Weight weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    LeafReaderContext leaf = searcher.getTopReaderContext().leaves().get(0);
    Scorer scorer = weight.scorer(leaf);
    NumericDocValues docValues = DocValues.getNumeric(leaf.reader(), "x");
    DocIdSetIterator docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            long number = docValues.longValue();
            sumByKey.compute(number, (key, oldValue) -> {
                if (oldValue == null) {
                    return number;
                } else {
                    return oldValue + number;
                }
            });
        }
    }
    return sumByKey;
}
 
Example #7
Source File: LongValueFacetCounts.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 
Example #8
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 
Example #9
Source File: AbstractAtomicOrdinalsFieldData.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
public static AtomicOrdinalsFieldData empty() {
    return new AbstractAtomicOrdinalsFieldData() {

        @Override
        public long ramBytesUsed() {
            return 0;
        }
        
        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

        @Override
        public void close() {
        }

        @Override
        public RandomAccessOrds getOrdinalsValues() {
            return DocValues.emptySortedSet();
        }
    };
}
 
Example #10
Source File: GlobalOrdinalsWithScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 
Example #11
Source File: ToParentBlockJoinSortField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getIntComparator(int numHits) {
  return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 
Example #12
Source File: ToParentBlockJoinSortField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 
Example #13
Source File: TopLevelJoinQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private SortedSetDocValues validateAndFetchDocValues(SolrIndexSearcher solrSearcher, String fieldName, String querySide) throws IOException {
  final IndexSchema schema = solrSearcher.getSchema();
  final SchemaField field = schema.getFieldOrNull(fieldName);
  if (field == null) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, querySide + " field '" + fieldName + "' does not exist");
  }

  if (!field.hasDocValues()) {
    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
        "'top-level' join queries require both 'from' and 'to' fields to have docValues, but " + querySide +
            " field [" + fieldName +  "] does not.");
  }

  final LeafReader leafReader = solrSearcher.getSlowAtomicReader();
  if (field.multiValued()) {
    return DocValues.getSortedSet(leafReader, fieldName);
  }
  return DocValues.singleton(DocValues.getSorted(leafReader, fieldName));
}
 
Example #14
Source File: GroupingLongCollectorBenchmark.java    From crate with Apache License 2.0 6 votes vote down vote up
@Benchmark
public LongObjectHashMap<Long> measureGroupingOnSortedNumericDocValues() throws Exception {
    var weight = searcher.createWeight(new MatchAllDocsQuery(), ScoreMode.COMPLETE_NO_SCORES, 1.0f);
    var leaf = searcher.getTopReaderContext().leaves().get(0);
    var scorer = weight.scorer(leaf);
    var docValues = DocValues.getSortedNumeric(leaf.reader(), "y");
    var docIt = scorer.iterator();
    LongObjectHashMap<Long> sumByKey = new LongObjectHashMap<>();
    for (int docId = docIt.nextDoc(); docId != DocIdSetIterator.NO_MORE_DOCS; docId = docIt.nextDoc()) {
        if (docValues.advanceExact(docId)) {
            if (docValues.docValueCount() == 1) {
                long number = docValues.nextValue();
                sumByKey.compute(number, (key, oldValue) -> {
                    if (oldValue == null) {
                        return number;
                    } else {
                        return oldValue + number;
                    }
                });
            }
        }
    }
    return sumByKey;
}
 
Example #15
Source File: TestSortRandom.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(context.docBase ^ seed);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        assertEquals(docID, idSource.nextDoc());
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #16
Source File: SortedNumericDVIndexFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

        NumericDocValues single = DocValues.unwrapSingleton(raw);
        if (single != null) {
            return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw));
        } else {
            return new MultiFloatValues(raw);
        }
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #17
Source File: Geo3DPointDistanceComparator.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
  LeafReader reader = context.reader();
  FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info != null) {
    Geo3DDocValuesField.checkCompatible(info);
  }
  currentDocs = DocValues.getSortedNumeric(reader, field);
  return this;
}
 
Example #18
Source File: SortedSetDVBytesAtomicFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public RandomAccessOrds getOrdinalsValues() {
    try {
        return FieldData.maybeSlowRandomAccessOrds(DocValues.getSortedSet(reader, field));
    } catch (IOException e) {
        throw new IllegalStateException("cannot load docvalues", e);
    }
}
 
Example #19
Source File: SortedNumericDVIndexFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDocValues getLongValues() {
    try {
        return DocValues.getSortedNumeric(reader, field);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #20
Source File: SortedNumericDVIndexFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
        return FieldData.sortableLongBitsToDoubles(raw);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #21
Source File: DistanceValueSource.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Returns the FunctionValues used by the function query.
 */
@Override
public DoubleValues getValues(LeafReaderContext readerContext, DoubleValues scores) throws IOException {
  LeafReader reader = readerContext.reader();

  final NumericDocValues ptX = DocValues.getNumeric(reader, strategy.getFieldNameX());
  final NumericDocValues ptY = DocValues.getNumeric(reader, strategy.getFieldNameY());

  return DoubleValues.withDefault(new DoubleValues() {

    private final Point from = DistanceValueSource.this.from;
    private final DistanceCalculator calculator = strategy.getSpatialContext().getDistCalc();

    @Override
    public double doubleValue() throws IOException {
      double x = Double.longBitsToDouble(ptX.longValue());
      double y = Double.longBitsToDouble(ptY.longValue());
      return calculator.distance(from, x, y) * multiplier;
    }

    @Override
    public boolean advanceExact(int doc) throws IOException {
      return ptX.advanceExact(doc) && ptY.advanceExact(doc);
    }

  }, nullValue);
}
 
Example #22
Source File: MultiOrdinals.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public RandomAccessOrds ordinals(ValuesHolder values) {
    if (multiValued) {
        return new MultiDocs(this, values);
    } else {
        return (RandomAccessOrds) DocValues.singleton(new SingleDocs(this, values));
    }
}
 
Example #23
Source File: BytesBinaryDVIndexFieldData.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public BytesBinaryDVAtomicFieldData load(LeafReaderContext context) {
    try {
        return new BytesBinaryDVAtomicFieldData(DocValues.getBinary(context.reader(), fieldNames.indexName()));
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example #24
Source File: DocTermsIndexDocValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static SortedDocValues open(LeafReaderContext context, String field) throws IOException {
  try {
    return DocValues.getSorted(context.reader(), field);
  } catch (RuntimeException e) {
    throw new DocTermsIndexException(field, e);
  }
}
 
Example #25
Source File: GlobalOrdinalsWithScoreCollector.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 
Example #26
Source File: GlobalOrdinalsCollector.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 
Example #27
Source File: IndexSortSortedNumericDocValuesRangeQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  Weight fallbackWeight = fallbackQuery.createWeight(searcher, scoreMode, boost);

  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      SortedNumericDocValues sortedNumericValues = DocValues.getSortedNumeric(context.reader(), field);
      NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);

      if (numericValues != null) {
        Sort indexSort = context.reader().getMetaData().getSort();
        if (indexSort != null
            && indexSort.getSort().length > 0
            && indexSort.getSort()[0].getField().equals(field)) {

          SortField sortField = indexSort.getSort()[0];
          DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);
          return new ConstantScoreScorer(this, score(), scoreMode, disi);
        }
      }
      return fallbackWeight.scorer(context);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      // Both queries should always return the same values, so we can just check
      // if the fallback query is cacheable.
      return fallbackWeight.isCacheable(ctx);
    }
  };
}
 
Example #28
Source File: SortedSetSortField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public FieldComparator<?> getComparator(int numHits, int sortPos) {
  return new FieldComparator.TermOrdValComparator(numHits, getField(), missingValue == STRING_LAST) {
    @Override
    protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
      return SortedSetSelector.wrap(DocValues.getSortedSet(context.reader(), field), selector);
    }
  };
}
 
Example #29
Source File: StringValue.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void setNextReader(LeafReaderContext context) throws IOException {
  if (ordinalMap != null) {
    toGlobal = ordinalMap.getGlobalOrds(context.ord);
  }
  docValues = DocValues.getSorted(context.reader(), field);
  lastDocID = 0;
}
 
Example #30
Source File: XYPointDistanceComparator.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafFieldComparator getLeafComparator(LeafReaderContext context) throws IOException {
  LeafReader reader = context.reader();
  FieldInfo info = reader.getFieldInfos().fieldInfo(field);
  if (info != null) {
    XYDocValuesField.checkCompatible(info);
  }
  currentDocs = DocValues.getSortedNumeric(reader, field);
  valuesDocID = -1;
  return this;
}