Java Code Examples for org.apache.lucene.index.SortedNumericDocValues

The following examples show how to use org.apache.lucene.index.SortedNumericDocValues. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: Elasticsearch   Source File: CardinalityAggregator.java    License: Apache License 2.0 6 votes vote down vote up
/**
 * Return a {@link MurmurHash3Values} instance that returns each value as its hash.
 */
public static MurmurHash3Values cast(final SortedNumericDocValues values) {
    return new MurmurHash3Values() {
        @Override
        public void setDocument(int docId) {
            values.setDocument(docId);
        }
        @Override
        public int count() {
            return values.count();
        }
        @Override
        public long valueAt(int index) {
            return values.valueAt(index);
        }
    };
}
 
Example 2
Source Project: lucene-solr   Source File: DocValuesAdapter.java    License: Apache License 2.0 6 votes vote down vote up
private Optional<DocValues> createSortedNumericDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedNumericDocValues snvalues = IndexUtils.getSortedNumericDocValues(reader, field);

  if (snvalues.advanceExact(docid)) {
    List<Long> numericValues = new ArrayList<>();

    int dvCount = snvalues.docValueCount();
    for (int i = 0; i < dvCount; i++) {
      numericValues.add(snvalues.nextValue());
    }

    DocValues dv = DocValues.of(
        dvType,
        Collections.emptyList(),
        numericValues
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 
Example 3
Source Project: lucene-solr   Source File: TestPrefixCompletionQuery.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 
Example 4
Source Project: lucene-solr   Source File: LongValueFacetCounts.java    License: Apache License 2.0 6 votes vote down vote up
private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 
Example 5
Source Project: lucene-solr   Source File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getIntComparator(int numHits) {
  return new FieldComparator.IntComparator(numHits, getField(), (Integer) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 
Example 6
Source Project: lucene-solr   Source File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getLongComparator(int numHits) {
  return new FieldComparator.LongComparator(numHits, getField(), (Long) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children));
    }
  };
}
 
Example 7
Source Project: lucene-solr   Source File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 
Example 8
Source Project: lucene-solr   Source File: ToParentBlockJoinSortField.java    License: Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getDoubleComparator(int numHits) {
  return new FieldComparator.DoubleComparator(numHits, getField(), (Double) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableDoubleBits(super.longValue());
        }
      };
    }
  };
}
 
Example 9
@Override
public SortedNumericDocValues getLongValues() {
    try {
        return DocValues.getSortedNumeric(reader, field);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example 10
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

        NumericDocValues single = DocValues.unwrapSingleton(raw);
        if (single != null) {
            return FieldData.singleton(new SingleFloatValues(single), DocValues.unwrapSingletonBits(raw));
        } else {
            return new MultiFloatValues(raw);
        }
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example 11
@Override
public SortedNumericDoubleValues getDoubleValues() {
    try {
        SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);
        return FieldData.sortableLongBitsToDoubles(raw);
    } catch (IOException e) {
        throw new IllegalStateException("Cannot load doc values", e);
    }
}
 
Example 12
Source Project: Elasticsearch   Source File: AtomicLongFieldData.java    License: Apache License 2.0 5 votes vote down vote up
public static AtomicNumericFieldData empty(final int maxDoc) {
    return new AtomicLongFieldData(0) {

        @Override
        public SortedNumericDocValues getLongValues() {
            return DocValues.emptySortedNumeric(maxDoc);
        }

        @Override
        public Collection<Accountable> getChildResources() {
            return Collections.emptyList();
        }

    };
}
 
Example 13
Source Project: lucene-solr   Source File: TestMemoryIndex.java    License: Apache License 2.0 5 votes vote down vote up
public void testDocValues_resetIterator() throws Exception {
  Document doc = new Document();

  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("f")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("d")));
  doc.add(new SortedSetDocValuesField("sorted_set", new BytesRef("c")));

  doc.add(new SortedNumericDocValuesField("sorted_numeric", 33L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 32L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 31L));
  doc.add(new SortedNumericDocValuesField("sorted_numeric", 30L));

  MemoryIndex mi = MemoryIndex.fromDocument(doc, analyzer);
  LeafReader leafReader = mi.createSearcher().getIndexReader().leaves().get(0).reader();

  SortedSetDocValues sortedSetDocValues = leafReader.getSortedSetDocValues("sorted_set");
  assertEquals(3, sortedSetDocValues.getValueCount());
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedSetDocValues.advanceExact(0));
    assertEquals(0L, sortedSetDocValues.nextOrd());
    assertEquals(1L, sortedSetDocValues.nextOrd());
    assertEquals(2L, sortedSetDocValues.nextOrd());
    assertEquals(SortedSetDocValues.NO_MORE_ORDS, sortedSetDocValues.nextOrd());
  }

  SortedNumericDocValues sortedNumericDocValues = leafReader.getSortedNumericDocValues("sorted_numeric");
  for (int times = 0; times < 3; times++) {
    assertTrue(sortedNumericDocValues.advanceExact(0));
    assertEquals(5, sortedNumericDocValues.docValueCount());
    assertEquals(30L, sortedNumericDocValues.nextValue());
    assertEquals(31L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(32L, sortedNumericDocValues.nextValue());
    assertEquals(33L, sortedNumericDocValues.nextValue());
  }

}
 
Example 14
Source Project: lucene-solr   Source File: LongValueFacetCounts.java    License: Apache License 2.0 5 votes vote down vote up
/** Counts directly from SortedNumericDocValues. */
private void countMultiValued(String field, List<MatchingDocs> matchingDocs) throws IOException {

  for (MatchingDocs hits : matchingDocs) {
    SortedNumericDocValues values = hits.context.reader().getSortedNumericDocValues(field);
    if (values == null) {
      // this field has no doc values for this segment
      continue;
    }

    NumericDocValues singleValues = DocValues.unwrapSingleton(values);

    if (singleValues != null) {
      countOneSegment(singleValues, hits);
    } else {

      DocIdSetIterator it = ConjunctionDISI.intersectIterators(
                               Arrays.asList(hits.bits.iterator(), values));
    
      for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
        int limit = values.docValueCount();
        totCount += limit;
        for (int i = 0; i < limit; i++) {
          increment(values.nextValue());
        }
      }
    }
  }
}
 
Example 15
Source Project: lucene-solr   Source File: BlockJoinSelector.java    License: Apache License 2.0 5 votes vote down vote up
/** Wraps the provided {@link SortedNumericDocValues} in order to only select
 *  one value per parent among its {@code children} using the configured
 *  {@code selection} type. */
public static NumericDocValues wrap(SortedNumericDocValues sortedNumerics, Type selection, BitSet parents, DocIdSetIterator children) {
  NumericDocValues values;
  switch (selection) {
    case MIN:
      values = SortedNumericSelector.wrap(sortedNumerics, SortedNumericSelector.Type.MIN, SortField.Type.LONG);
      break;
    case MAX:
      values = SortedNumericSelector.wrap(sortedNumerics, SortedNumericSelector.Type.MAX, SortField.Type.LONG);
      break;
    default:
      throw new AssertionError();
  }
  return wrap(values, selection, parents, children);
}
 
Example 16
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  Weight fallbackWeight = fallbackQuery.createWeight(searcher, scoreMode, boost);

  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      SortedNumericDocValues sortedNumericValues = DocValues.getSortedNumeric(context.reader(), field);
      NumericDocValues numericValues = DocValues.unwrapSingleton(sortedNumericValues);

      if (numericValues != null) {
        Sort indexSort = context.reader().getMetaData().getSort();
        if (indexSort != null
            && indexSort.getSort().length > 0
            && indexSort.getSort()[0].getField().equals(field)) {

          SortField sortField = indexSort.getSort()[0];
          DocIdSetIterator disi = getDocIdSetIterator(sortField, context, numericValues);
          return new ConstantScoreScorer(this, score(), scoreMode, disi);
        }
      }
      return fallbackWeight.scorer(context);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      // Both queries should always return the same values, so we can just check
      // if the fallback query is cacheable.
      return fallbackWeight.isCacheable(ctx);
    }
  };
}
 
Example 17
Source Project: lucene-solr   Source File: DocValuesNumbersQuery.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
      return new ConstantScoreScorer(this, score(), scoreMode, new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          int count = values.docValueCount();
          for(int i=0;i<count;i++) {
            if (numbers.contains(values.nextValue())) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // lookup in the set
        }
      });
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }

  };
}
 
Example 18
Source Project: lucene-solr   Source File: AssertingDocValuesFormat.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOException {
  if (merging) {
    AssertingCodec.assertThread("DocValuesProducer", creationThread);
  }
  assert field.getDocValuesType() == DocValuesType.SORTED_NUMERIC;
  SortedNumericDocValues values = in.getSortedNumeric(field);
  assert values != null;
  return new AssertingLeafReader.AssertingSortedNumericDocValues(values, maxDoc);
}
 
Example 19
Source Project: lucene-solr   Source File: Lucene80DocValuesConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue,
    long min, long gcd, Map<Long, Integer> encode) throws IOException {
  DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue);
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      long v = values.nextValue();
      if (encode == null) {
        writer.add((v - min) / gcd);
      } else {
        writer.add(encode.get(v));
      }
    }
  }
  writer.finish();
}
 
Example 20
Source Project: lucene-solr   Source File: Lucene80DocValuesConsumer.java    License: Apache License 2.0 5 votes vote down vote up
private long writeValuesMultipleBlocks(SortedNumericDocValues values, long gcd) throws IOException {
  long[] offsets = new long[ArrayUtil.oversize(1, Long.BYTES)];
  int offsetsIndex = 0;
  final long[] buffer = new long[NUMERIC_BLOCK_SIZE];
  final ByteBuffersDataOutput encodeBuffer = ByteBuffersDataOutput.newResettableInstance();
  int upTo = 0;
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      buffer[upTo++] = values.nextValue();
      if (upTo == NUMERIC_BLOCK_SIZE) {
        offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
        offsets[offsetsIndex++] = data.getFilePointer();
        writeBlock(buffer, NUMERIC_BLOCK_SIZE, gcd, encodeBuffer);
        upTo = 0;
      }
    }
  }
  if (upTo > 0) {
    offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
    offsets[offsetsIndex++] = data.getFilePointer();
    writeBlock(buffer, upTo, gcd, encodeBuffer);
  }

  // All blocks has been written. Flush the offset jump-table
  final long offsetsOrigo = data.getFilePointer();
  for (int i = 0 ; i < offsetsIndex ; i++) {
    data.writeLong(offsets[i]);
  }
  data.writeLong(offsetsOrigo);
  return offsetsOrigo;
}
 
Example 21
Source Project: lucene-solr   Source File: IntervalFacets.java    License: Apache License 2.0 5 votes vote down vote up
private void accumIntervalWithMultipleValues(SortedNumericDocValues longs) throws IOException {
  // longs should be already positioned to the correct doc
  assert longs.docID() != -1;
  final int docValueCount = longs.docValueCount();
  assert docValueCount > 0: "Should have at least one value for this document";
  int currentInterval = 0;
  for (int i = 0; i < docValueCount; i++) {
    boolean evaluateNextInterval = true;
    long value = longs.nextValue();
    while (evaluateNextInterval && currentInterval < intervals.length) {
      IntervalCompareResult result = intervals[currentInterval].includes(value);
      switch (result) {
        case INCLUDED:
          /*
           * Increment the current interval and move to the next one using
           * the same value
           */
          intervals[currentInterval].incCount();
          currentInterval++;
          break;
        case LOWER_THAN_START:
          /*
           * None of the next intervals will match this value (all of them have 
           * higher start value). Move to the next value for this document. 
           */
          evaluateNextInterval = false;
          break;
        case GREATER_THAN_END:
          /*
           * Next interval may match this value
           */
          currentInterval++;
          break;
      }
      //Maybe return if currentInterval == intervals.length?
    }
   }
}
 
Example 22
@Override
public SortedNumericDocValues getSortedNumericDocValues(final String field) throws IOException {
    return isFls(field) ? in.getSortedNumericDocValues(field) : null;
}
 
Example 23
Source Project: Elasticsearch   Source File: CardinalityAggregator.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * Return a {@link MurmurHash3Values} instance that computes hashes on the fly for each long value.
 */
public static MurmurHash3Values hash(SortedNumericDocValues values) {
    return new Long(values);
}
 
Example 24
Source Project: Elasticsearch   Source File: CardinalityAggregator.java    License: Apache License 2.0 4 votes vote down vote up
public Long(SortedNumericDocValues values) {
    this.values = values;
}
 
Example 25
Source Project: Elasticsearch   Source File: GeoDistanceParser.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public SortedNumericDocValues longValues(LeafReaderContext ctx) {
    throw new UnsupportedOperationException();
}
 
Example 26
Source Project: Elasticsearch   Source File: LongTermsAggregator.java    License: Apache License 2.0 4 votes vote down vote up
protected SortedNumericDocValues getValues(ValuesSource.Numeric valuesSource, LeafReaderContext ctx) throws IOException {
    return valuesSource.longValues(ctx);
}
 
Example 27
Source Project: Elasticsearch   Source File: DoubleTermsAggregator.java    License: Apache License 2.0 4 votes vote down vote up
@Override
protected SortedNumericDocValues getValues(Numeric valuesSource, LeafReaderContext ctx) throws IOException {
    return FieldData.toSortableLongBits(valuesSource.doubleValues(ctx));
}
 
Example 28
Source Project: Elasticsearch   Source File: GeoHashGridParser.java    License: Apache License 2.0 4 votes vote down vote up
@Override
public SortedNumericDocValues longValues(LeafReaderContext ctx) {
    return new CellValues(valuesSource.geoPointValues(ctx), precision);
}
 
Example 29
Source Project: Elasticsearch   Source File: ScriptDocValues.java    License: Apache License 2.0 4 votes vote down vote up
public Longs(SortedNumericDocValues values) {
    this.values = values;
}
 
Example 30
Source Project: Elasticsearch   Source File: ScriptDocValues.java    License: Apache License 2.0 4 votes vote down vote up
public SortedNumericDocValues getInternalValues() {
    return this.values;
}