org.apache.lucene.util.NumericUtils Java Examples

The following examples show how to use org.apache.lucene.util.NumericUtils. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: QueryBuilderHelper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
private BytesRef valueForSearch(Object value) {
    if (value == null) return null;
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseValue(value), 0, bytesRef); // 0 because of exact match
    return bytesRef.get();
}
 
Example #2
Source File: IpFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public BytesRef indexedValueForSearch(Object value) {
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseValue(value), 0, bytesRef); // 0 because of exact match
    return bytesRef.get();
}
 
Example #3
Source File: ToParentBlockJoinSortField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private FieldComparator<?> getFloatComparator(int numHits) {
  return new FieldComparator.FloatComparator(numHits, getField(), (Float) missingValue) {
    @Override
    protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
      SortedNumericDocValues sortedNumeric = DocValues.getSortedNumeric(context.reader(), field);
      final BlockJoinSelector.Type type = order
          ? BlockJoinSelector.Type.MAX
          : BlockJoinSelector.Type.MIN;
      final BitSet parents = parentFilter.getBitSet(context);
      final BitSet children = childFilter.getBitSet(context);
      if (children == null) {
        return DocValues.emptyNumeric();
      }
      return new FilterNumericDocValues(BlockJoinSelector.wrap(sortedNumeric, type, parents, toIter(children))) {
        @Override
        public long longValue() throws IOException {
          // undo the numericutils sortability
          return NumericUtils.sortableFloatBits((int) super.longValue());
        }
      };
    }
  };
}
 
Example #4
Source File: SortedNumericSortField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public SortField readSortField(DataInput in) throws IOException {
  SortedNumericSortField sf = new SortedNumericSortField(in.readString(), readType(in), in.readInt() == 1, readSelectorType(in));
  if (in.readInt() == 1) {
    switch (sf.type) {
      case INT:
        sf.setMissingValue(in.readInt());
        break;
      case LONG:
        sf.setMissingValue(in.readLong());
        break;
      case FLOAT:
        sf.setMissingValue(NumericUtils.sortableIntToFloat(in.readInt()));
        break;
      case DOUBLE:
        sf.setMissingValue(NumericUtils.sortableLongToDouble(in.readLong()));
        break;
      default:
        throw new AssertionError();
    }
  }
  return sf;
}
 
Example #5
Source File: IndexSorter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public ComparableProvider[] getComparableProviders(List<? extends LeafReader> readers) throws IOException {
  ComparableProvider[] providers = new ComparableProvider[readers.size()];
  final float missingValue;
  if (this.missingValue != null) {
    missingValue = this.missingValue;
  } else {
    missingValue = 0.0f;
  }

  for(int readerIndex=0;readerIndex<readers.size();readerIndex++) {
    final NumericDocValues values = valuesProvider.get(readers.get(readerIndex));

    providers[readerIndex] = docID -> {
      float value = missingValue;
      if (values.advanceExact(docID)) {
        value = Float.intBitsToFloat((int) values.longValue());
      }
      return NumericUtils.floatToSortableInt(value);
    };
  }
  return providers;
}
 
Example #6
Source File: TestLegacyTerms.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testFloatFieldMinMax() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter w = new RandomIndexWriter(random(), dir);
  int numDocs = atLeast(100);
  float minValue = Float.POSITIVE_INFINITY;
  float maxValue = Float.NEGATIVE_INFINITY;
  for(int i=0;i<numDocs;i++ ){
    Document doc = new Document();
    float num = random().nextFloat();
    minValue = Math.min(num, minValue);
    maxValue = Math.max(num, maxValue);
    doc.add(new LegacyFloatField("field", num, Field.Store.NO));
    w.addDocument(doc);
  }
  
  IndexReader r = w.getReader();
  Terms terms = MultiTerms.getTerms(r, "field");
  assertEquals(minValue, NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMinInt(terms)), 0.0f);
  assertEquals(maxValue, NumericUtils.sortableIntToFloat(LegacyNumericUtils.getMaxInt(terms)), 0.0f);

  r.close();
  w.close();
  dir.close();
}
 
Example #7
Source File: NumericFieldType.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected Query getRangeQueryForMultiValuedFloatDocValues(SchemaField sf, String min, String max, boolean minInclusive, boolean maxInclusive) {
  float minVal,maxVal;
  if (min == null) {
    minVal = Float.NEGATIVE_INFINITY;
  } else {
    minVal = parseFloatFromUser(sf.getName(), min);
    if (!minInclusive) {
      if (minVal == Float.POSITIVE_INFINITY) return new MatchNoDocsQuery();
      minVal = FloatPoint.nextUp(minVal);
    }
  }
  if (max == null) {
    maxVal = Float.POSITIVE_INFINITY;
  } else {
    maxVal = parseFloatFromUser(sf.getName(), max);
    if (!maxInclusive) {
      if (maxVal == Float.NEGATIVE_INFINITY) return new MatchNoDocsQuery();
      maxVal = FloatPoint.nextDown(maxVal);
    }
  }
  Long minBits = (long)NumericUtils.floatToSortableInt(minVal);
  Long maxBits = (long)NumericUtils.floatToSortableInt(maxVal);
  return numericDocValuesRangeQuery(sf.getName(), minBits, maxBits, true, true, true);
}
 
Example #8
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testDoubles() throws Exception {
  double[] vals=new double[]{
    Double.NEGATIVE_INFINITY, -2.3E25, -1.0E15, -1.0, -1.0E-1, -1.0E-2, -0.0, 
    +0.0, 1.0E-2, 1.0E-1, 1.0, 1.0E15, 2.3E25, Double.POSITIVE_INFINITY, Double.NaN
  };
  long[] longVals=new long[vals.length];
  
  // check forward and back conversion
  for (int i=0; i<vals.length; i++) {
    longVals[i]= NumericUtils.doubleToSortableLong(vals[i]);
    assertTrue( "forward and back conversion should generate same double", Double.compare(vals[i], NumericUtils.sortableLongToDouble(longVals[i]))==0 );
  }
  
  // check sort order (prefixVals should be ascending)
  for (int i=1; i<longVals.length; i++) {
    assertTrue( "check sort order", longVals[i-1] < longVals[i] );
  }
}
 
Example #9
Source File: LongFieldMapper.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public BytesRef indexedValueForSearch(Object value) {
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(parseLongValue(value), 0, bytesRef);  // 0 because of exact match
    return bytesRef.get();
}
 
Example #10
Source File: MultiFieldWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
static LongFunction<Object> bitsToValue(FieldType fieldType) {
  switch (fieldType.getNumberType()) {
    case LONG:
      return (bits)-> bits;
    case DATE:
      return (bits)-> new Date(bits);
    case INTEGER:
      return (bits)-> (int)bits;
    case FLOAT:
      return (bits)-> NumericUtils.sortableIntToFloat((int)bits);
    case DOUBLE:
      return (bits)-> NumericUtils.sortableLongToDouble(bits);
    default:
      throw new AssertionError("Unsupported NumberType: " + fieldType.getNumberType());
  }
}
 
Example #11
Source File: BytesRefTermStream.java    From siren-join with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public BytesRef next() {
  BytesRefBuilder b = new BytesRefBuilder();
  NumericUtils.longToPrefixCoded((int) values.valueAt(this.count++), 0, b);
  return b.toBytesRef();
}
 
Example #12
Source File: TrieField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Object toObject(SchemaField sf, BytesRef term) {
  switch (type) {
    case INTEGER:
      return LegacyNumericUtils.prefixCodedToInt(term);
    case FLOAT:
      return NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(term));
    case LONG:
      return LegacyNumericUtils.prefixCodedToLong(term);
    case DOUBLE:
      return NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(term));
    case DATE:
      return new Date(LegacyNumericUtils.prefixCodedToLong(term));
    default:
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
  }
}
 
Example #13
Source File: TrieField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public String indexedToReadable(String _indexedForm) {
  final BytesRef indexedForm = new BytesRef(_indexedForm);
  switch (type) {
    case INTEGER:
      return Integer.toString( LegacyNumericUtils.prefixCodedToInt(indexedForm) );
    case FLOAT:
      return Float.toString( NumericUtils.sortableIntToFloat(LegacyNumericUtils.prefixCodedToInt(indexedForm)) );
    case LONG:
      return Long.toString( LegacyNumericUtils.prefixCodedToLong(indexedForm) );
    case DOUBLE:
      return Double.toString( NumericUtils.sortableLongToDouble(LegacyNumericUtils.prefixCodedToLong(indexedForm)) );
    case DATE:
      return Instant.ofEpochMilli(LegacyNumericUtils.prefixCodedToLong(indexedForm)).toString();
    default:
      throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown type for trie field: " + type);
  }
}
 
Example #14
Source File: FloatFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    float minValue = NumericUtils.sortableIntToFloat(NumericUtils.getMinInt(terms));
    float maxValue = NumericUtils.sortableIntToFloat(NumericUtils.getMaxInt(terms));
    return new FieldStats.Float(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 
Example #15
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testSortableFloatNaN() {
  final int plusInf = NumericUtils.floatToSortableInt(Float.POSITIVE_INFINITY);
  for (float nan : FLOAT_NANs) {
    assertTrue(Float.isNaN(nan));
    final int sortable = NumericUtils.floatToSortableInt(nan);
    assertTrue("Float not sorted correctly: " + nan + ", int repr: " 
        + sortable + ", positive inf.: " + plusInf, sortable > plusInf);
  }
}
 
Example #16
Source File: BKDWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Pick the next dimension to split.
 * @param minPackedValue the min values for all dimensions
 * @param maxPackedValue the max values for all dimensions
 * @param parentSplits how many times each dim has been split on the parent levels
 * @return the dimension to split
 */
protected int split(byte[] minPackedValue, byte[] maxPackedValue, int[] parentSplits) {
  // First look at whether there is a dimension that has split less than 2x less than
  // the dim that has most splits, and return it if there is such a dimension and it
  // does not only have equals values. This helps ensure all dimensions are indexed.
  int maxNumSplits = 0;
  for (int numSplits : parentSplits) {
    maxNumSplits = Math.max(maxNumSplits, numSplits);
  }
  for (int dim = 0; dim < numIndexDims; ++dim) {
    final int offset = dim * bytesPerDim;
    if (parentSplits[dim] < maxNumSplits / 2 &&
        Arrays.compareUnsigned(minPackedValue, offset, offset + bytesPerDim, maxPackedValue, offset, offset + bytesPerDim) != 0) {
      return dim;
    }
  }

  // Find which dim has the largest span so we can split on it:
  int splitDim = -1;
  for(int dim=0;dim<numIndexDims;dim++) {
    NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
    if (splitDim == -1 || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
      System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
      splitDim = dim;
    }
  }

  //System.out.println("SPLIT: " + splitDim);
  return splitDim;
}
 
Example #17
Source File: EnumFieldType.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void readableToIndexed(CharSequence val, BytesRefBuilder result) {
  final String s = val.toString();
  if (s == null)
    return;

  result.grow(Integer.BYTES);
  result.setLength(Integer.BYTES);
  final Integer intValue = enumMapping.stringValueToIntValue(s);
  NumericUtils.intToSortableBytes(intValue, result.bytes(), 0);
}
 
Example #18
Source File: LatLonBoundingBox.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** encodes a two-dimensional geopoint (lat, lon) into a byte array */
static void encode(double lat, double lon, byte[] result, int offset) {
  if (result == null) {
    result = new byte[BYTES*4];
  }
  NumericUtils.intToSortableBytes(encodeLatitude(lat), result, offset);
  NumericUtils.intToSortableBytes(encodeLongitude(lon), result, offset + BYTES);
}
 
Example #19
Source File: SumAggregation.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
public void apply(SumDoubleState state, int doc) throws IOException {
    if (values.advanceExact(doc) && values.docValueCount() == 1) {
        state.sum += NumericUtils.sortableLongToDouble(values.nextValue());
        state.hadValue = true;
    }
}
 
Example #20
Source File: FloatColumnReferenceTest.java    From crate with Apache License 2.0 5 votes vote down vote up
@Override
protected void insertValues(IndexWriter writer) throws Exception {
    for (float f = -0.5f; f < 10.0f; f++) {
        Document doc = new Document();
        doc.add(new SortedNumericDocValuesField(column, NumericUtils.floatToSortableInt(f)));
        writer.addDocument(doc);
    }
}
 
Example #21
Source File: DateFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinLong(terms);
    long maxValue = NumericUtils.getMaxLong(terms);
    return new FieldStats.Date(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue, dateTimeFormatter()
    );
}
 
Example #22
Source File: FloatFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public BytesRef indexedValueForSearch(Object value) {
    int intValue = NumericUtils.floatToSortableInt(parseValue(value));
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.intToPrefixCoded(intValue, 0, bytesRef);   // 0 because of exact match
    return bytesRef.get();
}
 
Example #23
Source File: InetAddressPoint.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Return the {@link InetAddress} that compares immediately greater than
 * {@code address}.
 * @throws ArithmeticException if the provided address is the
 *              {@link #MAX_VALUE maximum ip address}
 */
public static InetAddress nextUp(InetAddress address) {
  if (address.equals(MAX_VALUE)) {
    throw new ArithmeticException("Overflow: there is no greater InetAddress than "
        + address.getHostAddress());
  }
  byte[] delta = new byte[BYTES];
  delta[BYTES-1] = 1;
  byte[] nextUpBytes = new byte[InetAddressPoint.BYTES];
  NumericUtils.add(InetAddressPoint.BYTES, 0, encode(address), delta, nextUpBytes);
  return decode(nextUpBytes);
}
 
Example #24
Source File: DoubleFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public BytesRef indexedValueForSearch(Object value) {
    long longValue = NumericUtils.doubleToSortableLong(parseDoubleValue(value));
    BytesRefBuilder bytesRef = new BytesRefBuilder();
    NumericUtils.longToPrefixCoded(longValue, 0, bytesRef);   // 0 because of exact match
    return bytesRef.get();
}
 
Example #25
Source File: LongFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinLong(terms);
    long maxValue = NumericUtils.getMaxLong(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 
Example #26
Source File: SimpleTextBKDWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
protected int split(byte[] minPackedValue, byte[] maxPackedValue) {
  // Find which dim has the largest span so we can split on it:
  int splitDim = -1;
  for(int dim=0;dim<numIndexDims;dim++) {
    NumericUtils.subtract(bytesPerDim, dim, maxPackedValue, minPackedValue, scratchDiff);
    if (splitDim == -1 || Arrays.compareUnsigned(scratchDiff, 0, bytesPerDim, scratch1, 0, bytesPerDim) > 0) {
      System.arraycopy(scratchDiff, 0, scratch1, 0, bytesPerDim);
      splitDim = dim;
    }
  }

  //System.out.println("SPLIT: " + splitDim);
  return splitDim;
}
 
Example #27
Source File: IntegerFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinInt(terms);
    long maxValue = NumericUtils.getMaxInt(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 
Example #28
Source File: TestValueSources.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@BeforeClass
public static void beforeClass() throws Exception {
  dir = newDirectory();
  analyzer = new MockAnalyzer(random());
  IndexWriterConfig iwConfig = newIndexWriterConfig(analyzer);
  iwConfig.setMergePolicy(newLogMergePolicy());
  RandomIndexWriter iw = new RandomIndexWriter(random(), dir, iwConfig);
  for (String [] doc : documents) {
    Document document = new Document();
    document.add(new StringField("id", doc[0], Field.Store.NO));
    document.add(new SortedDocValuesField("id", new BytesRef(doc[0])));
    document.add(new NumericDocValuesField("double", Double.doubleToRawLongBits(Double.parseDouble(doc[1]))));
    document.add(new NumericDocValuesField("float", Float.floatToRawIntBits(Float.parseFloat(doc[2]))));
    document.add(new NumericDocValuesField("int", Integer.parseInt(doc[3])));
    document.add(new NumericDocValuesField("long", Long.parseLong(doc[4])));
    document.add(new StringField("string", doc[5], Field.Store.NO));
    document.add(new SortedDocValuesField("string", new BytesRef(doc[5])));
    document.add(new TextField("text", doc[6], Field.Store.NO));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[7]))));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[8]))));
    document.add(new SortedNumericDocValuesField("floatMv", NumericUtils.floatToSortableInt(Float.parseFloat(doc[9]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[7]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[8]))));
    document.add(new SortedNumericDocValuesField("doubleMv", NumericUtils.doubleToSortableLong(Double.parseDouble(doc[9]))));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[10])));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[11])));
    document.add(new SortedNumericDocValuesField("intMv", Long.parseLong(doc[12])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[10])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[11])));
    document.add(new SortedNumericDocValuesField("longMv", Long.parseLong(doc[12])));
    iw.addDocument(document);
  }
  
  reader = iw.getReader();
  searcher = newSearcher(reader);
  iw.close();
}
 
Example #29
Source File: ByteFieldMapper.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public FieldStats stats(Terms terms, int maxDoc) throws IOException {
    long minValue = NumericUtils.getMinInt(terms);
    long maxValue = NumericUtils.getMaxInt(terms);
    return new FieldStats.Long(
        maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue
    );
}
 
Example #30
Source File: SortedNumericStatsValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private Number toCorrectType(long value) {
  switch (numberType) {
    case INTEGER:
    case LONG:
      return value;
    case FLOAT:
      return NumericUtils.sortableIntToFloat((int)value);
    case DOUBLE:
      return NumericUtils.sortableLongToDouble(value);
    default:
      throw new AssertionError("Unsupported number type");
  }
}