Java Code Examples for org.apache.lucene.index.SortedNumericDocValues#docValueCount()

The following examples show how to use org.apache.lucene.index.SortedNumericDocValues#docValueCount() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: lucene-solr   File: DocValuesAdapter.java    License: Apache License 2.0 6 votes vote down vote up
private Optional<DocValues> createSortedNumericDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedNumericDocValues snvalues = IndexUtils.getSortedNumericDocValues(reader, field);

  if (snvalues.advanceExact(docid)) {
    List<Long> numericValues = new ArrayList<>();

    int dvCount = snvalues.docValueCount();
    for (int i = 0; i < dvCount; i++) {
      numericValues.add(snvalues.nextValue());
    }

    DocValues dv = DocValues.of(
        dvType,
        Collections.emptyList(),
        numericValues
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 
Example 2
@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 
Example 3
private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 
Example 4
/** Counts directly from SortedNumericDocValues. */
private void countMultiValued(String field, List<MatchingDocs> matchingDocs) throws IOException {

  for (MatchingDocs hits : matchingDocs) {
    SortedNumericDocValues values = hits.context.reader().getSortedNumericDocValues(field);
    if (values == null) {
      // this field has no doc values for this segment
      continue;
    }

    NumericDocValues singleValues = DocValues.unwrapSingleton(values);

    if (singleValues != null) {
      countOneSegment(singleValues, hits);
    } else {

      DocIdSetIterator it = ConjunctionDISI.intersectIterators(
                               Arrays.asList(hits.bits.iterator(), values));
    
      for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
        int limit = values.docValueCount();
        totCount += limit;
        for (int i = 0; i < limit; i++) {
          increment(values.nextValue());
        }
      }
    }
  }
}
 
Example 5
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
      return new ConstantScoreScorer(this, score(), scoreMode, new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          int count = values.docValueCount();
          for(int i=0;i<count;i++) {
            if (numbers.contains(values.nextValue())) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // lookup in the set
        }
      });
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }

  };
}
 
Example 6
private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue,
    long min, long gcd, Map<Long, Integer> encode) throws IOException {
  DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue);
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      long v = values.nextValue();
      if (encode == null) {
        writer.add((v - min) / gcd);
      } else {
        writer.add(encode.get(v));
      }
    }
  }
  writer.finish();
}
 
Example 7
private long writeValuesMultipleBlocks(SortedNumericDocValues values, long gcd) throws IOException {
  long[] offsets = new long[ArrayUtil.oversize(1, Long.BYTES)];
  int offsetsIndex = 0;
  final long[] buffer = new long[NUMERIC_BLOCK_SIZE];
  final ByteBuffersDataOutput encodeBuffer = ByteBuffersDataOutput.newResettableInstance();
  int upTo = 0;
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      buffer[upTo++] = values.nextValue();
      if (upTo == NUMERIC_BLOCK_SIZE) {
        offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
        offsets[offsetsIndex++] = data.getFilePointer();
        writeBlock(buffer, NUMERIC_BLOCK_SIZE, gcd, encodeBuffer);
        upTo = 0;
      }
    }
  }
  if (upTo > 0) {
    offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
    offsets[offsetsIndex++] = data.getFilePointer();
    writeBlock(buffer, upTo, gcd, encodeBuffer);
  }

  // All blocks has been written. Flush the offset jump-table
  final long offsetsOrigo = data.getFilePointer();
  for (int i = 0 ; i < offsetsIndex ; i++) {
    data.writeLong(offsets[i]);
  }
  data.writeLong(offsetsOrigo);
  return offsetsOrigo;
}
 
Example 8
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  return new ConstantScoreWeight(this, boost) {

    final Component2D tree = LatLonGeometry.create(polygons);
    final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createComponentPredicate(tree);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (polygonPredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 1000f; // TODO: what should it be?
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 
Example 9
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (distancePredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 100f; // TODO: what should it be?
        }

      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 
Example 10
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  return new ConstantScoreWeight(this, boost) {

    final Component2D component2D = XYGeometry.create(geometries);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final double x = XYEncodingUtils.decode((int) (value >>> 32));
            final double y = XYEncodingUtils.decode((int) (value & 0xFFFFFFFF));
            if (component2D.contains(x, y)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 1000f; // TODO: what should it be?
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }
  };
}
 
Example 11
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {
        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            if (lat < minLatitude || lat > maxLatitude) {
              // not within latitude range
              continue;
            }

            final int lon = (int) (value & 0xFFFFFFFF);
            if (crossesDateline) {
              if (lon > maxLongitude && lon < minLongitude) {
                // not within longitude range
                continue;
              }
            } else {
              if (lon < minLongitude || lon > maxLongitude) {
                // not within longitude range
                continue;
              }
            }

            return true;
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // 5 comparisons
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 
Example 12
private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));
  conf.setRAMBufferSizeMB(-1);
  conf.setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE*3);
  final LongSupplier values = blocksOfVariousBPV();
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    
    int valueCount = (int) counts.getAsLong();
    long valueArray[] = new long[valueCount];
    for (int j = 0; j < valueCount; j++) {
      long value = values.getAsLong();
      valueArray[j] = value;
      doc.add(new SortedNumericDocValuesField("dv", value));
    }
    Arrays.sort(valueArray);
    for (int j = 0; j < valueCount; j++) {
      doc.add(new StoredField("stored", Long.toString(valueArray[j])));
    }
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  writer.forceMerge(1);

  writer.close();
  
  // compare
  DirectoryReader ir = DirectoryReader.open(dir);
  TestUtil.checkReader(ir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
    for (int i = 0; i < r.maxDoc(); i++) {
      if (i > docValues.docID()) {
        docValues.nextDoc();
      }
      String expected[] = r.document(i).getValues("stored");
      if (i < docValues.docID()) {
        assertEquals(0, expected.length);
      } else {
        String actual[] = new String[docValues.docValueCount()];
        for (int j = 0; j < actual.length; j++) {
          actual[j] = Long.toString(docValues.nextValue());
        }
        assertArrayEquals(expected, actual);
      }
    }
  }
  ir.close();
  dir.close();
}