Java Code Examples for org.apache.lucene.index.SortedNumericDocValues#docValueCount()

@Override
public Bits getBits(final LeafReaderContext context) throws IOException {
  final int maxDoc = context.reader().maxDoc();
  FixedBitSet bits = new FixedBitSet(maxDoc);
  final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
  int docID;
  while ((docID = values.nextDoc()) != NO_MORE_DOCS) {
    final int count = values.docValueCount();
    for (int i = 0; i < count; ++i) {
      final long v = values.nextValue();
      if (v >= min && v <= max) {
        bits.set(docID);
        break;
      }
    }
  }
  return bits;
}
 

private void countAllMultiValued(IndexReader reader, String field) throws IOException {

    for (LeafReaderContext context : reader.leaves()) {

      SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        // this field has no doc values for this segment
        continue;
      }
      NumericDocValues singleValues = DocValues.unwrapSingleton(values);
      if (singleValues != null) {
        countAllOneSegment(singleValues);
      } else {
        int doc;
        while ((doc = values.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
          int limit = values.docValueCount();
          totCount += limit;
          for (int i = 0; i < limit; i++) {
            increment(values.nextValue());
          }
        }
      }
    }
  }
 

private Optional<DocValues> createSortedNumericDocValues(int docid, String field, DocValuesType dvType)
    throws IOException {
  SortedNumericDocValues snvalues = IndexUtils.getSortedNumericDocValues(reader, field);

  if (snvalues.advanceExact(docid)) {
    List<Long> numericValues = new ArrayList<>();

    int dvCount = snvalues.docValueCount();
    for (int i = 0; i < dvCount; i++) {
      numericValues.add(snvalues.nextValue());
    }

    DocValues dv = DocValues.of(
        dvType,
        Collections.emptyList(),
        numericValues
    );
    return Optional.of(dv);
  }

  return Optional.empty();
}
 

/** Counts directly from SortedNumericDocValues. */
private void countMultiValued(String field, List<MatchingDocs> matchingDocs) throws IOException {

  for (MatchingDocs hits : matchingDocs) {
    SortedNumericDocValues values = hits.context.reader().getSortedNumericDocValues(field);
    if (values == null) {
      // this field has no doc values for this segment
      continue;
    }

    NumericDocValues singleValues = DocValues.unwrapSingleton(values);

    if (singleValues != null) {
      countOneSegment(singleValues, hits);
    } else {

      DocIdSetIterator it = ConjunctionDISI.intersectIterators(
                               Arrays.asList(hits.bits.iterator(), values));
    
      for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
        int limit = values.docValueCount();
        totCount += limit;
        for (int i = 0; i < limit; i++) {
          increment(values.nextValue());
        }
      }
    }
  }
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = DocValues.getSortedNumeric(context.reader(), field);
      return new ConstantScoreScorer(this, score(), scoreMode, new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          int count = values.docValueCount();
          for(int i=0;i<count;i++) {
            if (numbers.contains(values.nextValue())) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // lookup in the set
        }
      });
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }

  };
}
 

private void writeValuesSingleBlock(SortedNumericDocValues values, long numValues, int numBitsPerValue,
    long min, long gcd, Map<Long, Integer> encode) throws IOException {
  DirectWriter writer = DirectWriter.getInstance(data, numValues, numBitsPerValue);
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      long v = values.nextValue();
      if (encode == null) {
        writer.add((v - min) / gcd);
      } else {
        writer.add(encode.get(v));
      }
    }
  }
  writer.finish();
}
 

private long writeValuesMultipleBlocks(SortedNumericDocValues values, long gcd) throws IOException {
  long[] offsets = new long[ArrayUtil.oversize(1, Long.BYTES)];
  int offsetsIndex = 0;
  final long[] buffer = new long[NUMERIC_BLOCK_SIZE];
  final ByteBuffersDataOutput encodeBuffer = ByteBuffersDataOutput.newResettableInstance();
  int upTo = 0;
  for (int doc = values.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = values.nextDoc()) {
    for (int i = 0, count = values.docValueCount(); i < count; ++i) {
      buffer[upTo++] = values.nextValue();
      if (upTo == NUMERIC_BLOCK_SIZE) {
        offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
        offsets[offsetsIndex++] = data.getFilePointer();
        writeBlock(buffer, NUMERIC_BLOCK_SIZE, gcd, encodeBuffer);
        upTo = 0;
      }
    }
  }
  if (upTo > 0) {
    offsets = ArrayUtil.grow(offsets, offsetsIndex+1);
    offsets[offsetsIndex++] = data.getFilePointer();
    writeBlock(buffer, upTo, gcd, encodeBuffer);
  }

  // All blocks has been written. Flush the offset jump-table
  final long offsetsOrigo = data.getFilePointer();
  for (int i = 0 ; i < offsetsIndex ; i++) {
    data.writeLong(offsets[i]);
  }
  data.writeLong(offsetsOrigo);
  return offsetsOrigo;
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  return new ConstantScoreWeight(this, boost) {

    final Component2D tree = LatLonGeometry.create(polygons);
    final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createComponentPredicate(tree);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (polygonPredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 1000f; // TODO: what should it be?
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    private final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            final int lon = (int) (value & 0xFFFFFFFF);
            if (distancePredicate.test(lat, lon)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 100f; // TODO: what should it be?
        }

      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  return new ConstantScoreWeight(this, boost) {

    final Component2D component2D = XYGeometry.create(geometries);

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final double x = XYEncodingUtils.decode((int) (value >>> 32));
            final double y = XYEncodingUtils.decode((int) (value & 0xFFFFFFFF));
            if (component2D.contains(x, y)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 1000f; // TODO: what should it be?
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }
  };
}
 

@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedNumericDocValues values = context.reader().getSortedNumericDocValues(field);
      if (values == null) {
        return null;
      }

      final TwoPhaseIterator iterator = new TwoPhaseIterator(values) {
        @Override
        public boolean matches() throws IOException {
          for (int i = 0, count = values.docValueCount(); i < count; ++i) {
            final long value = values.nextValue();
            final int lat = (int) (value >>> 32);
            if (lat < minLatitude || lat > maxLatitude) {
              // not within latitude range
              continue;
            }

            final int lon = (int) (value & 0xFFFFFFFF);
            if (crossesDateline) {
              if (lon > maxLongitude && lon < minLongitude) {
                // not within longitude range
                continue;
              }
            } else {
              if (lon < minLongitude || lon > maxLongitude) {
                // not within longitude range
                continue;
              }
            }

            return true;
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 5; // 5 comparisons
        }
      };
      return new ConstantScoreScorer(this, boost, scoreMode, iterator);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 

private void doTestSortedNumericBlocksOfVariousBitsPerValue(LongSupplier counts) throws Exception {
  Directory dir = newDirectory();
  IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
  conf.setMaxBufferedDocs(atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE));
  conf.setRAMBufferSizeMB(-1);
  conf.setMergePolicy(newLogMergePolicy(random().nextBoolean()));
  IndexWriter writer = new IndexWriter(dir, conf);
  
  final int numDocs = atLeast(Lucene80DocValuesFormat.NUMERIC_BLOCK_SIZE*3);
  final LongSupplier values = blocksOfVariousBPV();
  for (int i = 0; i < numDocs; i++) {
    Document doc = new Document();
    
    int valueCount = (int) counts.getAsLong();
    long valueArray[] = new long[valueCount];
    for (int j = 0; j < valueCount; j++) {
      long value = values.getAsLong();
      valueArray[j] = value;
      doc.add(new SortedNumericDocValuesField("dv", value));
    }
    Arrays.sort(valueArray);
    for (int j = 0; j < valueCount; j++) {
      doc.add(new StoredField("stored", Long.toString(valueArray[j])));
    }
    writer.addDocument(doc);
    if (random().nextInt(31) == 0) {
      writer.commit();
    }
  }
  writer.forceMerge(1);

  writer.close();
  
  // compare
  DirectoryReader ir = DirectoryReader.open(dir);
  TestUtil.checkReader(ir);
  for (LeafReaderContext context : ir.leaves()) {
    LeafReader r = context.reader();
    SortedNumericDocValues docValues = DocValues.getSortedNumeric(r, "dv");
    for (int i = 0; i < r.maxDoc(); i++) {
      if (i > docValues.docID()) {
        docValues.nextDoc();
      }
      String expected[] = r.document(i).getValues("stored");
      if (i < docValues.docID()) {
        assertEquals(0, expected.length);
      } else {
        String actual[] = new String[docValues.docValueCount()];
        for (int j = 0; j < actual.length; j++) {
          actual[j] = Long.toString(docValues.nextValue());
        }
        assertArrayEquals(expected, actual);
      }
    }
  }
  ir.close();
  dir.close();
}