org.apache.lucene.util.BitSetIterator Java Examples

The following examples show how to use org.apache.lucene.util.BitSetIterator. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void assertAdvanceEquality(IndexedDISI disi, BitSetIterator disi2, int step) throws IOException {
  int index = -1;
  while (true) {
    int target = disi2.docID() + step;
    int doc;
    do {
      doc = disi2.nextDoc();
      index++;
    } while (doc < target);
    assertEquals(doc, disi.advance(target));
    if (doc == DocIdSetIterator.NO_MORE_DOCS) {
      break;
    }
    assertEquals("Expected equality using step " + step + " at docID " + doc, index, disi.index());
  }
}
 
Example #2
Source File: TaggerRequestHandler.java    From SolrTextTagger with Apache License 2.0 6 votes vote down vote up
private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException {
  //Now we must supply a Solr DocList and add it to the response.
  //  Typically this is gotten via a SolrIndexSearcher.search(), but in this case we
  //  know exactly what documents to return, the order doesn't matter nor does
  //  scoring.
  //  Ideally an implementation of DocList could be directly implemented off
  //  of a BitSet, but there are way too many methods to implement for a minor
  //  payoff.
  int matchDocs = matchDocIdsBS.cardinality();
  int[] docIds = new int[ Math.min(rows, matchDocs) ];
  DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1);
  for (int i = 0; i < docIds.length; i++) {
    docIds[i] = docIdIter.nextDoc();
  }
  return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f);
}
 
Example #3
Source File: ExportWriter.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected void identifyLowestSortingUnexportedDocs(List<LeafReaderContext> leaves, SortDoc sortDoc, SortQueue queue) throws IOException {
  queue.reset();
  SortDoc top = queue.top();
  for (int i = 0; i < leaves.size(); i++) {
    sortDoc.setNextReader(leaves.get(i));
    DocIdSetIterator it = new BitSetIterator(sets[i], 0); // cost is not useful here
    int docId;
    while ((docId = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
      sortDoc.setValues(docId);
      if (top.lessThan(sortDoc)) {
        top.setValues(sortDoc);
        top = queue.updateTop();
      }
    }
  }
}
 
Example #4
Source File: ExpandComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 
Example #5
Source File: TaggerRequestHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException {
  //Now we must supply a Solr DocList and add it to the response.
  //  Typically this is gotten via a SolrIndexSearcher.search(), but in this case we
  //  know exactly what documents to return, the order doesn't matter nor does
  //  scoring.
  //  Ideally an implementation of DocList could be directly implemented off
  //  of a BitSet, but there are way too many methods to implement for a minor
  //  payoff.
  int matchDocs = matchDocIdsBS.cardinality();
  int[] docIds = new int[ Math.min(rows, matchDocs) ];
  DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1);
  for (int i = 0; i < docIds.length; i++) {
    docIds[i] = docIdIter.nextDoc();
  }
  return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f, TotalHits.Relation.EQUAL_TO);
}
 
Example #6
Source File: BlockJoinParentQParser.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, org.apache.lucene.search.ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(BitSetProducerQuery.this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      BitSet bitSet = bitSetProducer.getBitSet(context);
      if (bitSet == null) {
        return null;
      }
      DocIdSetIterator disi = new BitSetIterator(bitSet, bitSet.approximateCardinality());
      return new ConstantScoreScorer(this, boost, scoreMode, disi);
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return getCache();
    }
  };
}
 
Example #7
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void assertAdvanceExactRandomized(IndexedDISI disi, BitSetIterator disi2, int disi2length, int step)
    throws IOException {
  int index = -1;
  Random random = random();
  for (int target = 0; target < disi2length; ) {
    target += TestUtil.nextInt(random, 0, step);
    int doc = disi2.docID();
    while (doc < target) {
      doc = disi2.nextDoc();
      index++;
    }

    boolean exists = disi.advanceExact(target);
    assertEquals(doc == target, exists);
    if (exists) {
      assertEquals(index, disi.index());
    } else if (random.nextBoolean()) {
      assertEquals(doc, disi.nextDoc());
      // This is a bit strange when doc == NO_MORE_DOCS as the index overcounts in the disi2 while-loop
      assertEquals(index, disi.index());
      target = doc;
    }
  }
}
 
Example #8
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testOneDocMissingFixed() throws IOException {
  int maxDoc = 9699;
  final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
  FixedBitSet set = new FixedBitSet(maxDoc);
  set.set(0, maxDoc);
  set.clear(1345);
  try (Directory dir = newDirectory()) {

    final int cardinality = set.cardinality();
    long length;
    int jumpTableentryCount;
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
      length = out.getFilePointer();
    }

    int step = 16000;
    try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
      IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
      BitSetIterator disi2 = new BitSetIterator(set, cardinality);
      assertAdvanceEquality(disi, disi2, step);
    }
  }
}
 
Example #9
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testPositionNotZero() throws IOException {
  final int BLOCKS = 10;
  final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable

  BitSet set = createSetWithRandomBlocks(BLOCKS);
  try (Directory dir = newDirectory()) {
    final int cardinality = set.cardinality();
    int jumpTableEntryCount;
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      jumpTableEntryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
    }
    try (IndexInput fullInput = dir.openInput("foo", IOContext.DEFAULT)) {
      IndexInput blockData =
          IndexedDISI.createBlockSlice(fullInput, "blocks", 0, fullInput.length(), jumpTableEntryCount);
      blockData.seek(random().nextInt((int) blockData.length()));

      RandomAccessInput jumpTable = IndexedDISI.createJumpTable(fullInput, 0, fullInput.length(), jumpTableEntryCount);
      IndexedDISI disi = new IndexedDISI(blockData, jumpTable, jumpTableEntryCount, denseRankPower, cardinality);
      // This failed at some point during LUCENE-8585 development as it did not reset the slice position
      disi.advanceExact(BLOCKS*65536-1);
    }
  }
}
 
Example #10
Source File: TestTermAutomatonQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      int maxDoc = context.reader().maxDoc();
      FixedBitSet bits = new FixedBitSet(maxDoc);
      Random random = new Random(seed ^ context.docBase);
      for(int docID=0;docID<maxDoc;docID++) {
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
        }
      }
      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #11
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private void assertAdvanceBeyondEnd(BitSet set, Directory dir) throws IOException {
  final int cardinality = set.cardinality();
  final byte denseRankPower = 9; // Not tested here so fixed to isolate factors
  long length;
  int jumpTableentryCount;
  try (IndexOutput out = dir.createOutput("bar", IOContext.DEFAULT)) {
    jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
  }

  try (IndexInput in = dir.openInput("bar", IOContext.DEFAULT)) {
    BitSetIterator disi2 = new BitSetIterator(set, cardinality);
    int doc = disi2.docID();
    int index = 0;
    while (doc < cardinality) {
      doc = disi2.nextDoc();
      index++;
    }

    IndexedDISI disi = new IndexedDISI(in, 0L, in.length(), jumpTableentryCount, denseRankPower, cardinality);
    // Advance 1 docID beyond end
    assertFalse("There should be no set bit beyond the valid docID range", disi.advanceExact(set.length()));
    disi.advance(doc); // Should be the special docID signifyin NO_MORE_DOCS from the BitSetIterator
    assertEquals("The index when advancing beyond the last defined docID should be correct",
        index, disi.index()+1); // disi.index()+1 as the while-loop also counts the NO_MORE_DOCS
  }
}
 
Example #12
Source File: ConjunctionDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private int doNext(int doc) throws IOException {
  advanceLead: for (;; doc = lead.nextDoc()) {
    if (doc >= minLength) {
      return NO_MORE_DOCS;
    }
    for (BitSet bitSet : bitSets) {
      if (bitSet.get(doc) == false) {
        continue advanceLead;
      }
    }
    for (BitSetIterator iterator : bitSetIterators) {
      iterator.setDocId(doc);
    }
    return doc;
  }
}
 
Example #13
Source File: SolrIndexSplitter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      RTimerTree t = timings.sub("findDocsToDelete");
      t.resume();
      FixedBitSet set = findDocsToDelete(context);
      t.pause();
      if (log.isInfoEnabled()) {
        log.info("### partition={}, leaf={}, maxDoc={}, numDels={}, setLen={}, setCard={}"
        , partition, context, context.reader().maxDoc()
        ,context.reader().numDeletedDocs(), set.length(), set.cardinality());
      }
      Bits liveDocs = context.reader().getLiveDocs();
      if (liveDocs != null) {
        // check that we don't delete already deleted docs
        FixedBitSet dels = FixedBitSet.copyOf(liveDocs);
        dels.flip(0, dels.length());
        dels.and(set);
        if (dels.cardinality() > 0) {
          log.error("### INVALID DELS {}", dels.cardinality());
        }
      }
      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(set, set.length()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }

    @Override
    public String toString() {
      return "weight(shardSplittingQuery,part" + partition + ")";
    }
  };
}
 
Example #14
Source File: TermsIncludingScoreQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
SVInOrderScorer(Weight weight, TermsEnum termsEnum, int maxDoc, long cost) throws IOException {
  super(weight);
  FixedBitSet matchingDocs = new FixedBitSet(maxDoc);
  this.scores = new float[maxDoc];
  fillDocsAndScores(matchingDocs, termsEnum);
  this.matchingDocsIterator = new BitSetIterator(matchingDocs, cost);
  this.cost = cost;
}
 
Example #15
Source File: ConjunctionDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static DocIdSetIterator createConjunction(
    List<DocIdSetIterator> allIterators,
    List<TwoPhaseIterator> twoPhaseIterators) {
  long minCost = allIterators.stream().mapToLong(DocIdSetIterator::cost).min().getAsLong();
  List<BitSetIterator> bitSetIterators = new ArrayList<>();
  List<DocIdSetIterator> iterators = new ArrayList<>();
  for (DocIdSetIterator iterator : allIterators) {
    if (iterator.cost() > minCost && iterator instanceof BitSetIterator) {
      // we put all bitset iterators into bitSetIterators
      // except if they have the minimum cost, since we need
      // them to lead the iteration in that case
      bitSetIterators.add((BitSetIterator) iterator);
    } else {
      iterators.add(iterator);
    }
  }

  DocIdSetIterator disi;
  if (iterators.size() == 1) {
    disi = iterators.get(0);
  } else {
    disi = new ConjunctionDISI(iterators);
  }

  if (bitSetIterators.size() > 0) {
    disi = new BitSetConjunctionDISI(disi, bitSetIterators);
  }

  if (twoPhaseIterators.isEmpty() == false) {
    disi = TwoPhaseIterator.asDocIdSetIterator(new ConjunctionTwoPhaseIterator(disi, twoPhaseIterators));
  }

  return disi;
}
 
Example #16
Source File: ConjunctionDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
BitSetConjunctionDISI(DocIdSetIterator lead, Collection<BitSetIterator> bitSetIterators) {
  this.lead = lead;
  assert bitSetIterators.size() > 0;
  this.bitSetIterators = bitSetIterators.toArray(new BitSetIterator[0]);
  // Put the least costly iterators first so that we exit as soon as possible
  ArrayUtil.timSort(this.bitSetIterators, (a, b) -> Long.compare(a.cost(), b.cost()));
  this.bitSets = new BitSet[this.bitSetIterators.length];
  int minLen = Integer.MAX_VALUE;
  for (int i = 0; i < this.bitSetIterators.length; ++i) {
    BitSet bitSet = this.bitSetIterators[i].getBitSet();
    this.bitSets[i] = bitSet;
    minLen = Math.min(minLen, bitSet.length());
  }
  this.minLength = minLen;
}
 
Example #17
Source File: BitDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public DocIterator iterator() {
  return new DocIterator() {
    private final BitSetIterator iter = new BitSetIterator(bits, 0L); // cost is not useful here
    private int pos = iter.nextDoc();
    @Override
    public boolean hasNext() {
      return pos != DocIdSetIterator.NO_MORE_DOCS;
    }

    @Override
    public Integer next() {
      return nextDoc();
    }

    @Override
    public void remove() {
      bits.clear(pos);
    }

    @Override
    public int nextDoc() {
      int old=pos;
      pos=iter.nextDoc();
      return old;
    }

    @Override
    public float score() {
      return 0.0f;
    }
  };
}
 
Example #18
Source File: TestFieldCacheSortRandom.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(seed ^ context.docBase);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          assertEquals(docID, idSource.advance(docID));
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return true;
    }
  };
}
 
Example #19
Source File: TestDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public DocSet getIntDocSet(FixedBitSet bs) {
  int[] docs = new int[bs.cardinality()];
  BitSetIterator iter = new BitSetIterator(bs, 0);
  for (int i=0; i<docs.length; i++) {
    docs[i] = iter.nextDoc();
  }
  return new SortedIntDocSet(docs);
}
 
Example #20
Source File: TestDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public DocSlice getDocSlice(FixedBitSet bs) {
  int len = bs.cardinality();
  int[] arr = new int[len+5];
  arr[0]=10; arr[1]=20; arr[2]=30; arr[arr.length-1]=1; arr[arr.length-2]=2;
  int offset = 3;
  int end = offset + len;

  BitSetIterator iter = new BitSetIterator(bs, 0);
  // put in opposite order... DocLists are not ordered.
  for (int i=end-1; i>=offset; i--) {
    arr[i] = iter.nextDoc();
  }

  return new DocSlice(offset, len, arr, null, len*2, 100.0f, TotalHits.Relation.EQUAL_TO);
}
 
Example #21
Source File: TestScorerPerf.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(docs, docs.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #22
Source File: TestSortRandom.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      Random random = new Random(context.docBase ^ seed);
      final int maxDoc = context.reader().maxDoc();
      final NumericDocValues idSource = DocValues.getNumeric(context.reader(), "id");
      assertNotNull(idSource);
      final FixedBitSet bits = new FixedBitSet(maxDoc);
      for(int docID=0;docID<maxDoc;docID++) {
        assertEquals(docID, idSource.nextDoc());
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
          matchValues.add(docValues.get((int) idSource.longValue()));
        }
      }

      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #23
Source File: CheckJoinIndex.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Check that the given index is good to use for block joins.
 * @throws IllegalStateException if the index does not have an appropriate structure
 */
public static void check(IndexReader reader, BitSetProducer parentsFilter) throws IOException {
  for (LeafReaderContext context : reader.leaves()) {
    if (context.reader().maxDoc() == 0) {
      continue;
    }
    final BitSet parents = parentsFilter.getBitSet(context);
    if (parents == null || parents.cardinality() == 0) {
      throw new IllegalStateException("Every segment should have at least one parent, but " + context.reader() + " does not have any");
    }
    if (parents.get(context.reader().maxDoc() - 1) == false) {
      throw new IllegalStateException("The last document of a segment must always be a parent, but " + context.reader() + " has a child as a last doc");
    }
    final Bits liveDocs = context.reader().getLiveDocs();
    if (liveDocs != null) {
      int prevParentDoc = -1;
      DocIdSetIterator it = new BitSetIterator(parents, 0L);
      for (int parentDoc = it.nextDoc(); parentDoc != DocIdSetIterator.NO_MORE_DOCS; parentDoc = it.nextDoc()) {
        final boolean parentIsLive = liveDocs.get(parentDoc);
        for (int child = prevParentDoc + 1; child != parentDoc; child++) {
          final boolean childIsLive = liveDocs.get(child);
          if (parentIsLive != childIsLive) {
            if (childIsLive) {
              throw new IllegalStateException("Parent doc " + parentDoc + " of segment " + context.reader() + " is live but has a deleted child document " + child);
            } else {
              throw new IllegalStateException("Parent doc " + parentDoc + " of segment " + context.reader() + " is deleted but has a live child document " + child);
            }
          }
        }
        prevParentDoc = parentDoc;
      }
    }
  }
}
 
Example #24
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void assertSingleStepEquality(IndexedDISI disi, BitSetIterator disi2) throws IOException {
  int i = 0;
  for (int doc = disi2.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = disi2.nextDoc()) {
    assertEquals(doc, disi.nextDoc());
    assertEquals(i++, disi.index());
  }
  assertEquals(DocIdSetIterator.NO_MORE_DOCS, disi.nextDoc());
}
 
Example #25
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void doTestAllSingleJump(BitSet set, Directory dir) throws IOException {
  final int cardinality = set.cardinality();
  final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
  long length;
  int jumpTableentryCount;
  try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
    jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
    length = out.getFilePointer();
  }

  try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
    for (int i = 0; i < set.length(); i++) {
      IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
      assertEquals("The bit at " + i + " should be correct with advanceExact", set.get(i), disi.advanceExact(i));

      IndexedDISI disi2 = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
      disi2.advance(i);
      // Proper sanity check with jump tables as an error could make them seek backwards
      assertTrue("The docID should at least be " + i + " after advance(" + i + ") but was " + disi2.docID(),
          i <= disi2.docID());
      if (set.get(i)) {
        assertEquals("The docID should be present with advance", i, disi2.docID());
      } else {
        assertNotSame("The docID should not be present with advance", i, disi2.docID());
      }
    }
  }
}
 
Example #26
Source File: ShapeQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private Scorer getContainsDenseScorer(LeafReader reader, Weight weight, final float boost, ScoreMode scoreMode) throws IOException {
  final FixedBitSet result = new FixedBitSet(reader.maxDoc());
  final long[] cost = new long[]{0};
  // Get potential  documents.
  final FixedBitSet excluded = new FixedBitSet(reader.maxDoc());
  values.intersect(getContainsDenseVisitor(query, result, excluded, cost));
  result.andNot(excluded);
  final DocIdSetIterator iterator = new BitSetIterator(result, cost[0]);
  return new ConstantScoreScorer(weight, boost, scoreMode, iterator);
}
 
Example #27
Source File: BlockJoinSelector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** creates an iterator for the given bitset */
protected static BitSetIterator toIter(BitSet children) {
  return new BitSetIterator(children, 0);
}
 
Example #28
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void finish() throws IOException {
  if(contexts.length == 0) {
    return;
  }

  int currentContext = 0;
  int currentDocBase = 0;
  this.collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField);
  int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
  leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
  ScoreAndDoc dummy = new ScoreAndDoc();
  leafDelegate.setScorer(dummy);
  DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
  int globalDoc = -1;
  int nullScoreIndex = 0;
  IntIntHashMap cmap = collapseStrategy.getCollapseMap();
  IntFloatDynamicMap scores = collapseStrategy.getScores();
  FloatArrayList nullScores = collapseStrategy.getNullScores();
  MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
  float nullScore = collapseStrategy.getNullScore();

  while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {

    while(globalDoc >= nextDocBase) {
      currentContext++;
      currentDocBase = contexts[currentContext].docBase;
      nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
      leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
      leafDelegate.setScorer(dummy);
      this.collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.collapseField);
    }

    int contextDoc = globalDoc-currentDocBase;

    if(this.needsScores){
      int collapseValue;
      if (collapseValues.advanceExact(contextDoc)) {
        collapseValue = (int) collapseValues.longValue();
      } else {
        collapseValue = 0;
      }
      
      if(collapseValue != nullValue) {
        int pointer = cmap.get(collapseValue);
        dummy.score = scores.get(pointer);
      } else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
        //Its an elevated doc so no score is needed
        dummy.score = 0F;
      } else if (nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
        dummy.score = nullScore;
      } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
        dummy.score = nullScores.get(nullScoreIndex++);
      }
    }

    dummy.docId = contextDoc;
    leafDelegate.collect(contextDoc);
  }

  if(delegate instanceof DelegatingCollector) {
    ((DelegatingCollector) delegate).finish();
  }
}
 
Example #29
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void finish() throws IOException {
  if(contexts.length == 0) {
    return;
  }

  int currentContext = 0;
  int currentDocBase = 0;

  this.collapseValues = collapseValuesProducer.getSorted(null);
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  if(ordinalMap != null) {
    this.segmentValues = this.multiSortedDocValues.values[currentContext];
    this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
  } else {
    this.segmentValues = collapseValues;
  }

  int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
  leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
  ScoreAndDoc dummy = new ScoreAndDoc();
  leafDelegate.setScorer(dummy);
  DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
  int globalDoc = -1;
  int nullScoreIndex = 0;
  IntFloatDynamicMap scores = collapseStrategy.getScores();
  FloatArrayList nullScores = collapseStrategy.getNullScores();
  float nullScore = collapseStrategy.getNullScore();

  MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
  while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {


    while(globalDoc >= nextDocBase) {
      currentContext++;
      currentDocBase = contexts[currentContext].docBase;
      nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
      leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
      leafDelegate.setScorer(dummy);
      if(ordinalMap != null) {
        this.segmentValues = this.multiSortedDocValues.values[currentContext];
        this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
      }
    }

    int contextDoc = globalDoc-currentDocBase;

    if(this.needsScores){
      int ord = -1;
      if(this.ordinalMap != null) {
        //Handle ordinalMapping case
        if (segmentValues.advanceExact(contextDoc)) {
          ord = (int) segmentOrdinalMap.get(segmentValues.ordValue());
        }
      } else {
        //Handle top Level FieldCache or Single Segment Case
        if (segmentValues.advanceExact(globalDoc)) {
          ord = segmentValues.ordValue();
        }
      }

      if(ord > -1) {
        dummy.score = scores.get(ord);
      } else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
        //It's an elevated doc so no score is needed
        dummy.score = 0F;
      } else if (nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
        dummy.score = nullScore;
      } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
        dummy.score = nullScores.get(nullScoreIndex++);
      }
    }

    dummy.docId = contextDoc;
    leafDelegate.collect(contextDoc);
  }

  if(delegate instanceof DelegatingCollector) {
    ((DelegatingCollector) delegate).finish();
  }
}
 
Example #30
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void finish() throws IOException {
  if(contexts.length == 0) {
    return;
  }

  if(nullScore > -1) {
    collapsedSet.set(nullDoc);
  }

  //Handle the boosted docs.
  if(this.boostKeys != null) {
    int s = boostKeys.size();
    for(int i=0; i<s; i++) {
      int key = this.boostKeys.get(i);
      if(key != nullValue) {
        cmap.remove(key);
      }
      //Add the boosted docs to the collapsedSet
      this.collapsedSet.set(boostDocs.get(i));
    }
  }

  Iterator<IntLongCursor> it1 = cmap.iterator();

  while(it1.hasNext()) {
    IntLongCursor cursor = it1.next();
    int doc = (int)cursor.value;
    collapsedSet.set(doc);
  }

  int currentContext = 0;
  int currentDocBase = 0;

  collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field);
  int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
  leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
  ScoreAndDoc dummy = new ScoreAndDoc();
  leafDelegate.setScorer(dummy);
  DocIdSetIterator it = new BitSetIterator(collapsedSet, 0L); // cost is not useful here
  int globalDoc = -1;
  int nullScoreIndex = 0;
  while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {

    while(globalDoc >= nextDocBase) {
      currentContext++;
      currentDocBase = contexts[currentContext].docBase;
      nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
      leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
      leafDelegate.setScorer(dummy);
      collapseValues = DocValues.getNumeric(contexts[currentContext].reader(), this.field);
    }

    int contextDoc = globalDoc-currentDocBase;
    int collapseValue;
    if (collapseValues.advanceExact(contextDoc)) {
      collapseValue = (int) collapseValues.longValue();
    } else {
      collapseValue = 0;
    }

    if(collapseValue != nullValue) {
      long scoreDoc = cmap.get(collapseValue);
      dummy.score = Float.intBitsToFloat((int)(scoreDoc>>32));
    } else if(boosts && mergeBoost.boost(globalDoc)) {
      //Ignore so boosted documents don't mess up the null scoring policies.
    } else if (nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
      dummy.score = nullScore;
    } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
      dummy.score = nullScores.get(nullScoreIndex++);
    }

    dummy.docId = contextDoc;
    leafDelegate.collect(contextDoc);
  }

  if(delegate instanceof DelegatingCollector) {
    ((DelegatingCollector) delegate).finish();
  }
}