Java Code Examples for org.apache.lucene.util.FixedBitSet#get()

@Override
protected void fillDocsAndScores(FixedBitSet matchingDocs, TermsEnum termsEnum) throws IOException {
  BytesRef spare = new BytesRef();
  PostingsEnum postingsEnum = null;
  for (int i = 0; i < terms.size(); i++) {
    if (termsEnum.seekExact(terms.get(ords[i], spare))) {
      postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
      float score = TermsIncludingScoreQuery.this.scores[ords[i]];
      for (int doc = postingsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = postingsEnum.nextDoc()) {
        // I prefer this:
        /*if (scores[doc] < score) {
          scores[doc] = score;
          matchingDocs.set(doc);
        }*/
        // But this behaves the same as MVInnerScorer and only then the tests will pass:
        if (!matchingDocs.get(doc)) {
          scores[doc] = score;
          matchingDocs.set(doc);
        }
      }
    }
  }
}
 

public void testBuildDocMap() {
  final int maxDoc = TestUtil.nextInt(random(), 1, 128);
  final int numDocs = TestUtil.nextInt(random(), 0, maxDoc);
  final FixedBitSet liveDocs = new FixedBitSet(maxDoc);
  for (int i = 0; i < numDocs; ++i) {
    while (true) {
      final int docID = random().nextInt(maxDoc);
      if (!liveDocs.get(docID)) {
        liveDocs.set(docID);
        break;
      }
    }
  }

  final PackedLongValues docMap = MergeState.removeDeletes(maxDoc, liveDocs);

  // assert the mapping is compact
  for (int i = 0, del = 0; i < maxDoc; ++i) {
    if (liveDocs.get(i) == false) {
      ++del;
    } else {
      assertEquals(i - del, docMap.get(i));
    }
  }
}
 

@Override
public NumericDocValues getNormValues(String field) throws IOException {
  NumericDocValues ndv = super.getNormValues(field);
  if (ndv == null) {
    return null;
  }
  FixedBitSet docsWithTerms = new FixedBitSet(maxDoc());
  TermsEnum termsEnum = terms(field).iterator();
  PostingsEnum postings = null;
  while (termsEnum.next() != null) {
    postings = termsEnum.postings(postings, PostingsEnum.NONE);
    docsWithTerms.or(postings);
  }
  return new FilterNumericDocValues(ndv) {
    @Override
    public long longValue() throws IOException {
      return docsWithTerms.get(docID()) ? super.longValue() : 0L;
    }
  };
}
 

public static Vector weightedSuperposition(
    BinaryVector v1, double weight1, BinaryVector v2, double weight2) {
  BinaryVector conclusion = (BinaryVector) VectorFactory.createZeroVector(VectorType.BINARY, v1.getDimension());
  FixedBitSet cVote = conclusion.bitSet;
  FixedBitSet v1vote = v1.bitSet;
  FixedBitSet v2vote = v2.bitSet;

  Random random = new Random();
  random.setSeed(Bobcat.asLong(v1.writeLongToString())); 

  for (int x = 0; x < v1.getDimension(); x++) {
    double probability = 0;
    if (v1vote.get(x)) probability += weight1 / (weight1 + weight2);
    if (v2vote.get(x)) probability += weight2 / (weight1 + weight2);

    if (random.nextDouble() <= probability)
      cVote.set(x);
  }
  return conclusion;
}
 

/** pp was just advanced. If that caused a repeater collision, resolve by advancing the lesser
 * of the two colliding pps. Note that there can only be one collision, as by the initialization
 * there were no collisions before pp was advanced.  */
private boolean advanceRpts(PhrasePositions pp) throws IOException {
  if (pp.rptGroup < 0) {
    return true; // not a repeater
  }
  PhrasePositions[] rg = rptGroups[pp.rptGroup];
  FixedBitSet bits = new FixedBitSet(rg.length); // for re-queuing after collisions are resolved
  int k0 = pp.rptInd;
  int k;
  while((k=collide(pp)) >= 0) {
    pp = lesser(pp, rg[k]); // always advance the lesser of the (only) two colliding pps
    if (!advancePP(pp)) {
      return false; // exhausted
    }
    if (k != k0) { // careful: mark only those currently in the queue
      bits = FixedBitSet.ensureCapacity(bits, k);
      bits.set(k); // mark that pp2 need to be re-queued
    }
  }
  // collisions resolved, now re-queue
  // empty (partially) the queue until seeing all pps advanced for resolving collisions
  int n = 0;
  // TODO would be good if we can avoid calling cardinality() in each iteration!
  int numBits = bits.length(); // larges bit we set
  while (bits.cardinality() > 0) {
    PhrasePositions pp2 = pq.pop();
    rptStack[n++] = pp2;
    if (pp2.rptGroup >= 0 
        && pp2.rptInd < numBits  // this bit may not have been set
        && bits.get(pp2.rptInd)) {
      bits.clear(pp2.rptInd);
    }
  }
  // add back to queue
  for (int i=n-1; i>=0; i--) {
    pq.add(rptStack[i]);
  }
  return true;
}
 

private static int dedup(int[] arr, int length, FixedBitSet acceptDocs) {
  int pos = 0;
  int previous = -1;
  for (int i = 0; i < length; ++i) {
    final int value = arr[i];
    // assert value >= previous;
    if (value != previous && (acceptDocs == null || acceptDocs.get(value))) {
      arr[pos++] = value;
      previous = value;
    }
  }
  return pos;
}
 

static void generate(int maxSize, int bitsToSet) {
  bs = new FixedBitSet(maxSize);
  ids = new int[bitsToSet];
  int count=0;
  if (maxSize>0) {
    for (int i=0; i<bitsToSet; i++) {
      int id=rand.nextInt(maxSize);
      if (!bs.get(id)) {
        bs.set(id);
        ids[count++]=id;
      }
    }
  }
  bds = new BitDocSet(bs,bitsToSet);
}
 

public void testMultiValuedRandom1() throws IOException {
  boolean withDeletes = random().nextBoolean();
  Directory dir = newDirectory();
  IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random()));
  Sort indexSort = new Sort(new SortedNumericSortField("foo", SortField.Type.LONG));
  iwc.setIndexSort(indexSort);
  IndexWriter w = new IndexWriter(dir, iwc);
  final int numDocs = atLeast(200);
  final FixedBitSet deleted = new FixedBitSet(numDocs);
  for (int i = 0; i < numDocs; ++i) {
    Document doc = new Document();
    int num = random().nextInt(10);
    for (int j = 0; j < num; j++) {
      doc.add(new SortedNumericDocValuesField("foo", random().nextInt(2000)));
    }
    doc.add(new StringField("id", Integer.toString(i), Store.YES));
    doc.add(new NumericDocValuesField("id", i));
    w.addDocument(doc);
    if (random().nextInt(5) == 0) {
      w.getReader().close();
    } else if (random().nextInt(30) == 0) {
      w.forceMerge(2);
    } else if (random().nextInt(4) == 0) {
      final int id = TestUtil.nextInt(random(), 0, i);
      deleted.set(id);
      w.deleteDocuments(new Term("id", Integer.toString(id)));
    }
  }

  DirectoryReader reader = w.getReader();
  // Now check that the index is consistent
  IndexSearcher searcher = newSearcher(reader);
  for (int i = 0; i < numDocs; ++i) {
    TermQuery termQuery = new TermQuery(new Term("id", Integer.toString(i)));
    final TopDocs topDocs = searcher.search(termQuery, 1);
    if (deleted.get(i)) {
      assertEquals(0, topDocs.totalHits.value);
    } else {
      assertEquals(1, topDocs.totalHits.value);
      NumericDocValues values = MultiDocValues.getNumericValues(reader, "id");
      assertEquals(topDocs.scoreDocs[0].doc, values.advance(topDocs.scoreDocs[0].doc));
      assertEquals(i, values.longValue());
      Document document = reader.document(topDocs.scoreDocs[0].doc);
      assertEquals(Integer.toString(i), document.get("id"));
    }
  }

  reader.close();
  w.close();
  dir.close();
}
 

/** test random generated lines */
protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // line
    Object queryLine = randomQueryLine(shapes);
    Component2D queryLine2D = toLine2D(queryLine);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), POINT_LINE_RELATIONS);
    Query query = newLineQuery(FIELD_NAME, queryRelation, queryLine);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryLine2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryLine);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated polygons */
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryPolygon = randomQueryPolygon();
    Component2D queryPoly2D = toPolygon2D(queryPolygon);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryPolygon);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated point queries */
protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
    }

    Object[] queryPoints = nextPoints();
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Component2D queryPoly2D;
    Query query;
    if (queryRelation == QueryRelation.CONTAINS) {
      queryPoly2D = toPoint2D(queryPoints[0]);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints[0]);
    } else {
      queryPoly2D = toPoint2D(queryPoints);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints);
    }

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;

      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  rect=Points(" + Arrays.toString(queryPoints) + ")\n");
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 

/** test random generated circles */
protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryCircle = randomQueryCircle();
    Component2D queryCircle2D = toCircle2D(queryCircle);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newDistanceQuery(FIELD_NAME, queryRelation, queryCircle);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryCircle2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  distanceQuery=" + queryCircle.toString());
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}