Java Code Examples for org.apache.lucene.util.FixedBitSet#clear()

public void testOneDocMissingFixed() throws IOException {
  int maxDoc = 9699;
  final byte denseRankPower = rarely() ? -1 : (byte) (random().nextInt(7)+7); // sane + chance of disable
  FixedBitSet set = new FixedBitSet(maxDoc);
  set.set(0, maxDoc);
  set.clear(1345);
  try (Directory dir = newDirectory()) {

    final int cardinality = set.cardinality();
    long length;
    int jumpTableentryCount;
    try (IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT)) {
      jumpTableentryCount = IndexedDISI.writeBitSet(new BitSetIterator(set, cardinality), out, denseRankPower);
      length = out.getFilePointer();
    }

    int step = 16000;
    try (IndexInput in = dir.openInput("foo", IOContext.DEFAULT)) {
      IndexedDISI disi = new IndexedDISI(in, 0L, length, jumpTableentryCount, denseRankPower, cardinality);
      BitSetIterator disi2 = new BitSetIterator(set, cardinality);
      assertAdvanceEquality(disi, disi2, step);
    }
  }
}
 

@Override
public DocSet andNot(DocSet other) {
  FixedBitSet newbits = bits.clone();
  if (other instanceof BitDocSet) {
    newbits.andNot(((BitDocSet) other).bits);
  } else {
    DocIterator iter = other.iterator();
    while (iter.hasNext()) {
      int doc = iter.nextDoc();
      if (doc < newbits.length()) {
        newbits.clear(doc);
      }
    }
  }
  return new BitDocSet(newbits);
}
 

/** pp was just advanced. If that caused a repeater collision, resolve by advancing the lesser
 * of the two colliding pps. Note that there can only be one collision, as by the initialization
 * there were no collisions before pp was advanced.  */
private boolean advanceRpts(PhrasePositions pp) throws IOException {
  if (pp.rptGroup < 0) {
    return true; // not a repeater
  }
  PhrasePositions[] rg = rptGroups[pp.rptGroup];
  FixedBitSet bits = new FixedBitSet(rg.length); // for re-queuing after collisions are resolved
  int k0 = pp.rptInd;
  int k;
  while((k=collide(pp)) >= 0) {
    pp = lesser(pp, rg[k]); // always advance the lesser of the (only) two colliding pps
    if (!advancePP(pp)) {
      return false; // exhausted
    }
    if (k != k0) { // careful: mark only those currently in the queue
      bits = FixedBitSet.ensureCapacity(bits, k);
      bits.set(k); // mark that pp2 need to be re-queued
    }
  }
  // collisions resolved, now re-queue
  // empty (partially) the queue until seeing all pps advanced for resolving collisions
  int n = 0;
  // TODO would be good if we can avoid calling cardinality() in each iteration!
  int numBits = bits.length(); // larges bit we set
  while (bits.cardinality() > 0) {
    PhrasePositions pp2 = pq.pop();
    rptStack[n++] = pp2;
    if (pp2.rptGroup >= 0 
        && pp2.rptInd < numBits  // this bit may not have been set
        && bits.get(pp2.rptInd)) {
      bits.clear(pp2.rptInd);
    }
  }
  // add back to queue
  for (int i=n-1; i>=0; i--) {
    pq.add(rptStack[i]);
  }
  return true;
}
 

public void testOneDocMissing() throws IOException {
  int maxDoc = TestUtil.nextInt(random(), 1, 1000000);
  FixedBitSet set = new FixedBitSet(maxDoc);
  set.set(0, maxDoc);
  set.clear(random().nextInt(maxDoc));
  try (Directory dir = newDirectory()) {
    doTest(set, dir);
  }
}
 

public void testFewMissingDocs() throws IOException {
  try (Directory dir = newDirectory()) {
    int numIters = atLeast(10);
    for (int iter = 0; iter < numIters; ++iter) {
      int maxDoc = TestUtil.nextInt(random(), 1, 100000);
      FixedBitSet set = new FixedBitSet(maxDoc);
      set.set(0, maxDoc);
      final int numMissingDocs = TestUtil.nextInt(random(), 2, 1000);
      for (int i = 0; i < numMissingDocs; ++i) {
        set.clear(random().nextInt(maxDoc));
      }
      doTest(set, dir);
    }
  }
}
 

private static FixedBitSet clearRandomBits(FixedBitSet other) {
  final FixedBitSet set = new FixedBitSet(other.length());
  set.or(other);
  for (int i = 0; i < set.length(); ++i) {
    if (random().nextBoolean()) {
      set.clear(i);
    }
  }
  return set;
}
 

@Override
public void reset() throws IOException {
  counts = null;
  for (FixedBitSet bits : arr) {
    if (bits == null) continue;
    bits.clear(0, bits.length());
  }
}
 

void writeDoc(SortDoc sortDoc,
                        List<LeafReaderContext> leaves,
                        EntryWriter ew) throws IOException {

  int ord = sortDoc.ord;
  FixedBitSet set = sets[ord];
  set.clear(sortDoc.docId);
  LeafReaderContext context = leaves.get(ord);
  int fieldIndex = 0;
  for (FieldWriter fieldWriter : fieldWriters) {
    if (fieldWriter.write(sortDoc, context.reader(), ew, fieldIndex)) {
      ++fieldIndex;
    }
  }
}
 

/**
 * Writes the docIDs from it to out, in logical blocks, one for each 65536 docIDs in monotonically
 * increasing gap-less order.
 * The caller must keep track of the number of jump-table entries (returned by this method) as well as the
 * denseRankPower and provide them when constructing an IndexedDISI for reading.
 * @param it  the document IDs.
 * @param out destination for the blocks.
 * @param denseRankPower for {@link Method#DENSE} blocks, a rank will be written every {@code 2^denseRankPower} docIDs.
 *                       Values < 7 (every 128 docIDs) or > 15 (every 32768 docIDs) disables DENSE rank.
 *                       Recommended values are 8-12: Every 256-4096 docIDs or 4-64 longs.
 *                       {@link #DEFAULT_DENSE_RANK_POWER} is 9: Every 512 docIDs.
 *                       This should be stored in meta and used when creating an instance of IndexedDISI.
 * @throws IOException if there was an error writing to out.
 * @return the number of jump-table entries following the blocks, -1 for no entries.
 *         This should be stored in meta and used when creating an instance of IndexedDISI.
 */
static short writeBitSet(DocIdSetIterator it, IndexOutput out, byte denseRankPower) throws IOException {
  final long origo = out.getFilePointer(); // All jumps are relative to the origo
  if ((denseRankPower < 7 || denseRankPower > 15) && denseRankPower != -1) {
    throw new IllegalArgumentException("Acceptable values for denseRankPower are 7-15 (every 128-32768 docIDs). " +
        "The provided power was " + denseRankPower + " (every " + (int)Math.pow(2, denseRankPower) + " docIDs)");
  }
  int totalCardinality = 0;
  int blockCardinality = 0;
  final FixedBitSet buffer = new FixedBitSet(1<<16);
  int[] jumps = new int[ArrayUtil.oversize(1, Integer.BYTES*2)];
  int prevBlock = -1;
  int jumpBlockIndex = 0;

  for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
    final int block = doc >>> 16;
    if (prevBlock != -1 && block != prevBlock) {
      // Track offset+index from previous block up to current
      jumps = addJumps(jumps, out.getFilePointer()-origo, totalCardinality, jumpBlockIndex, prevBlock+1);
      jumpBlockIndex = prevBlock+1;
      // Flush block
      flush(prevBlock, buffer, blockCardinality, denseRankPower, out);
      // Reset for next block
      buffer.clear(0, buffer.length());
      totalCardinality += blockCardinality;
      blockCardinality = 0;
    }
    buffer.set(doc & 0xFFFF);
    blockCardinality++;
    prevBlock = block;
  }
  if (blockCardinality > 0) {
    jumps = addJumps(jumps, out.getFilePointer()-origo, totalCardinality, jumpBlockIndex, prevBlock+1);
    totalCardinality += blockCardinality;
    flush(prevBlock, buffer, blockCardinality, denseRankPower, out);
    buffer.clear(0, buffer.length());
    prevBlock++;
  }
  final int lastBlock = prevBlock == -1 ? 0 : prevBlock; // There will always be at least 1 block (NO_MORE_DOCS)
  // Last entry is a SPARSE with blockIndex == 32767 and the single entry 65535, which becomes the docID NO_MORE_DOCS
  // To avoid creating 65K jump-table entries, only a single entry is created pointing to the offset of the
  // NO_MORE_DOCS block, with the jumpBlockIndex set to the logical EMPTY block after all real blocks.
  jumps = addJumps(jumps, out.getFilePointer()-origo, totalCardinality, lastBlock, lastBlock+1);
  buffer.set(DocIdSetIterator.NO_MORE_DOCS & 0xFFFF);
  flush(DocIdSetIterator.NO_MORE_DOCS >>> 16, buffer, 1, denseRankPower, out);
  // offset+index jump-table stored at the end
  return flushBlockJumps(jumps, lastBlock+1, out, origo);
}
 

private boolean isLinearizable(SequentialSpec spec, List<Event> history, BooleanSupplier terminateEarly) {
    LOGGER.debug("Checking history of size: {}: {}", history.size(), history);
    Object state = spec.initialState(); // the current state of the datatype
    final FixedBitSet linearized = new FixedBitSet(history.size() / 2); // the linearized prefix of the history

    final Cache cache = new Cache(); // cache of explored <state, linearized prefix> pairs
    final Deque<Tuple<Entry, Object>> calls = new LinkedList<>(); // path we're currently exploring

    final Entry headEntry = createLinkedEntries(history);
    Entry entry = headEntry.next; // current entry

    while (headEntry.next != null) {
        if (terminateEarly.getAsBoolean()) {
            return false;
        }
        if (entry.match != null) {
            final Optional<Object> maybeNextState = spec.nextState(state, entry.event.value, entry.match.event.value);
            boolean shouldExploreNextState = false;
            if (maybeNextState.isPresent()) {
                // check if we have already explored this linearization
                final FixedBitSet updatedLinearized = linearized.clone();
                updatedLinearized.set(entry.id);
                shouldExploreNextState = cache.add(maybeNextState.get(), updatedLinearized);
            }
            if (shouldExploreNextState) {
                calls.push(new Tuple<>(entry, state));
                state = maybeNextState.get();
                linearized.set(entry.id);
                entry.lift();
                entry = headEntry.next;
            } else {
                entry = entry.next;
            }
        } else {
            if (calls.isEmpty()) {
                return false;
            }
            final Tuple<Entry, Object> top = calls.pop();
            entry = top.v1();
            state = top.v2();
            linearized.clear(entry.id);
            entry.unlift();
            entry = entry.next;
        }
    }
    return true;
}