org.apache.lucene.util.FixedBitSet Java Examples

The following examples show how to use org.apache.lucene.util.FixedBitSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TestScorerPerf.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testConjunctions() throws Exception {
  // test many small sets... the bugs will be found on boundary conditions
  try (Directory d = newDirectory()) {
    IndexWriter iw = new IndexWriter(d, newIndexWriterConfig(new MockAnalyzer(random())));
    iw.addDocument(new Document());
    iw.close();

    try (DirectoryReader r = DirectoryReader.open(d)) {
      IndexSearcher s = newSearcher(r);
      s.setQueryCache(null);

      FixedBitSet[] sets = randBitSets(atLeast(1000), atLeast(10));

      int iterations = TEST_NIGHTLY ? atLeast(10000) : atLeast(500);
      doConjunctions(s, sets, iterations, atLeast(5));
      doNestedConjunctions(s, sets, iterations, atLeast(3), atLeast(3));
    }
  }
}
 
Example #2
Source File: BitDocSet.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public DocSet andNot(DocSet other) {
  FixedBitSet newbits = bits.clone();
  if (other instanceof BitDocSet) {
    newbits.andNot(((BitDocSet) other).bits);
  } else {
    DocIterator iter = other.iterator();
    while (iter.hasNext()) {
      int doc = iter.nextDoc();
      if (doc < newbits.length()) {
        newbits.clear(doc);
      }
    }
  }
  return new BitDocSet(newbits);
}
 
Example #3
Source File: TaggerRequestHandler.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private DocList getDocList(int rows, FixedBitSet matchDocIdsBS) throws IOException {
  //Now we must supply a Solr DocList and add it to the response.
  //  Typically this is gotten via a SolrIndexSearcher.search(), but in this case we
  //  know exactly what documents to return, the order doesn't matter nor does
  //  scoring.
  //  Ideally an implementation of DocList could be directly implemented off
  //  of a BitSet, but there are way too many methods to implement for a minor
  //  payoff.
  int matchDocs = matchDocIdsBS.cardinality();
  int[] docIds = new int[ Math.min(rows, matchDocs) ];
  DocIdSetIterator docIdIter = new BitSetIterator(matchDocIdsBS, 1);
  for (int i = 0; i < docIds.length; i++) {
    docIds[i] = docIdIter.nextDoc();
  }
  return new DocSlice(0, docIds.length, docIds, null, matchDocs, 1f, TotalHits.Relation.EQUAL_TO);
}
 
Example #4
Source File: HashQParserPlugin.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {

  SolrIndexSearcher solrIndexSearcher = (SolrIndexSearcher)searcher;
  IndexReaderContext context = solrIndexSearcher.getTopReaderContext();

  List<LeafReaderContext> leaves =  context.leaves();
  FixedBitSet[] fixedBitSets = new FixedBitSet[leaves.size()];

  for(LeafReaderContext leaf : leaves) {
    try {
      SegmentPartitioner segmentPartitioner = new SegmentPartitioner(leaf,worker,workers, keys, solrIndexSearcher);
      segmentPartitioner.run();
      fixedBitSets[segmentPartitioner.context.ord] = segmentPartitioner.docs;
    } catch(Exception e) {
      throw new IOException(e);
    }
  }

  ConstantScoreQuery constantScoreQuery = new ConstantScoreQuery(new BitsFilter(fixedBitSets));
  return searcher.rewrite(constantScoreQuery).createWeight(searcher, ScoreMode.COMPLETE_NO_SCORES, boost);
}
 
Example #5
Source File: BinaryVectorUtils.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
public static Vector weightedSuperposition(
    BinaryVector v1, double weight1, BinaryVector v2, double weight2) {
  BinaryVector conclusion = (BinaryVector) VectorFactory.createZeroVector(VectorType.BINARY, v1.getDimension());
  FixedBitSet cVote = conclusion.bitSet;
  FixedBitSet v1vote = v1.bitSet;
  FixedBitSet v2vote = v2.bitSet;

  Random random = new Random();
  random.setSeed(Bobcat.asLong(v1.writeLongToString())); 

  for (int x = 0; x < v1.getDimension(); x++) {
    double probability = 0;
    if (v1vote.get(x)) probability += weight1 / (weight1 + weight2);
    if (v2vote.get(x)) probability += weight2 / (weight1 + weight2);

    if (random.nextDouble() <= probability)
      cVote.set(x);
  }
  return conclusion;
}
 
Example #6
Source File: SortingLeafReader.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public BinaryDocValues getBinaryDocValues(String field) throws IOException {
  final BinaryDocValues oldDocValues = in.getBinaryDocValues(field);
  if (oldDocValues == null) return null;
  CachedBinaryDVs dvs;
  synchronized (cachedBinaryDVs) {
    dvs = cachedBinaryDVs.get(field);
    if (dvs == null) {
      FixedBitSet docsWithField = new FixedBitSet(maxDoc());
      BytesRef[] values = new BytesRef[maxDoc()];
      while (true) {
        int docID = oldDocValues.nextDoc();
        if (docID == NO_MORE_DOCS) {
          break;
        }
        int newDocID = docMap.oldToNew(docID);
        docsWithField.set(newDocID);
        values[newDocID] = BytesRef.deepCopyOf(oldDocValues.binaryValue());
      }
      dvs = new CachedBinaryDVs(values, docsWithField);
      cachedBinaryDVs.put(field, dvs);
    }
  }
  return new SortingBinaryDocValues(dvs);
}
 
Example #7
Source File: TermsIncludingScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
protected void fillDocsAndScores(FixedBitSet matchingDocs, TermsEnum termsEnum) throws IOException {
  BytesRef spare = new BytesRef();
  PostingsEnum postingsEnum = null;
  for (int i = 0; i < terms.size(); i++) {
    if (termsEnum.seekExact(terms.get(ords[i], spare))) {
      postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
      float score = TermsIncludingScoreQuery.this.scores[ords[i]];
      for (int doc = postingsEnum.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = postingsEnum.nextDoc()) {
        matchingDocs.set(doc);
        // In the case the same doc is also related to a another doc, a score might be overwritten. I think this
        // can only happen in a many-to-many relation
        scores[doc] = score;
      }
    }
  }
}
 
Example #8
Source File: BitSetHitStream.java    From siren-join with GNU Affero General Public License v3.0 6 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  current = new FixedBitSet(context.reader().maxDoc());
  fixedBitSets.add(context.ord, current);

  return new LeafCollector() {

    @Override
    public void setScorer(Scorer scorer) throws IOException {}

    @Override
    public void collect(int doc) throws IOException {
      current.set(doc);
      totalHits++;
    }

  };
}
 
Example #9
Source File: NumberRangeFacetsTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private Bits searchForDocBits(Query query) throws IOException {
  FixedBitSet bitSet = new FixedBitSet(indexSearcher.getIndexReader().maxDoc());
  indexSearcher.search(query,
      new SimpleCollector() {
        int leafDocBase;
        @Override
        public void collect(int doc) throws IOException {
          bitSet.set(leafDocBase + doc);
        }

        @Override
        protected void doSetNextReader(LeafReaderContext context) throws IOException {
          leafDocBase = context.docBase;
        }

        @Override
        public ScoreMode scoreMode() {
          return ScoreMode.COMPLETE_NO_SCORES;
        }
      });
  return bitSet;
}
 
Example #10
Source File: ExpandComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 
Example #11
Source File: SolrOwnerScorer.java    From SearchServices with GNU Lesser General Public License v3.0 6 votes vote down vote up
public static SolrOwnerScorer createOwnerScorer(Weight weight, LeafReaderContext context, SolrIndexSearcher searcher, String authority) throws IOException
{
    if (AuthorityType.getAuthorityType(authority) == AuthorityType.USER)
    {
        DocSet ownedDocs = (DocSet) searcher.cacheLookup(CacheConstants.ALFRESCO_OWNERLOOKUP_CACHE, authority);

        if (ownedDocs == null)
        {
            // Cache miss: query the index for docs where the owner matches the authority. 
            ownedDocs = searcher.getDocSet(new TermQuery(new Term(QueryConstants.FIELD_OWNER, authority)));
            searcher.cacheInsert(CacheConstants.ALFRESCO_OWNERLOOKUP_CACHE, authority, ownedDocs);
        }
        return new SolrOwnerScorer(weight, ownedDocs, context, searcher);
    }
    
    // Return an empty doc set, as the authority isn't a user.
    return new SolrOwnerScorer(weight, new BitDocSet(new FixedBitSet(0)), context, searcher);
}
 
Example #12
Source File: BlockJoin.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** childInput may also contain parents (i.e. a parent or below will all roll up to that parent) */
public static DocSet toParents(DocSet childInput, BitDocSet parentList, QueryContext qcontext) throws IOException {
  FixedBitSet parentBits = parentList.getBits();
  DocSetCollector collector = new DocSetCollector(qcontext.searcher().maxDoc());
  DocIterator iter = childInput.iterator();
  int currentParent = -1;
  while (iter.hasNext()) {
    int childDoc = iter.nextDoc(); // TODO: skipping
    if (childDoc <= currentParent) { // use <= since we also allow parents in the input
      // we already visited this parent
      continue;
    }
    currentParent = parentBits.nextSetBit(childDoc);
    if (currentParent != DocIdSetIterator.NO_MORE_DOCS) {
      // only collect the parent the first time we skip to it
      collector.collect( currentParent );
    }
  }
  return collector.getDocSet();
}
 
Example #13
Source File: IndexedDISI.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private static void flush(
    int block, FixedBitSet buffer, int cardinality, byte denseRankPower, IndexOutput out) throws IOException {
  assert block >= 0 && block < 65536;
  out.writeShort((short) block);
  assert cardinality > 0 && cardinality <= 65536;
  out.writeShort((short) (cardinality - 1));
  if (cardinality > MAX_ARRAY_LENGTH) {
    if (cardinality != 65536) { // all docs are set
      if (denseRankPower != -1) {
        final byte[] rank = createRank(buffer, denseRankPower);
        out.writeBytes(rank, rank.length);
      }
      for (long word : buffer.getBits()) {
        out.writeLong(word);
      }
    }
  } else {
    BitSetIterator it = new BitSetIterator(buffer, cardinality);
    for (int doc = it.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; doc = it.nextDoc()) {
      out.writeShort((short) doc);
    }
  }
}
 
Example #14
Source File: IndexFileBitSetTest.java    From incubator-retired-blur with Apache License 2.0 6 votes vote down vote up
@Test
public void test() throws IOException {
  Random random = new Random(_seed);
  int numBits = random.nextInt(10000000);
  FixedBitSet fixedBitSet = new FixedBitSet(numBits);
  populate(random, numBits, fixedBitSet);
  String id = "id";
  String segmentName = "seg1";
  RAMDirectory directory = new RAMDirectory();
  IndexFileBitSet indexFileBitSet = new IndexFileBitSet(numBits, id, segmentName, directory);
  assertFalse(indexFileBitSet.exists());
  indexFileBitSet.create(fixedBitSet.iterator());
  indexFileBitSet.load();
  checkEquals(fixedBitSet.iterator(), indexFileBitSet.iterator(), numBits);
  indexFileBitSet.close();
  
  String[] listAll = directory.listAll();
  for (String s : listAll) {
    System.out.println(s + " " + directory.fileLength(s));
  }
}
 
Example #15
Source File: TestTermAutomatonQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {
    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      int maxDoc = context.reader().maxDoc();
      FixedBitSet bits = new FixedBitSet(maxDoc);
      Random random = new Random(seed ^ context.docBase);
      for(int docID=0;docID<maxDoc;docID++) {
        if (random.nextFloat() <= density) {
          bits.set(docID);
          //System.out.println("  acc id=" + idSource.getInt(docID) + " docID=" + docID);
        }
      }
      return new ConstantScoreScorer(this, score(), scoreMode, new BitSetIterator(bits, bits.approximateCardinality()));
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return false;
    }
  };
}
 
Example #16
Source File: DocSetBuilder.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public DocSet build(FixedBitSet filter) {
  if (bitSet != null) {
    if (filter != null) {
      bitSet.and(filter);
    }
    return new BitDocSet(bitSet);
    // TODO - if this set will be cached, should we make it smaller if it's below DocSetUtil.smallSetSize?
  } else {
    LSBRadixSorter sorter = new LSBRadixSorter();
    sorter.sort(PackedInts.bitsRequired(maxDoc - 1), buffer, pos);
    final int l = dedup(buffer, pos, filter);
    assert l <= pos;
    return new SortedIntDocSet(buffer, l);  // TODO: have option to not shrink in the future if it will be a temporary set
  }
}
 
Example #17
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
private void assertIntRangeSplit(final int lower, final int upper, int precisionStep,
  final boolean useBitSet, final Iterable<Integer> expectedBounds, final Iterable<Integer> expectedShifts
) {
  final FixedBitSet bits=useBitSet ? new FixedBitSet(upper-lower+1) : null;
  final Iterator<Integer> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
  final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();
  
  LegacyNumericUtils.splitIntRange(new LegacyNumericUtils.IntRangeBuilder() {
    @Override
    public void addRange(int min, int max, int shift) {
      assertTrue("min, max should be inside bounds", min >= lower && min <= upper && max >= lower && max <= upper);
      if (useBitSet) for (int i = min; i <= max; i++) {
        assertFalse("ranges should not overlap", bits.getAndSet(i - lower));
        // extra exit condition to prevent overflow on MAX_VALUE
        if (i == max) break;
      }
      if (neededBounds == null)
        return;
      // make unsigned ints for easier display and understanding
      min ^= 0x80000000;
      max ^= 0x80000000;
      //System.out.println("0x"+Integer.toHexString(min>>>shift)+",0x"+Integer.toHexString(max>>>shift)+")/*shift="+shift+"*/,");
      assertEquals("shift", neededShifts.next().intValue(), shift);
      assertEquals("inner min bound", neededBounds.next().intValue(), min >>> shift);
      assertEquals("inner max bound", neededBounds.next().intValue(), max >>> shift);
    }
  }, precisionStep, lower, upper);
  
  if (useBitSet) {
    // after flipping all bits in the range, the cardinality should be zero
    bits.flip(0, upper-lower+1);
    assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality());
  }
}
 
Example #18
Source File: SolrIndexSplitter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
SplittingQuery(int partition, SchemaField field, DocRouter.Range[] rangesArr, HashBasedRouter hashRouter, String splitKey,
               Map<IndexReader.CacheKey, FixedBitSet[]> docsToDelete, AtomicInteger currentPartition) {
  this.partition = partition;
  this.field = field;
  this.rangesArr = rangesArr;
  this.hashRouter = hashRouter;
  this.splitKey = splitKey;
  this.docsToDelete = docsToDelete;
  this.currentPartition = currentPartition;
}
 
Example #19
Source File: TestIndexedDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testDenseMultiBlock() throws IOException {
  try (Directory dir = newDirectory()) {
    int maxDoc = 10 * 65536; // 10 blocks
    FixedBitSet set = new FixedBitSet(maxDoc);
    for (int i = 0; i < maxDoc; i += 2) { // Set every other to ensure dense
      set.set(i);
    }
    doTest(set, dir);
  }
}
 
Example #20
Source File: BitDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public DocSet intersection(DocSet other) {
  // intersection is overloaded in the smaller DocSets to be more
  // efficient, so dispatch off of it instead.
  if (!(other instanceof BitDocSet)) {
    return other.intersection(this);
  }

  // Default... handle with bitsets.
  FixedBitSet newbits = getFixedBitSetClone();
  newbits.and(other.getFixedBitSet());
  return new BitDocSet(newbits);
}
 
Example #21
Source File: TestConjunctionDISI.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static FixedBitSet randomSet(int maxDoc) {
  final int step = TestUtil.nextInt(random(), 1, 10);
  FixedBitSet set = new FixedBitSet(maxDoc);
  for (int doc = random().nextInt(step); doc < maxDoc; doc += TestUtil.nextInt(random(), 1, step)) {
    set.set(doc);
  }
  return set;
}
 
Example #22
Source File: NumericDocValuesWriter.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static SortingLeafReader.CachedNumericDVs sortDocValues(int maxDoc, Sorter.DocMap sortMap, NumericDocValues oldDocValues) throws IOException {
  FixedBitSet docsWithField = new FixedBitSet(maxDoc);
  long[] values = new long[maxDoc];
  while (true) {
    int docID = oldDocValues.nextDoc();
    if (docID == NO_MORE_DOCS) {
      break;
    }
    int newDocID = sortMap.oldToNew(docID);
    docsWithField.set(newDocID);
    values[newDocID] = oldDocValues.longValue();
  }
  return new SortingLeafReader.CachedNumericDVs(values, docsWithField);
}
 
Example #23
Source File: CheckIndex.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Sole constructor */
public VerifyPointsVisitor(String fieldName, int maxDoc, PointValues values) throws IOException {
  this.maxDoc = maxDoc;
  this.fieldName = fieldName;
  numDataDims = values.getNumDimensions();
  numIndexDims = values.getNumIndexDimensions();
  bytesPerDim = values.getBytesPerDimension();
  packedBytesCount = numDataDims * bytesPerDim;
  packedIndexBytesCount = numIndexDims * bytesPerDim;
  globalMinPackedValue = values.getMinPackedValue();
  globalMaxPackedValue = values.getMaxPackedValue();
  docsSeen = new FixedBitSet(maxDoc);
  lastMinPackedValue = new byte[packedIndexBytesCount];
  lastMaxPackedValue = new byte[packedIndexBytesCount];
  lastPackedValue = new byte[packedBytesCount];

  if (values.getDocCount() > values.size()) {
    throw new RuntimeException("point values for field \"" + fieldName + "\" claims to have size=" + values.size() + " points and inconsistent docCount=" + values.getDocCount());
  }

  if (values.getDocCount() > maxDoc) {
    throw new RuntimeException("point values for field \"" + fieldName + "\" claims to have docCount=" + values.getDocCount() + " but that's greater than maxDoc=" + maxDoc);
  }

  if (globalMinPackedValue == null) {
    if (values.size() != 0) {
      throw new RuntimeException("getMinPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
    }
  } else if (globalMinPackedValue.length != packedIndexBytesCount) {
    throw new RuntimeException("getMinPackedValue for field \"" + fieldName + "\" return length=" + globalMinPackedValue.length + " array, but should be " + packedBytesCount);
  }
  if (globalMaxPackedValue == null) {
    if (values.size() != 0) {
      throw new RuntimeException("getMaxPackedValue is null points for field \"" + fieldName + "\" yet size=" + values.size());
    }
  } else if (globalMaxPackedValue.length != packedIndexBytesCount) {
    throw new RuntimeException("getMaxPackedValue for field \"" + fieldName + "\" return length=" + globalMaxPackedValue.length + " array, but should be " + packedBytesCount);
  }
}
 
Example #24
Source File: BinaryVector.java    From semanticvectors with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
public BinaryVector(int dimension) {
  // Check "multiple-of-64" constraint, to facilitate permutation of 64-bit chunks
  if (dimension % 64 != 0) {
    throw new IllegalArgumentException("Dimension should be a multiple of 64: "
        + dimension + " will lead to trouble!");
  }
  this.dimension = dimension;
  this.bitSet = new FixedBitSet(dimension);
  this.isSparse = true;
  this.random = new Random();

}
 
Example #25
Source File: SloppyPhraseMatcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** map each term to the single group that contains it */ 
private HashMap<Term,Integer> termGroups(LinkedHashMap<Term,Integer> tord, ArrayList<FixedBitSet> bb) throws IOException {
  HashMap<Term,Integer> tg = new HashMap<>();
  Term[] t = tord.keySet().toArray(new Term[0]);
  for (int i=0; i<bb.size(); i++) { // i is the group no.
    FixedBitSet bits = bb.get(i);
    for (int ord = bits.nextSetBit(0); ord != DocIdSetIterator.NO_MORE_DOCS; ord = ord + 1 >= bits.length() ? DocIdSetIterator.NO_MORE_DOCS : bits.nextSetBit(ord + 1)) {
      tg.put(t[ord],i);
    }
  }
  return tg;
}
 
Example #26
Source File: SloppyPhraseMatcher.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** bit-sets - for each repeating pp, for each of its repeating terms, the term ordinal values is set */
private ArrayList<FixedBitSet> ppTermsBitSets(PhrasePositions[] rpp, HashMap<Term,Integer> tord) {
  ArrayList<FixedBitSet> bb = new ArrayList<>(rpp.length);
  for (PhrasePositions pp : rpp) {
    FixedBitSet b = new FixedBitSet(tord.size());
    Integer ord;
    for (Term t: pp.terms) {
      if ((ord=tord.get(t))!=null) {
        b.set(ord);
      }
    }
    bb.add(b);
  }
  return bb;
}
 
Example #27
Source File: BitSetPerf.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static void randomSets(int maxSize, int bitsToSet, BitSet target1, FixedBitSet target2) {
  for (int i=0; i<bitsToSet; i++) {
    int idx;
    do {
      idx = rand.nextInt(maxSize);
    } while (target2.getAndSet(idx));
    target1.set(idx);
  }
  /***
   int i=target1.cardinality();
   if (i!=bitsToSet || i!=target2.cardinality()) throw new RuntimeException();
   ***/
}
 
Example #28
Source File: Lucene50LiveDocsFormat.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
  long gen = info.getDelGen();
  String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
  final int length = info.info.maxDoc();
  try (ChecksumIndexInput input = dir.openChecksumInput(name, context)) {
    Throwable priorE = null;
    try {
      CodecUtil.checkIndexHeader(input, CODEC_NAME, VERSION_START, VERSION_CURRENT, 
                                   info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
      long data[] = new long[FixedBitSet.bits2words(length)];
      for (int i = 0; i < data.length; i++) {
        data[i] = input.readLong();
      }
      FixedBitSet fbs = new FixedBitSet(data, length);
      if (fbs.length() - fbs.cardinality() != info.getDelCount()) {
        throw new CorruptIndexException("bits.deleted=" + (fbs.length() - fbs.cardinality()) + 
                                        " info.delcount=" + info.getDelCount(), input);
      }
      return fbs.asReadOnlyBits();
    } catch (Throwable exception) {
      priorE = exception;
    } finally {
      CodecUtil.checkFooter(input, priorE);
    }
  }
  throw new AssertionError();
}
 
Example #29
Source File: BaseCompoundFormatTestCase.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testCheckIntegrity() throws IOException {
  Directory dir = newDirectory();
  String subFile = "_123.xyz";
  SegmentInfo si = newSegmentInfo(dir, "_123");
  try (IndexOutput os = dir.createOutput(subFile, newIOContext(random()))) {
    CodecUtil.writeIndexHeader(os, "Foo", 0, si.getId(), "suffix");
    for (int i = 0; i < 1024; i++) {
      os.writeByte((byte) i);
    }
    os.writeInt(CodecUtil.FOOTER_MAGIC);
    os.writeInt(0);
    long checksum = os.getChecksum();
    os.writeLong(checksum);
  }

  si.setFiles(Collections.singletonList(subFile));
  
  FileTrackingDirectoryWrapper writeTrackingDir = new FileTrackingDirectoryWrapper(dir);
  si.getCodec().compoundFormat().write(writeTrackingDir, si, IOContext.DEFAULT);
  final Set<String> createdFiles = writeTrackingDir.getFiles();

  ReadBytesDirectoryWrapper readTrackingDir = new ReadBytesDirectoryWrapper(dir);
  CompoundDirectory compoundDir = si.getCodec().compoundFormat().getCompoundReader(readTrackingDir, si, IOContext.READ);
  compoundDir.checkIntegrity();
  Map<String,FixedBitSet> readBytes = readTrackingDir.getReadBytes();
  assertEquals(createdFiles, readBytes.keySet());
  for (Map.Entry<String, FixedBitSet> entry : readBytes.entrySet()) {
    final String file = entry.getKey();
    final FixedBitSet set = entry.getValue().clone();
    set.flip(0, set.length());
    final int next = set.nextSetBit(0);
    assertEquals("Byte at offset " + next + " of " + file + " was not read", DocIdSetIterator.NO_MORE_DOCS, next);
  }
  compoundDir.close();
  dir.close();
}
 
Example #30
Source File: BitDocSet.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public int unionSize(DocSet other) {
  if (other instanceof BitDocSet) {
    // if we don't know our current size, this is faster than
    // size + other.size - intersection_size
    return (int) FixedBitSet.unionCount(this.bits, ((BitDocSet) other).bits);
  } else {
    // they had better not call us back!
    return other.unionSize(this);
  }
}