org.apache.lucene.util.LongBitSet Java Examples

The following examples show how to use org.apache.lucene.util.LongBitSet. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: TopLevelJoinQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
private BitsetBounds convertFromOrdinalsIntoToField(LongBitSet fromOrdBitSet, SortedSetDocValues fromDocValues,
                                                    LongBitSet toOrdBitSet, SortedSetDocValues toDocValues) throws IOException {
  long fromOrdinal = 0;
  long firstToOrd = BitsetBounds.NO_MATCHES;
  long lastToOrd = 0;

  while (fromOrdinal < fromOrdBitSet.length() && (fromOrdinal = fromOrdBitSet.nextSetBit(fromOrdinal)) >= 0) {
    final BytesRef fromBytesRef = fromDocValues.lookupOrd(fromOrdinal);
    final long toOrdinal = lookupTerm(toDocValues, fromBytesRef, lastToOrd);
    if (toOrdinal >= 0) {
      toOrdBitSet.set(toOrdinal);
      if (firstToOrd == BitsetBounds.NO_MATCHES) firstToOrd = toOrdinal;
      lastToOrd = toOrdinal;
    }
    fromOrdinal++;
  }

    return new BitsetBounds(firstToOrd, lastToOrd);
}
 
Example #2
Source File: GlobalOrdinalsWithScoreCollector.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
GlobalOrdinalsWithScoreCollector(String field, OrdinalMap ordinalMap, long valueCount, ScoreMode scoreMode, int min, int max) {
  if (valueCount > Integer.MAX_VALUE) {
    // We simply don't support more than
    throw new IllegalStateException("Can't collect more than [" + Integer.MAX_VALUE + "] ids");
  }
  this.field = field;
  this.doMinMax = !(min <= 0 && max == Integer.MAX_VALUE);
  this.min = min;
  this.max = max;;
  this.ordinalMap = ordinalMap;
  this.collectedOrds = new LongBitSet(valueCount);
  if (scoreMode != ScoreMode.None) {
    this.scores = new Scores(valueCount, unset());
  } else {
    this.scores = null;
  }
  if (scoreMode == ScoreMode.Avg || doMinMax) {
    this.occurrences = new Occurrences(valueCount);
  } else {
    this.occurrences = null;
  }
}
 
Example #3
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
GlobalOrdinalsQuery(LongBitSet foundOrds, String joinField, OrdinalMap globalOrds, Query toQuery,
                    Query fromQuery, Object indexReaderContextId) {
  this.foundOrds = foundOrds;
  this.joinField = joinField;
  this.globalOrds = globalOrds;
  this.toQuery = toQuery;
  this.fromQuery = fromQuery;
  this.indexReaderContextId = indexReaderContextId;

  this.ramBytesUsed = BASE_RAM_BYTES +
      RamUsageEstimator.sizeOfObject(this.foundOrds) +
      RamUsageEstimator.sizeOfObject(this.globalOrds) +
      RamUsageEstimator.sizeOfObject(this.joinField) +
      RamUsageEstimator.sizeOfObject(this.fromQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED) +
      RamUsageEstimator.sizeOfObject(this.toQuery, RamUsageEstimator.QUERY_DEFAULT_RAM_BYTES_USED);
}
 
Example #4
Source File: TestLegacyNumericUtils.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Note: The neededBounds Iterable must be unsigned (easier understanding what's happening) */
private void assertLongRangeSplit(final long lower, final long upper, int precisionStep,
  final boolean useBitSet, final Iterable<Long> expectedBounds, final Iterable<Integer> expectedShifts
) {
  // Cannot use FixedBitSet since the range could be long:
  final LongBitSet bits=useBitSet ? new LongBitSet(upper-lower+1) : null;
  final Iterator<Long> neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator();
  final Iterator<Integer> neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator();

  LegacyNumericUtils.splitLongRange(new LegacyNumericUtils.LongRangeBuilder() {
    @Override
    public void addRange(long min, long max, int shift) {
      assertTrue("min, max should be inside bounds", min >= lower && min <= upper && max >= lower && max <= upper);
      if (useBitSet) for (long l = min; l <= max; l++) {
        assertFalse("ranges should not overlap", bits.getAndSet(l - lower));
        // extra exit condition to prevent overflow on MAX_VALUE
        if (l == max) break;
      }
      if (neededBounds == null || neededShifts == null)
        return;
      // make unsigned longs for easier display and understanding
      min ^= 0x8000000000000000L;
      max ^= 0x8000000000000000L;
      //System.out.println("0x"+Long.toHexString(min>>>shift)+"L,0x"+Long.toHexString(max>>>shift)+"L)/*shift="+shift+"*/,");
      assertEquals("shift", neededShifts.next().intValue(), shift);
      assertEquals("inner min bound", neededBounds.next().longValue(), min >>> shift);
      assertEquals("inner max bound", neededBounds.next().longValue(), max >>> shift);
    }
  }, precisionStep, lower, upper);
  
  if (useBitSet) {
    // after flipping all bits in the range, the cardinality should be zero
    bits.flip(0,upper-lower+1);
    assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality());
  }
}
 
Example #5
Source File: ChildrenConstantScoreQuery.java    From Elasticsearch with Apache License 2.0 5 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
    if (remaining == 0) {
        return null;
    }

    if (shortCircuitFilter != null) {
        DocIdSet docIdSet = shortCircuitFilter.getDocIdSet(context, null);
        if (!Lucene.isEmpty(docIdSet)) {
            DocIdSetIterator iterator = docIdSet.iterator();
            if (iterator != null) {
                return ConstantScorer.create(iterator, this, queryWeight);
            }
        }
        return null;
    }

    DocIdSet parentDocIdSet = this.parentFilter.getDocIdSet(context, null);
    if (!Lucene.isEmpty(parentDocIdSet)) {
        // We can't be sure of the fact that liveDocs have been applied, so we apply it here. The "remaining"
        // count down (short circuit) logic will then work as expected.
        parentDocIdSet = BitsFilteredDocIdSet.wrap(parentDocIdSet, context.reader().getLiveDocs());
        DocIdSetIterator innerIterator = parentDocIdSet.iterator();
        if (innerIterator != null) {
            LongBitSet parentOrds = collector.parentOrds;
            SortedDocValues globalValues = globalIfd.load(context).getOrdinalsValues(parentType);
            if (globalValues != null) {
                DocIdSetIterator parentIdIterator = new ParentOrdIterator(innerIterator, parentOrds, globalValues, this);
                return ConstantScorer.create(parentIdIterator, this, queryWeight);
            }
        }
    }
    return null;
}
 
Example #6
Source File: TopLevelJoinQuery.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private static LongBitSet findFieldOrdinalsMatchingQuery(Query q, String field, SolrIndexSearcher searcher, SortedSetDocValues docValues) throws IOException {
  final LongBitSet fromOrdBitSet = new LongBitSet(docValues.getValueCount());
  final Collector fromCollector = new MultiValueTermOrdinalCollector(field, docValues, fromOrdBitSet);

  searcher.search(q, fromCollector);

  return fromOrdBitSet;
}
 
Example #7
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public OrdinalMapScorer(Weight weight, float score, LongBitSet foundOrds, SortedDocValues values, DocIdSetIterator approximationScorer, LongValues segmentOrdToGlobalOrdLookup) {
  super(weight, values, approximationScorer);
  this.score = score;
  this.foundOrds = foundOrds;
  this.segmentOrdToGlobalOrdLookup = segmentOrdToGlobalOrdLookup;
}
 
Example #8
Source File: BlockLocks.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public BlockLocks(long numBits) {
  int length = LongBitSet.bits2words(numBits);
  bits = new AtomicLongArray(length);
  wlen = length;
}
 
Example #9
Source File: MultiValueTermOrdinalCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public MultiValueTermOrdinalCollector(String fieldName, SortedSetDocValues topLevelDocValues, LongBitSet topLevelDocValuesBitSet) {
  this.fieldName = fieldName;
  this.topLevelDocValues = topLevelDocValues;
  this.topLevelDocValuesBitSet = topLevelDocValuesBitSet;
}
 
Example #10
Source File: TopLevelJoinQuery.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  if (! (searcher instanceof SolrIndexSearcher)) {
    log.debug("Falling back to JoinQueryWeight because searcher [{}] is not the required SolrIndexSearcher", searcher);
    return super.createWeight(searcher, scoreMode, boost);
  }

  final SolrIndexSearcher solrSearcher = (SolrIndexSearcher) searcher;
  final JoinQueryWeight weight = new JoinQueryWeight(solrSearcher, ScoreMode.COMPLETE_NO_SCORES, 1.0f);
  final SolrIndexSearcher fromSearcher = weight.fromSearcher;
  final SolrIndexSearcher toSearcher = weight.toSearcher;

  try {
    final SortedSetDocValues topLevelFromDocValues = validateAndFetchDocValues(fromSearcher, fromField, "from");
    final SortedSetDocValues topLevelToDocValues = validateAndFetchDocValues(toSearcher, toField, "to");
    if (topLevelFromDocValues.getValueCount() == 0 || topLevelToDocValues.getValueCount() == 0) {
      return createNoMatchesWeight(boost);
    }

    final LongBitSet fromOrdBitSet = findFieldOrdinalsMatchingQuery(q, fromField, fromSearcher, topLevelFromDocValues);
    final LongBitSet toOrdBitSet = new LongBitSet(topLevelToDocValues.getValueCount());
    final BitsetBounds toBitsetBounds = convertFromOrdinalsIntoToField(fromOrdBitSet, topLevelFromDocValues, toOrdBitSet, topLevelToDocValues);

    final boolean toMultivalued = toSearcher.getSchema().getFieldOrNull(toField).multiValued();
    return new ConstantScoreWeight(this, boost) {
      public Scorer scorer(LeafReaderContext context) throws IOException {
        if (toBitsetBounds.lower == BitsetBounds.NO_MATCHES) {
          return null;
        }

        final DocIdSetIterator toApproximation = (toMultivalued) ? context.reader().getSortedSetDocValues(toField) :
            context.reader().getSortedDocValues(toField);
        if (toApproximation == null) {
          return null;
        }

        final int docBase = context.docBase;
        return new ConstantScoreScorer(this, this.score(), scoreMode, new TwoPhaseIterator(toApproximation) {
          public boolean matches() throws IOException {
            final boolean hasDoc = topLevelToDocValues.advanceExact(docBase + approximation.docID());
            if (hasDoc) {
              for (long ord = topLevelToDocValues.nextOrd(); ord != -1L; ord = topLevelToDocValues.nextOrd()) {
                if (toOrdBitSet.get(ord)) {
                  return true;
                }
              }
            }
            return false;
          }

          public float matchCost() {
            return 10.0F;
          }
        });

      }

      public boolean isCacheable(LeafReaderContext ctx) {
        return false;
      }
    };
  } catch (IOException e) {
    throw new RuntimeException(e);
  }
}
 
Example #11
Source File: TermsQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public Weight createWeight(IndexSearcher searcher, final ScoreMode scoreMode, float boost) throws IOException {
  if (! (searcher instanceof SolrIndexSearcher)) {
    log.debug("Falling back to DocValuesTermsQuery because searcher [{}] is not the required SolrIndexSearcher", searcher);
    return super.createWeight(searcher, scoreMode, boost);
  }

  topLevelDocValues = DocValues.getSortedSet(((SolrIndexSearcher)searcher).getSlowAtomicReader(), fieldName);
  topLevelTermOrdinals = new LongBitSet(topLevelDocValues.getValueCount());
  PrefixCodedTerms.TermIterator iterator = getTerms().iterator();

  long lastTermOrdFound = 0;
  for(BytesRef term = iterator.next(); term != null; term = iterator.next()) {
    long currentTermOrd = lookupTerm(topLevelDocValues, term, lastTermOrdFound);
    if (currentTermOrd >= 0L) {
      matchesAtLeastOneTerm = true;
      topLevelTermOrdinals.set(currentTermOrd);
      lastTermOrdFound = currentTermOrd;
    }
  }

  return new ConstantScoreWeight(this, boost) {
    public Scorer scorer(LeafReaderContext context) throws IOException {
      if (! matchesAtLeastOneTerm) {
        return null;
      }
      
      SortedSetDocValues segmentDocValues = context.reader().getSortedSetDocValues(fieldName);
      if (segmentDocValues == null) {
        return null;
      }

      final int docBase = context.docBase;
      return new ConstantScoreScorer(this, this.score(), scoreMode, new TwoPhaseIterator(segmentDocValues) {
        public boolean matches() throws IOException {
          topLevelDocValues.advanceExact(docBase + approximation.docID());
          for(long ord = topLevelDocValues.nextOrd(); ord != -1L; ord = topLevelDocValues.nextOrd()) {
            if (topLevelTermOrdinals.get(ord)) {
              return true;
            }
          }

          return false;
        }

        public float matchCost() {
          return 10.0F;
        }
      });

    }

    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, new String[]{fieldName});
    }
  };
}
 
Example #12
Source File: DocValuesConsumer.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
BitsFilteredTermsEnum(TermsEnum in, LongBitSet liveTerms) {
  super(in, false); // <-- not passing false here wasted about 3 hours of my time!!!!!!!!!!!!!
  assert liveTerms != null;
  this.liveTerms = liveTerms;
}
 
Example #13
Source File: DocValuesTermsQuery.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
  return new ConstantScoreWeight(this, boost) {

    @Override
    public Scorer scorer(LeafReaderContext context) throws IOException {
      final SortedSetDocValues values = DocValues.getSortedSet(context.reader(), field);
      final LongBitSet bits = new LongBitSet(values.getValueCount());
      boolean matchesAtLeastOneTerm = false;
      TermIterator iterator = termData.iterator();
      for (BytesRef term = iterator.next(); term != null; term = iterator.next()) {
        final long ord = values.lookupTerm(term);
        if (ord >= 0) {
          matchesAtLeastOneTerm = true;
          bits.set(ord);
        }
      }
      if (matchesAtLeastOneTerm == false) {
        return null;
      }
      return new ConstantScoreScorer(this, score(), scoreMode, new TwoPhaseIterator(values) {

        @Override
        public boolean matches() throws IOException {
          for (long ord = values.nextOrd(); ord != SortedSetDocValues.NO_MORE_ORDS; ord = values.nextOrd()) {
            if (bits.get(ord)) {
              return true;
            }
          }
          return false;
        }

        @Override
        public float matchCost() {
          return 3; // lookup in a bitset
        }

      });
    }

    @Override
    public boolean isCacheable(LeafReaderContext ctx) {
      return DocValues.isCacheable(ctx, field);
    }

  };
}
 
Example #14
Source File: GlobalOrdinalsCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public LongBitSet getCollectorOrdinals() {
  return collectedOrds;
}
 
Example #15
Source File: GlobalOrdinalsCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
GlobalOrdinalsCollector(String field, OrdinalMap ordinalMap, long valueCount) {
  this.field = field;
  this.ordinalMap = ordinalMap;
  this.collectedOrds = new LongBitSet(valueCount);
}
 
Example #16
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public SegmentOrdinalScorer(Weight weight, float score, LongBitSet foundOrds, SortedDocValues values, DocIdSetIterator approximationScorer) {
  super(weight, values, approximationScorer);
  this.score = score;
  this.foundOrds = foundOrds;
}
 
Example #17
Source File: HyperLogLogPlusPlus.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
void ensureCapacity(long bit) {
    impl = LongBitSet.ensureCapacity(impl, bit);
}
 
Example #18
Source File: SolrOpenBitSetAdapter.java    From SearchServices with GNU Lesser General Public License v3.0 4 votes vote down vote up
@Override
public void set(long index) 
{
	delegate = LongBitSet.ensureCapacity(delegate, index);
	delegate.set(index);
}
 
Example #19
Source File: SolrOpenBitSetAdapter.java    From SearchServices with GNU Lesser General Public License v3.0 4 votes vote down vote up
public SolrOpenBitSetAdapter()
{
	delegate = new LongBitSet(64);
}
 
Example #20
Source File: ChildrenConstantScoreQuery.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
private ParentOrdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, SortedDocValues ordinals, ParentWeight parentWeight) {
    super(innerIterator);
    this.parentOrds = parentOrds;
    this.ordinals = ordinals;
    this.parentWeight = parentWeight;
}
 
Example #21
Source File: ChildrenConstantScoreQuery.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
private ParentOrdCollector(IndexParentChildFieldData indexFieldData, long maxOrd, String parentType) {
    // TODO: look into reusing LongBitSet#bits array
    this.parentOrds = new LongBitSet(maxOrd + 1);
    this.indexFieldData = indexFieldData;
    this.parentType = parentType;
}
 
Example #22
Source File: ParentConstantScoreQuery.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
ParentOrdsCollector(IndexParentChildFieldData globalIfd, long maxOrd, String parentType) {
    this.parentOrds = new LongBitSet(maxOrd);
    this.globalIfd = globalIfd;
    this.parentType = parentType;
}
 
Example #23
Source File: ParentConstantScoreQuery.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
ChildrenDocIdIterator(DocIdSetIterator innerIterator, LongBitSet parentOrds, SortedDocValues globalOrdinals) {
    super(innerIterator);
    this.parentOrds = parentOrds;
    this.globalOrdinals = globalOrdinals;
}
 
Example #24
Source File: GlobalOrdinalsStringTermsAggregator.java    From Elasticsearch with Apache License 2.0 4 votes vote down vote up
private FilteredOrdinals(RandomAccessOrds inner, LongBitSet accepted) {
    this.inner = inner;
    this.accepted = accepted;
}
 
Example #25
Source File: IncludeExclude.java    From Elasticsearch with Apache License 2.0 votes vote down vote up
public abstract LongBitSet acceptedGlobalOrdinals(RandomAccessOrds globalOrdinals, ValuesSource.Bytes.WithOrdinals valueSource) throws IOException;