Java Code Examples for org.apache.lucene.index.DocValues#getSorted()

The following examples show how to use org.apache.lucene.index.DocValues#getSorted() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: FieldFacetStats.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void facetMissingNum(int docID) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }
  
  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  if (docID == topLevelSortedValues.docID()) {
    int ord = topLevelSortedValues.ordValue();
    Integer missingCount = missingStats.get(ord);
    if (missingCount == null) {
      missingStats.put(ord, 1);
    } else {
      missingStats.put(ord, missingCount + 1);
    }
  }
}
 
Example 2
Source File: BooleanField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public void doSetNextReader(LeafReaderContext context) throws IOException {
  docValues = DocValues.getSorted(context.reader(), fieldName);

  // figure out what ord maps to true
  int numOrds = docValues.getValueCount();
  // if no values in the segment, default trueOrd to something other then -1 (missing)
  int trueOrd = -2;
  for (int i=0; i<numOrds; i++) {
    final BytesRef br = docValues.lookupOrd(i);
    if (br.length==1 && br.bytes[br.offset]=='T') {
      trueOrd = i;
      break;
    }
  }

  this.trueOrd = trueOrd;
}
 
Example 3
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }

  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }
  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (foundOrds.get(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  return Explanation.match(score(), "A match, join value " + Term.toString(joinValue));
}
 
Example 4
Source File: GlobalOrdinalsQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = approximationWeight.scorer(context);
  if (approximationScorer == null) {
    return null;
  }
  if (globalOrds != null) {
    return new OrdinalMapScorer(this, score(), foundOrds, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } {
    return new SegmentOrdinalScorer(this, score(), foundOrds, values, approximationScorer.iterator());
  }
}
 
Example 5
Source File: SortField.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/**
 * Returns an {@link IndexSorter} used for sorting index segments by this SortField.
 *
 * If the SortField cannot be used for index sorting (for example, if it uses scores or
 * other query-dependent values) then this method should return {@code null}
 *
 * SortFields that implement this method should also implement a companion
 * {@link SortFieldProvider} to serialize and deserialize the sort in index segment
 * headers
 *
 * @lucene.experimental
 */
public IndexSorter getIndexSorter() {
  switch (type) {
    case STRING:
      return new IndexSorter.StringSorter(Provider.NAME, missingValue, reverse, reader -> DocValues.getSorted(reader, field));
    case INT:
      return new IndexSorter.IntSorter(Provider.NAME, (Integer)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case LONG:
      return new IndexSorter.LongSorter(Provider.NAME, (Long)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case DOUBLE:
      return new IndexSorter.DoubleSorter(Provider.NAME, (Double)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    case FLOAT:
      return new IndexSorter.FloatSorter(Provider.NAME, (Float)missingValue, reverse, reader -> DocValues.getNumeric(reader, field));
    default: return null;
  }
}
 
Example 6
Source File: GlobalOrdinalsWithScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Explanation explain(LeafReaderContext context, int doc) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return Explanation.noMatch("Not a match");
  }
  if (values.advance(doc) != doc) {
    return Explanation.noMatch("Not a match");
  }

  int segmentOrd = values.ordValue();
  BytesRef joinValue = values.lookupOrd(segmentOrd);

  int ord;
  if (globalOrds != null) {
    ord = (int) globalOrds.getGlobalOrds(context.ord).get(segmentOrd);
  } else {
    ord = segmentOrd;
  }
  if (collector.match(ord) == false) {
    return Explanation.noMatch("Not a match, join value " + Term.toString(joinValue));
  }

  float score = collector.score(ord);
  return Explanation.match(score, "A match, join value " + Term.toString(joinValue));
}
 
Example 7
Source File: GlobalOrdinalsWithScoreQuery.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Override
public Scorer scorer(LeafReaderContext context) throws IOException {
  SortedDocValues values = DocValues.getSorted(context.reader(), joinField);
  if (values == null) {
    return null;
  }

  Scorer approximationScorer = in.scorer(context);
  if (approximationScorer == null) {
    return null;
  } else if (globalOrds != null) {
    return new OrdinalMapScorer(this, collector, values, approximationScorer.iterator(), globalOrds.getGlobalOrds(context.ord));
  } else {
    return new SegmentOrdinalScorer(this, collector, values, approximationScorer.iterator());
  }
}
 
Example 8
Source File: TermGroupSelector.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
  this.docValues = DocValues.getSorted(readerContext.reader(), field);
  this.ordsToGroupIds.clear();
  BytesRef scratch = new BytesRef();
  for (int i = 0; i < values.size(); i++) {
    values.get(i, scratch);
    int ord = this.docValues.lookupTerm(scratch);
    if (ord >= 0)
      ordsToGroupIds.put(ord, i);
  }
}
 
Example 9
Source File: StringValue.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void setNextReader(LeafReaderContext context) throws IOException {
  if (ordinalMap != null) {
    toGlobal = ordinalMap.getGlobalOrds(context.ord);
  }
  docValues = DocValues.getSorted(context.reader(), field);
  lastDocID = 0;
}
 
Example 10
Source File: GlobalOrdinalsCollector.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 
Example 11
Source File: GlobalOrdinalsWithScoreCollector.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public LeafCollector getLeafCollector(LeafReaderContext context) throws IOException {
  SortedDocValues docTermOrds = DocValues.getSorted(context.reader(), field);
  if (ordinalMap != null) {
    LongValues segmentOrdToGlobalOrdLookup = ordinalMap.getGlobalOrds(context.ord);
    return new OrdinalMapCollector(docTermOrds, segmentOrdToGlobalOrdLookup);
  } else {
    return new SegmentOrdinalCollector(docTermOrds);
  }
}
 
Example 12
Source File: DocTermsIndexDocValues.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static SortedDocValues open(LeafReaderContext context, String field) throws IOException {
  try {
    return DocValues.getSorted(context.reader(), field);
  } catch (RuntimeException e) {
    throw new DocTermsIndexException(field, e);
  }
}
 
Example 13
Source File: TestJoinUtil.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void testMinMaxDocs() throws Exception {
  Directory dir = newDirectory();
  RandomIndexWriter iw = new RandomIndexWriter(
      random(),
      dir,
      newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.KEYWORD, false))
  );

  int minChildDocsPerParent = 2;
  int maxChildDocsPerParent = 16;
  int numParents = RandomNumbers.randomIntBetween(random(), 16, 64);
  int[] childDocsPerParent = new int[numParents];
  for (int p = 0; p < numParents; p++) {
    String parentId = Integer.toString(p);
    Document parentDoc = new Document();
    parentDoc.add(new StringField("id", parentId, Field.Store.YES));
    parentDoc.add(new StringField("type", "to", Field.Store.NO));
    parentDoc.add(new SortedDocValuesField("join_field", new BytesRef(parentId)));
    iw.addDocument(parentDoc);
    int numChildren = RandomNumbers.randomIntBetween(random(), minChildDocsPerParent, maxChildDocsPerParent);
    childDocsPerParent[p] = numChildren;
    for (int c = 0; c < numChildren; c++) {
      String childId = Integer.toString(p + c);
      Document childDoc = new Document();
      childDoc.add(new StringField("id", childId, Field.Store.YES));
      childDoc.add(new StringField("type", "from", Field.Store.NO));
      childDoc.add(new SortedDocValuesField("join_field", new BytesRef(parentId)));
      iw.addDocument(childDoc);
    }
  }
  iw.close();

  IndexSearcher searcher = new IndexSearcher(DirectoryReader.open(dir));
  SortedDocValues[] values = new SortedDocValues[searcher.getIndexReader().leaves().size()];
  for (LeafReaderContext leadContext : searcher.getIndexReader().leaves()) {
    values[leadContext.ord] = DocValues.getSorted(leadContext.reader(), "join_field");
  }
  OrdinalMap ordinalMap = OrdinalMap.build(
      null, values, PackedInts.DEFAULT
  );
  Query fromQuery = new TermQuery(new Term("type", "from"));
  Query toQuery = new TermQuery(new Term("type", "to"));

  int iters = RandomNumbers.randomIntBetween(random(), 3, 9);
  for (int i = 1; i <= iters; i++) {
    final ScoreMode scoreMode = ScoreMode.values()[random().nextInt(ScoreMode.values().length)];
    int min = RandomNumbers.randomIntBetween(random(), minChildDocsPerParent, maxChildDocsPerParent - 1);
    int max = RandomNumbers.randomIntBetween(random(), min, maxChildDocsPerParent);
    if (VERBOSE) {
      System.out.println("iter=" + i);
      System.out.println("scoreMode=" + scoreMode);
      System.out.println("min=" + min);
      System.out.println("max=" + max);
    }
    Query joinQuery = JoinUtil.createJoinQuery("join_field", fromQuery, toQuery, searcher, scoreMode, ordinalMap, min, max);
    TotalHitCountCollector collector = new TotalHitCountCollector();
    searcher.search(joinQuery, collector);
    int expectedCount = 0;
    for (int numChildDocs : childDocsPerParent) {
      if (numChildDocs >= min && numChildDocs <= max) {
        expectedCount++;
      }
    }
    assertEquals(expectedCount, collector.getTotalHits());
  }

  searcher.getIndexReader().close();
  dir.close();
}
 
Example 14
Source File: FieldComparator.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** Retrieves the SortedDocValues for the field in this segment */
protected SortedDocValues getSortedDocValues(LeafReaderContext context, String field) throws IOException {
  return DocValues.getSorted(context.reader(), field);
}
 
Example 15
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public SortedDocValues getSorted(FieldInfo ignored) throws IOException {
  return DocValues.getSorted(searcher.getSlowAtomicReader(), collapseField);
}
 
Example 16
Source File: DocValuesAcc.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void setNextReader(LeafReaderContext readerContext) throws IOException {
  super.setNextReader(readerContext);
  values = DocValues.getSorted(readerContext.reader(), sf.getName());
}
 
Example 17
Source File: FieldFacetStats.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public boolean facetTermNum(int docID, int statsTermNum) throws IOException {
  if (topLevelSortedValues == null) {
    topLevelSortedValues = DocValues.getSorted(topLevelReader, name);
  }

  if (docID > topLevelSortedValues.docID()) {
    topLevelSortedValues.advance(docID);
  }
 
  int term;
  if (docID == topLevelSortedValues.docID()) {
    term = topLevelSortedValues.ordValue();
  } else {
    term = -1;
  }
  
  int arrIdx = term;
  if (arrIdx >= 0 && arrIdx < topLevelSortedValues.getValueCount()) {
    final String key;
    if (term == -1) {
      key = null;
    } else {
      key = topLevelSortedValues.lookupOrd(term).utf8ToString();
    }
    while (facetStatsTerms.size() <= statsTermNum) {
      facetStatsTerms.add(new HashMap<String, Integer>());
    }
    
    
    final Map<String, Integer> statsTermCounts = facetStatsTerms.get(statsTermNum);
    Integer statsTermCount = statsTermCounts.get(key);
    if (statsTermCount == null) {
      statsTermCounts.put(key, 1);
    } else {
      statsTermCounts.put(key, statsTermCount + 1);
    }
    return true;
  }
  
  return false;
}
 
Example 18
Source File: TermGroupFacetCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
protected void doSetNextReader(LeafReaderContext context) throws IOException {
  if (segmentFacetCounts != null) {
    segmentResults.add(createSegmentResult());
  }

  groupFieldTermsIndex = DocValues.getSorted(context.reader(), groupField);
  facetFieldDocTermOrds = DocValues.getSortedSet(context.reader(), facetField);
  facetFieldNumTerms = (int) facetFieldDocTermOrds.getValueCount();
  if (facetFieldNumTerms == 0) {
    facetOrdTermsEnum = null;
  } else {
    facetOrdTermsEnum = facetFieldDocTermOrds.termsEnum();
  }
  // [facetFieldNumTerms() + 1] for all possible facet values and docs not containing facet field
  segmentFacetCounts = new int[facetFieldNumTerms + 1];
  segmentTotalCount = 0;

  segmentGroupedFacetHits.clear();
  for (GroupedFacetHit groupedFacetHit : groupedFacetHits) {
    int groupOrd = groupedFacetHit.groupValue == null ? -1 : groupFieldTermsIndex.lookupTerm(groupedFacetHit.groupValue);
    if (groupedFacetHit.groupValue != null && groupOrd < 0) {
      continue;
    }

    int facetOrd;
    if (groupedFacetHit.facetValue != null) {
      if (facetOrdTermsEnum == null || !facetOrdTermsEnum.seekExact(groupedFacetHit.facetValue)) {
        continue;
      }
      facetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      facetOrd = facetFieldNumTerms;
    }

    // (facetFieldDocTermOrds.numTerms() + 1) for all possible facet values and docs not containing facet field
    int segmentGroupedFacetsIndex = groupOrd * (facetFieldNumTerms + 1) + facetOrd;
    segmentGroupedFacetHits.put(segmentGroupedFacetsIndex);
  }

  if (facetPrefix != null) {
    TermsEnum.SeekStatus seekStatus;
    if (facetOrdTermsEnum != null) {
      seekStatus = facetOrdTermsEnum.seekCeil(facetPrefix);
    } else {
      seekStatus = TermsEnum.SeekStatus.END;
    }

    if (seekStatus != TermsEnum.SeekStatus.END) {
      startFacetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      startFacetOrd = 0;
      endFacetOrd = 0;
      return;
    }

    BytesRefBuilder facetEndPrefix = new BytesRefBuilder();
    facetEndPrefix.append(facetPrefix);
    facetEndPrefix.append(UnicodeUtil.BIG_TERM);
    seekStatus = facetOrdTermsEnum.seekCeil(facetEndPrefix.get());
    if (seekStatus != TermsEnum.SeekStatus.END) {
      endFacetOrd = (int) facetOrdTermsEnum.ord();
    } else {
      endFacetOrd = facetFieldNumTerms; // Don't include null...
    }
  } else {
    startFacetOrd = 0;
    endFacetOrd = facetFieldNumTerms + 1;
  }
}