org.apache.lucene.index.MultiDocValues Java Examples

The following examples show how to use org.apache.lucene.index.MultiDocValues. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: HasChildQueryParser.java    From Elasticsearch with Apache License 2.0 6 votes vote down vote up
@Override
public Query rewrite(IndexReader reader) throws IOException {
    if (getBoost() != 1.0F) {
        return super.rewrite(reader);
    }
    if (reader instanceof DirectoryReader) {
        String joinField = ParentFieldMapper.joinField(parentType);
        IndexSearcher indexSearcher = new IndexSearcher(reader);
        indexSearcher.setQueryCache(null);
        indexSearcher.setSimilarity(similarity);
        IndexParentChildFieldData indexParentChildFieldData = parentChildIndexFieldData.loadGlobal((DirectoryReader) reader);
        MultiDocValues.OrdinalMap ordinalMap = ParentChildIndexFieldData.getOrdinalMap(indexParentChildFieldData, parentType);
        return JoinUtil.createJoinQuery(joinField, innerQuery, toQuery, indexSearcher, scoreMode, ordinalMap, minChildren, maxChildren);
    } else {
        if (reader.leaves().isEmpty() && reader.numDocs() == 0) {
            // asserting reader passes down a MultiReader during rewrite which makes this
            // blow up since for this query to work we have to have a DirectoryReader otherwise
            // we can't load global ordinals - for this to work we simply check if the reader has no leaves
            // and rewrite to match nothing
            return new MatchNoDocsQuery();
        }
        throw new IllegalStateException("can't load global ordinals for reader of type: " + reader.getClass() + " must be a DirectoryReader");
    }
}
 
Example #2
Source File: ExpandComponent.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public GroupExpandCollector(SortedDocValues docValues, FixedBitSet groupBits, IntHashSet collapsedSet, int limit, Sort sort) throws IOException {
  int numGroups = collapsedSet.size();
  groups = new LongObjectHashMap<>(numGroups);
  DocIdSetIterator iterator = new BitSetIterator(groupBits, 0); // cost is not useful here
  int group;
  while ((group = iterator.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
    groups.put(group, getCollector(limit, sort));
  }

  this.collapsedSet = collapsedSet;
  this.groupBits = groupBits;
  this.docValues = docValues;
  if(docValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)docValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
}
 
Example #3
Source File: SortedSetDocValuesFacetCounts.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
/** Does all the "real work" of tallying up the counts. */
private final void countAll() throws IOException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  for(LeafReaderContext context : state.getReader().leaves()) {
    countOneSegment(ordinalMap, context.reader(), context.ord, null);
  }
}
 
Example #4
Source File: TestSimilarityProvider.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testBasics() throws Exception {
  // sanity check of norms writer
  // TODO: generalize
  NumericDocValues fooNorms = MultiDocValues.getNormValues(reader, "foo");
  NumericDocValues barNorms = MultiDocValues.getNormValues(reader, "bar");
  for (int i = 0; i < reader.maxDoc(); i++) {
    assertEquals(i, fooNorms.nextDoc());
    assertEquals(i, barNorms.nextDoc());
    assertFalse(fooNorms.longValue() == barNorms.longValue());
  }

  // sanity check of searching
  TopDocs foodocs = searcher.search(new TermQuery(new Term("foo", "brown")), 10);
  assertTrue(foodocs.totalHits.value > 0);
  TopDocs bardocs = searcher.search(new TermQuery(new Term("bar", "brown")), 10);
  assertTrue(bardocs.totalHits.value > 0);
  assertTrue(foodocs.scoreDocs[0].score < bardocs.scoreDocs[0].score);
}
 
Example #5
Source File: DocumentField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
static DocumentField of(FieldInfo finfo, IndexableField field, IndexReader reader, int docId)
    throws IOException {

  Objects.requireNonNull(finfo);
  Objects.requireNonNull(reader);

  DocumentField dfield = new DocumentField();

  dfield.name = finfo.name;
  dfield.idxOptions = finfo.getIndexOptions();
  dfield.hasTermVectors = finfo.hasVectors();
  dfield.hasPayloads = finfo.hasPayloads();
  dfield.hasNorms = finfo.hasNorms();

  if (finfo.hasNorms()) {
    NumericDocValues norms = MultiDocValues.getNormValues(reader, finfo.name);
    if (norms.advanceExact(docId)) {
      dfield.norm = norms.longValue();
    }
  }

  dfield.dvType = finfo.getDocValuesType();

  dfield.pointDimensionCount = finfo.getPointDimensionCount();
  dfield.pointNumBytes = finfo.getPointNumBytes();

  if (field != null) {
    dfield.isStored = field.fieldType().stored();
    dfield.stringValue = field.stringValue();
    if (field.binaryValue() != null) {
      dfield.binaryValue = BytesRef.deepCopyOf(field.binaryValue());
    }
    dfield.numericValue = field.numericValue();
  }

  return dfield;
}
 
Example #6
Source File: StringValue.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public StringValue(SortedDocValues globalDocValues, String field, IntComp comp)  {
  this.globalDocValues = globalDocValues;
  this.docValues = globalDocValues;
  if (globalDocValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.ordinalMap = ((MultiDocValues.MultiSortedDocValues) globalDocValues).mapping;
  }
  this.field = field;
  this.comp = comp;
  this.currentOrd = comp.resetValue();
  this.present = false;
}
 
Example #7
Source File: MinMaxAgg.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, sf, null);
  if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordMap = ((MultiDocValues.MultiSortedSetDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedSetDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 
Example #8
Source File: MinMaxAgg.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null);
  if (topLevel instanceof MultiDocValues.MultiSortedDocValues) {
    ordMap = ((MultiDocValues.MultiSortedDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 
Example #9
Source File: UniqueMultiDvSlotAcc.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void resetIterators() throws IOException {
  topLevel = FieldUtil.getSortedSetDocValues(fcontext.qcontext, field, null);
  nTerms = (int) topLevel.getValueCount();
  if (topLevel instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordMap = ((MultiDocValues.MultiSortedSetDocValues) topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedSetDocValues) topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 
Example #10
Source File: UniqueSinglevaluedSlotAcc.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public void resetIterators() throws IOException {
  super.resetIterators();
  topLevel = FieldUtil.getSortedDocValues(fcontext.qcontext, field, null);
  nTerms = topLevel.getValueCount();
  if (topLevel instanceof MultiDocValues.MultiSortedDocValues) {
    ordMap = ((MultiDocValues.MultiSortedDocValues)topLevel).mapping;
    subDvs = ((MultiDocValues.MultiSortedDocValues)topLevel).values;
  } else {
    ordMap = null;
    subDvs = null;
  }
}
 
Example #11
Source File: TestCollationDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collate(collator, s, startPoint) >= 0 && collate(collator, s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(startPoint + " <= " + s + " <= " + endPoint, collatorAccepts, luceneAccepts);
  }
}
 
Example #12
Source File: TestICUCollationDocValuesField.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void doTestRanges(IndexSearcher is, String startPoint, String endPoint, BytesRef startBR, BytesRef endBR, Collator collator) throws Exception { 
  SortedDocValues dvs = MultiDocValues.getSortedValues(is.getIndexReader(), "collated");
  for(int docID=0;docID<is.getIndexReader().maxDoc();docID++) {
    Document doc = is.doc(docID);
    String s = doc.getField("field").stringValue();
    boolean collatorAccepts = collator.compare(s, startPoint) >= 0 && collator.compare(s, endPoint) <= 0;
    assertEquals(docID, dvs.nextDoc());
    BytesRef br = dvs.binaryValue();
    boolean luceneAccepts = br.compareTo(startBR) >= 0 && br.compareTo(endBR) <= 0;
    assertEquals(collatorAccepts, luceneAccepts);
  }
}
 
Example #13
Source File: TestOrdinalMappingLeafReader.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
private void verifyResults(Directory indexDir, Directory taxoDir) throws IOException {
  DirectoryReader indexReader = DirectoryReader.open(indexDir);
  DirectoryTaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
  IndexSearcher searcher = newSearcher(indexReader);
  
  FacetsCollector collector = new FacetsCollector();
  FacetsCollector.search(searcher, new MatchAllDocsQuery(), 10, collector);

  // tag facets
  Facets tagFacets = new FastTaxonomyFacetCounts("$tags", taxoReader, facetConfig, collector);
  FacetResult result = tagFacets.getTopChildren(10, "tag");
  for (LabelAndValue lv: result.labelValues) {
    if (VERBOSE) {
      System.out.println(lv);
    }
    assertEquals(NUM_DOCS, lv.value.intValue());
  }
  
  // id facets
  Facets idFacets = new FastTaxonomyFacetCounts(taxoReader, facetConfig, collector);
  FacetResult idResult = idFacets.getTopChildren(10, "id");
  assertEquals(NUM_DOCS, idResult.childCount);
  assertEquals(NUM_DOCS * 2, idResult.value); // each "id" appears twice
  
  BinaryDocValues bdv = MultiDocValues.getBinaryValues(indexReader, "bdv");
  BinaryDocValues cbdv = MultiDocValues.getBinaryValues(indexReader, "cbdv");
  for (int i = 0; i < indexReader.maxDoc(); i++) {
    assertEquals(i, bdv.nextDoc());
    assertEquals(i, cbdv.nextDoc());
    assertEquals(Integer.parseInt(cbdv.binaryValue().utf8ToString()), Integer.parseInt(bdv.binaryValue().utf8ToString())*2);
  }
  IOUtils.close(indexReader, taxoReader);
}
 
Example #14
Source File: ConcurrentSortedSetDocValuesFacetCounts.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Does all the "real work" of tallying up the counts. */
private final void countAll() throws IOException, InterruptedException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  List<Future<Void>> results = new ArrayList<>();

  for (LeafReaderContext context : state.getReader().leaves()) {
    results.add(exec.submit(new CountOneSegment(context.reader(), null, ordinalMap, context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 
Example #15
Source File: ConcurrentSortedSetDocValuesFacetCounts.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Does all the "real work" of tallying up the counts. */
private final void count(List<MatchingDocs> matchingDocs) throws IOException, InterruptedException {

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  IndexReader reader = state.getReader();
  List<Future<Void>> results = new ArrayList<>();

  for (MatchingDocs hits : matchingDocs) {
    // LUCENE-5090: make sure the provided reader context "matches"
    // the top-level reader passed to the
    // SortedSetDocValuesReaderState, else cryptic
    // AIOOBE can happen:
    if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) {
      throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
    }
    
    results.add(exec.submit(new CountOneSegment(hits.context.reader(), hits, ordinalMap, hits.context.ord)));
  }

  for (Future<Void> result : results) {
    try {
      result.get();
    } catch (ExecutionException ee) {
      // Theoretically cause can be null; guard against that.
      Throwable cause = ee.getCause();
      throw IOUtils.rethrowAlways(cause != null ? cause : ee);
    }
  }
}
 
Example #16
Source File: SortedSetDocValuesFacetCounts.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Does all the "real work" of tallying up the counts. */
private final void count(List<MatchingDocs> matchingDocs) throws IOException {
  //System.out.println("ssdv count");

  OrdinalMap ordinalMap;

  // TODO: is this right?  really, we need a way to
  // verify that this ordinalMap "matches" the leaves in
  // matchingDocs...
  if (dv instanceof MultiDocValues.MultiSortedSetDocValues && matchingDocs.size() > 1) {
    ordinalMap = ((MultiSortedSetDocValues) dv).mapping;
  } else {
    ordinalMap = null;
  }
  
  IndexReader reader = state.getReader();

  for(MatchingDocs hits : matchingDocs) {

    // LUCENE-5090: make sure the provided reader context "matches"
    // the top-level reader passed to the
    // SortedSetDocValuesReaderState, else cryptic
    // AIOOBE can happen:
    if (ReaderUtil.getTopLevelContext(hits.context).reader() != reader) {
      throw new IllegalStateException("the SortedSetDocValuesReaderState provided to this class does not match the reader being searched; you must create a new SortedSetDocValuesReaderState every time you open a new IndexReader");
    }

    countOneSegment(ordinalMap, hits.context.reader(), hits.context.ord, hits);
  }
}
 
Example #17
Source File: DocumentDictionary.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Creates an iterator over term, weight and payload fields from the lucene
 * index. setting <code>withPayload</code> to false, implies an iterator
 * over only term and weight.
 */
public DocumentInputIterator(boolean hasPayloads, boolean hasContexts) throws IOException {
  this.hasPayloads = hasPayloads;
  this.hasContexts = hasContexts;
  docCount = reader.maxDoc() - 1;
  weightValues = (weightField != null) ? MultiDocValues.getNumericValues(reader, weightField) : null;
  liveDocs = (reader.leaves().size() > 0) ? MultiBits.getLiveDocs(reader) : null;
  relevantFields = getRelevantFields(new String [] {field, weightField, payloadField, contextsField});
}
 
Example #18
Source File: BaseShapeTestCase.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** test random generated lines */
protected void verifyRandomLineQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // line
    Object queryLine = randomQueryLine(shapes);
    Component2D queryLine2D = toLine2D(queryLine);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), POINT_LINE_RELATIONS);
    Query query = newLineQuery(FIELD_NAME, queryRelation, queryLine);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryLine2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryLine);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 
Example #19
Source File: BaseShapeTestCase.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** test random generated polygons */
protected void verifyRandomPolygonQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryPolygon = randomQueryPolygon();
    Component2D queryPoly2D = toPolygon2D(queryPolygon);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newPolygonQuery(FIELD_NAME, queryRelation, queryPolygon);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  queryPolygon=" + queryPolygon);
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 
Example #20
Source File: BaseShapeTestCase.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** test random generated point queries */
protected void verifyRandomPointQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter+1) + " of " + iters + " s=" + s);
    }

    Object[] queryPoints = nextPoints();
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Component2D queryPoly2D;
    Query query;
    if (queryRelation == QueryRelation.CONTAINS) {
      queryPoly2D = toPoint2D(queryPoints[0]);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints[0]);
    } else {
      queryPoly2D = toPoint2D(queryPoints);
      query = newPointsQuery(FIELD_NAME, queryRelation, queryPoints);
    }

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;

      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryPoly2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  rect=Points(" + Arrays.toString(queryPoints) + ")\n");
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 
Example #21
Source File: BaseShapeTestCase.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
/** test random generated circles */
protected void verifyRandomDistanceQueries(IndexReader reader, Object... shapes) throws Exception {
  IndexSearcher s = newSearcher(reader);

  final int iters = scaledIterationCount(shapes.length);

  Bits liveDocs = MultiBits.getLiveDocs(s.getIndexReader());
  int maxDoc = s.getIndexReader().maxDoc();

  for (int iter = 0; iter < iters; ++iter) {
    if (VERBOSE) {
      System.out.println("\nTEST: iter=" + (iter + 1) + " of " + iters + " s=" + s);
    }

    // Polygon
    Object queryCircle = randomQueryCircle();
    Component2D queryCircle2D = toCircle2D(queryCircle);
    QueryRelation queryRelation = RandomPicks.randomFrom(random(), QueryRelation.values());
    Query query = newDistanceQuery(FIELD_NAME, queryRelation, queryCircle);

    if (VERBOSE) {
      System.out.println("  query=" + query + ", relation=" + queryRelation);
    }

    final FixedBitSet hits = new FixedBitSet(maxDoc);
    s.search(query, new SimpleCollector() {

      private int docBase;

      @Override
      public ScoreMode scoreMode() {
        return ScoreMode.COMPLETE_NO_SCORES;
      }

      @Override
      protected void doSetNextReader(LeafReaderContext context) throws IOException {
        docBase = context.docBase;
      }

      @Override
      public void collect(int doc) throws IOException {
        hits.set(docBase+doc);
      }
    });

    boolean fail = false;
    NumericDocValues docIDToID = MultiDocValues.getNumericValues(reader, "id");
    for (int docID = 0; docID < maxDoc; ++docID) {
      assertEquals(docID, docIDToID.nextDoc());
      int id = (int) docIDToID.longValue();
      boolean expected;
      if (liveDocs != null && liveDocs.get(docID) == false) {
        // document is deleted
        expected = false;
      } else if (shapes[id] == null) {
        expected = false;
      } else {
        expected = VALIDATOR.setRelation(queryRelation).testComponentQuery(queryCircle2D, shapes[id]);
      }

      if (hits.get(docID) != expected) {
        StringBuilder b = new StringBuilder();

        if (expected) {
          b.append("FAIL: id=" + id + " should match but did not\n");
        } else {
          b.append("FAIL: id=" + id + " should not match but did\n");
        }
        b.append("  relation=" + queryRelation + "\n");
        b.append("  query=" + query + " docID=" + docID + "\n");
        if (shapes[id] instanceof Object[]) {
          b.append("  shape=" + Arrays.toString((Object[]) shapes[id]) + "\n");
        } else {
          b.append("  shape=" + shapes[id] + "\n");
        }
        b.append("  deleted?=" + (liveDocs != null && liveDocs.get(docID) == false));
        b.append("  distanceQuery=" + queryCircle.toString());
        if (true) {
          fail("wrong hit (first of possibly more):\n\n" + b);
        } else {
          System.out.println(b.toString());
          fail = true;
        }
      }
    }
    if (fail) {
      fail("some hits were wrong");
    }
  }
}
 
Example #22
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public OrdScoreCollector(int maxDoc,
                         int segments,
                         DocValuesProducer collapseValuesProducer,
                         int nullPolicy,
                         IntIntHashMap boostDocsMap,
                         IndexSearcher searcher) throws IOException {
  this.maxDoc = maxDoc;
  this.contexts = new LeafReaderContext[segments];
  List<LeafReaderContext> con = searcher.getTopReaderContext().leaves();
  for(int i=0; i<con.size(); i++) {
    contexts[i] = con.get(i);
  }

  this.collapsedSet = new FixedBitSet(maxDoc);
  this.collapseValuesProducer = collapseValuesProducer;
  this.collapseValues = collapseValuesProducer.getSorted(null);

  int valueCount = collapseValues.getValueCount();
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  this.ords = new IntIntDynamicMap(valueCount, -1);
  this.scores = new IntFloatDynamicMap(valueCount, -Float.MAX_VALUE);
  this.nullPolicy = nullPolicy;
  if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
    nullScores = new FloatArrayList();
  }

  if(boostDocsMap != null) {
    this.boosts = true;
    this.boostOrds = new IntArrayList();
    this.boostDocs = new IntArrayList();
    int[] bd = new int[boostDocsMap.size()];
    Iterator<IntIntCursor> it =  boostDocsMap.iterator();
    int index = -1;
    while(it.hasNext()) {
      IntIntCursor cursor = it.next();
      bd[++index] = cursor.key;
    }

    Arrays.sort(bd);
    this.mergeBoost = new MergeBoost(bd);
  }
}
 
Example #23
Source File: CollapsingQParserPlugin.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
public void finish() throws IOException {
  if(contexts.length == 0) {
    return;
  }

  int currentContext = 0;
  int currentDocBase = 0;

  this.collapseValues = collapseValuesProducer.getSorted(null);
  if(collapseValues instanceof MultiDocValues.MultiSortedDocValues) {
    this.multiSortedDocValues = (MultiDocValues.MultiSortedDocValues)collapseValues;
    this.ordinalMap = multiSortedDocValues.mapping;
  }
  if(ordinalMap != null) {
    this.segmentValues = this.multiSortedDocValues.values[currentContext];
    this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
  } else {
    this.segmentValues = collapseValues;
  }

  int nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
  leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
  ScoreAndDoc dummy = new ScoreAndDoc();
  leafDelegate.setScorer(dummy);
  DocIdSetIterator it = new BitSetIterator(collapseStrategy.getCollapsedSet(), 0); // cost is not useful here
  int globalDoc = -1;
  int nullScoreIndex = 0;
  IntFloatDynamicMap scores = collapseStrategy.getScores();
  FloatArrayList nullScores = collapseStrategy.getNullScores();
  float nullScore = collapseStrategy.getNullScore();

  MergeBoost mergeBoost = collapseStrategy.getMergeBoost();
  while((globalDoc = it.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {


    while(globalDoc >= nextDocBase) {
      currentContext++;
      currentDocBase = contexts[currentContext].docBase;
      nextDocBase = currentContext+1 < contexts.length ? contexts[currentContext+1].docBase : maxDoc;
      leafDelegate = delegate.getLeafCollector(contexts[currentContext]);
      leafDelegate.setScorer(dummy);
      if(ordinalMap != null) {
        this.segmentValues = this.multiSortedDocValues.values[currentContext];
        this.segmentOrdinalMap = this.ordinalMap.getGlobalOrds(currentContext);
      }
    }

    int contextDoc = globalDoc-currentDocBase;

    if(this.needsScores){
      int ord = -1;
      if(this.ordinalMap != null) {
        //Handle ordinalMapping case
        if (segmentValues.advanceExact(contextDoc)) {
          ord = (int) segmentOrdinalMap.get(segmentValues.ordValue());
        }
      } else {
        //Handle top Level FieldCache or Single Segment Case
        if (segmentValues.advanceExact(globalDoc)) {
          ord = segmentValues.ordValue();
        }
      }

      if(ord > -1) {
        dummy.score = scores.get(ord);
      } else if (mergeBoost != null && mergeBoost.boost(globalDoc)) {
        //It's an elevated doc so no score is needed
        dummy.score = 0F;
      } else if (nullPolicy == CollapsingPostFilter.NULL_POLICY_COLLAPSE) {
        dummy.score = nullScore;
      } else if(nullPolicy == CollapsingPostFilter.NULL_POLICY_EXPAND) {
        dummy.score = nullScores.get(nullScoreIndex++);
      }
    }

    dummy.docId = contextDoc;
    leafDelegate.collect(contextDoc);
  }

  if(delegate instanceof DelegatingCollector) {
    ((DelegatingCollector) delegate).finish();
  }
}
 
Example #24
Source File: TestDiversifiedTopDocsCollector.java    From lucene-solr with Apache License 2.0 4 votes vote down vote up
@Override
public void setUp() throws Exception {
  super.setUp();

  // populate an index with documents - artist, song and weeksAtNumberOne
  dir = newDirectory();
  RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
  Document doc = new Document();

  Field yearField = newTextField("year", "", Field.Store.NO);
  SortedDocValuesField artistField = new SortedDocValuesField("artist",
      new BytesRef(""));
  Field weeksAtNumberOneField = new FloatDocValuesField("weeksAtNumberOne",
      0.0F);
  Field weeksStoredField = new StoredField("weeks", 0.0F);
  Field idField = newStringField("id", "", Field.Store.YES);
  Field songField = newTextField("song", "", Field.Store.NO);
  Field storedArtistField = newTextField("artistName", "", Field.Store.NO);

  doc.add(idField);
  doc.add(weeksAtNumberOneField);
  doc.add(storedArtistField);
  doc.add(songField);
  doc.add(weeksStoredField);
  doc.add(yearField);
  doc.add(artistField);

  parsedRecords.clear();
  for (int i = 0; i < hitsOfThe60s.length; i++) {
    String cols[] = hitsOfThe60s[i].split("\t");
    Record record = new Record(String.valueOf(i), cols[0], cols[1], cols[2],
        Float.parseFloat(cols[3]));
    parsedRecords.put(record.id, record);
    idField.setStringValue(record.id);
    yearField.setStringValue(record.year);
    storedArtistField.setStringValue(record.artist);
    artistField.setBytesValue(new BytesRef(record.artist));
    songField.setStringValue(record.song);
    weeksStoredField.setFloatValue(record.weeks);
    weeksAtNumberOneField.setFloatValue(record.weeks);
    writer.addDocument(doc);
    if (i % 10 == 0) {
      // Causes the creation of multiple segments for our test
      writer.commit();
    }
  }
  reader = writer.getReader();
  writer.close();
  searcher = newSearcher(reader);
  artistDocValues = MultiDocValues.getSortedValues(reader, "artist");
}