/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.monitor;
import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.complexPhrase.ComplexPhraseQueryParser;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.RegexpQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
public class TestHighlightingMatcher extends MonitorTestBase {
private static final Analyzer WHITESPACE = new WhitespaceAnalyzer();
public static Document buildDoc(String text) {
Document doc = new Document();
doc.add(newTextField(FIELD, text, Field.Store.NO));
return doc;
}
public void testSingleTermQueryMatchesSingleDocument() throws IOException {
try (Monitor monitor = newMonitor()) {
MonitorQuery mq = new MonitorQuery("query1", parse("test"));
monitor.register(mq);
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("this is a test document"),
HighlightsMatch.MATCHER);
assertEquals(1, matches.getMatchCount());
HighlightsMatch match = matches.matches("query1");
assertTrue(match.getHits(FIELD).contains(new HighlightsMatch.Hit(3, 10, 3, 14)));
}
}
public void testSinglePhraseQueryMatchesSingleDocument() throws IOException {
try (Monitor monitor = newMonitor()) {
MonitorQuery mq = new MonitorQuery("query1", parse("\"test document\""));
monitor.register(mq);
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("this is a test document"),
HighlightsMatch.MATCHER);
assertEquals(1, matches.getMatchCount());
HighlightsMatch m = matches.matches("query1");
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(3, 10, 4, 23)));
}
}
public void testToString() {
HighlightsMatch match = new HighlightsMatch("1");
match.addHit("field", 2, 3, -1, -1);
match.addHit("field", 0, 1, -1, -1);
match.addHit("afield", 0, 1, 0, 4);
assertEquals("Match(query=1){hits={afield=[0(0)->1(4)], field=[0(-1)->1(-1), 2(-1)->3(-1)]}}", match.toString());
}
public void testMultiFieldQueryMatches() throws IOException {
Document doc = new Document();
doc.add(newTextField("field1", "this is a test of field one", Field.Store.NO));
doc.add(newTextField("field2", "and this is an additional test", Field.Store.NO));
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("query1", parse("field1:test field2:test")));
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
assertEquals(1, matches.getMatchCount());
HighlightsMatch m = matches.matches("query1");
assertNotNull(m);
assertTrue(m.getFields().contains("field1"));
assertTrue(m.getHits("field1").contains(new HighlightsMatch.Hit(3, 10, 3, 14)));
assertTrue(m.getHits("field2").contains(new HighlightsMatch.Hit(5, 26, 5, 30)));
}
}
public void testQueryErrors() throws IOException {
try (Monitor monitor = new Monitor(ANALYZER, Presearcher.NO_FILTERING)) {
monitor.register(new MonitorQuery("1", parse("test")),
new MonitorQuery("2", new ThrowOnRewriteQuery()),
new MonitorQuery("3", parse("document")),
new MonitorQuery("4", parse("foo")));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("this is a test document"), HighlightsMatch.MATCHER);
assertEquals(4, matches.getQueriesRun());
assertEquals(2, matches.getMatchCount());
assertEquals(1, matches.getErrors().size());
}
}
public void testWildcards() throws IOException {
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", new RegexpQuery(new Term(FIELD, "he.*"))));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("hello world"), HighlightsMatch.MATCHER);
assertEquals(1, matches.getQueriesRun());
assertEquals(1, matches.getMatchCount());
assertEquals(1, matches.matches("1").getHitCount());
}
}
public void testWildcardCombinations() throws Exception {
final BooleanQuery bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "term1")), BooleanClause.Occur.MUST)
.add(new PrefixQuery(new Term(FIELD, "term2")), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(FIELD, "term3")), BooleanClause.Occur.MUST_NOT)
.build();
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", bq));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term1 term22 term4"), HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(2, m.getHitCount());
}
}
public void testDisjunctionMaxQuery() throws IOException {
final DisjunctionMaxQuery query = new DisjunctionMaxQuery(Arrays.asList(
new TermQuery(new Term(FIELD, "term1")), new PrefixQuery(new Term(FIELD, "term2"))
), 1.0f);
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", query));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term1 term2 term3"), HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(2, m.getHitCount());
}
}
public void testIdenticalMatches() throws Exception {
final BooleanQuery bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "term1")), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(FIELD, "term1")), BooleanClause.Occur.SHOULD)
.build();
try (Monitor monitor = new Monitor(ANALYZER)) {
monitor.register(new MonitorQuery("1", bq));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term1 term2"), HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(1, m.getHitCount());
}
}
public void testWildcardBooleanRewrites() throws Exception {
final Query wc = new PrefixQuery(new Term(FIELD, "term1"));
final Query wrapper = new BooleanQuery.Builder()
.add(wc, BooleanClause.Occur.MUST)
.build();
final Query wrapper2 = new BooleanQuery.Builder()
.add(wrapper, BooleanClause.Occur.MUST)
.build();
final BooleanQuery bq = new BooleanQuery.Builder()
.add(new PrefixQuery(new Term(FIELD, "term2")), BooleanClause.Occur.MUST)
.add(wrapper2, BooleanClause.Occur.MUST_NOT)
.build();
try (Monitor monitor = new Monitor(ANALYZER)) {
monitor.register(new MonitorQuery("1", bq));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term2 term"), HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(1, m.getHitCount());
matches = monitor.match(buildDoc("term2 term"), HighlightsMatch.MATCHER);
m = matches.matches("1");
assertNotNull(m);
assertEquals(1, m.getHitCount());
}
}
public void testWildcardProximityRewrites() throws Exception {
final SpanNearQuery snq = SpanNearQuery.newOrderedNearQuery(FIELD)
.addClause(new SpanMultiTermQueryWrapper<>(new WildcardQuery(new Term(FIELD, "term*"))))
.addClause(new SpanTermQuery(new Term(FIELD, "foo")))
.build();
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", snq));
MatchingQueries<HighlightsMatch> matches = monitor.match(buildDoc("term1 foo"), HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(2, m.getHitCount());
}
}
public void testDisjunctionWithOrderedNearSpans() throws Exception {
final Query bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.SHOULD)
.add(SpanNearQuery.newOrderedNearQuery(FIELD)
.addClause(new SpanTermQuery(new Term(FIELD, "b")))
.addClause(new SpanTermQuery(new Term(FIELD, "c")))
.setSlop(1)
.build(), BooleanClause.Occur.SHOULD)
.build();
final Query parent = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.MUST)
.add(bq, BooleanClause.Occur.MUST)
.build();
try (Monitor monitor = new Monitor(ANALYZER)) {
monitor.register(new MonitorQuery("1", parent));
Document doc = buildDoc("a b x x x x c");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(1, m.getHitCount());
}
}
public void testDisjunctionWithUnorderedNearSpans() throws Exception {
final Query bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.SHOULD)
.add(SpanNearQuery.newUnorderedNearQuery(FIELD)
.addClause(new SpanTermQuery(new Term(FIELD, "b")))
.addClause(new SpanTermQuery(new Term(FIELD, "c")))
.setSlop(1)
.build(), BooleanClause.Occur.SHOULD)
.build();
final Query parent = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.MUST)
.add(bq, BooleanClause.Occur.MUST)
.build();
try (Monitor monitor = new Monitor(ANALYZER)) {
monitor.register(new MonitorQuery("1", parent));
Document doc = buildDoc("a b x x x x c");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(1, m.getHitCount());
}
}
public void testEquality() {
HighlightsMatch m1 = new HighlightsMatch("1");
m1.addHit("field", 0, 1, 0, 1);
HighlightsMatch m2 = new HighlightsMatch("1");
m2.addHit("field", 0, 1, 0, 1);
HighlightsMatch m3 = new HighlightsMatch("1");
m3.addHit("field", 0, 2, 0, 1);
HighlightsMatch m4 = new HighlightsMatch("2");
m4.addHit("field", 0, 1, 0, 1);
assertEquals(m1, m2);
assertEquals(m1.hashCode(), m2.hashCode());
assertNotEquals(m1, m3);
assertNotEquals(m1, m4);
}
public void testMutliValuedFieldWithNonDefaultGaps() throws IOException {
Analyzer analyzer = new Analyzer() {
@Override
public int getPositionIncrementGap(String fieldName) {
return 1000;
}
@Override
public int getOffsetGap(String fieldName) {
return 2000;
}
@Override
protected TokenStreamComponents createComponents(String fieldName) {
return new TokenStreamComponents(new WhitespaceTokenizer());
}
};
MonitorQuery mq = new MonitorQuery("query", parse(FIELD + ":\"hello world\"~5"));
try (Monitor monitor = newMonitor(analyzer)) {
monitor.register(mq);
Document doc1 = new Document();
doc1.add(newTextField(FIELD, "hello world", Field.Store.NO));
doc1.add(newTextField(FIELD, "goodbye", Field.Store.NO));
MatchingQueries<HighlightsMatch> matcher1 = monitor.match(doc1, HighlightsMatch.MATCHER);
assertEquals(1, matcher1.getMatchCount());
HighlightsMatch m1 = matcher1.matches("query");
assertNotNull(m1);
assertTrue(m1.getFields().contains(FIELD));
assertTrue(m1.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 1, 11)));
Document doc2 = new Document();
doc1.add(newTextField(FIELD, "hello", Field.Store.NO));
doc1.add(newTextField(FIELD, "world", Field.Store.NO));
MatchingQueries<HighlightsMatch> matcher2 = monitor.match(doc2, HighlightsMatch.MATCHER);
assertNull(matcher2.matches("query"));
assertEquals(0, matcher2.getMatchCount());
Document doc3 = new Document();
doc3.add(newTextField(FIELD, "hello world", Field.Store.NO));
doc3.add(newTextField(FIELD, "hello goodbye world", Field.Store.NO));
MatchingQueries<HighlightsMatch> matcher3 = monitor.match(doc3, HighlightsMatch.MATCHER);
assertEquals(1, matcher3.getMatchCount());
HighlightsMatch m3 = matcher3.matches("query");
assertNotNull(m3);
assertTrue(m3.getFields().contains(FIELD));
assertTrue(m3.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 1, 11)));
assertTrue(m3.getHits(FIELD).contains(new HighlightsMatch.Hit(1002, 2011, 1004, 2030)));
}
}
public void testDisjunctionWithOrderedNearMatch() throws Exception {
final Query bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.SHOULD)
.add(SpanNearQuery.newOrderedNearQuery(FIELD)
.addClause(new SpanTermQuery(new Term(FIELD, "b")))
.addClause(new SpanTermQuery(new Term(FIELD, "c")))
.setSlop(1)
.build(), BooleanClause.Occur.SHOULD)
.build();
final Query parent = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.MUST)
.add(bq, BooleanClause.Occur.MUST)
.build();
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", parent));
Document doc = buildDoc("a b c");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(3, m.getHitCount());
assertTrue(m.getFields().contains(FIELD));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 0, 1)));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(1, 2, 1, 3)));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(2, 4, 2, 5)));
}
}
public void testUnorderedNearWithinOrderedNear() throws Exception {
final SpanQuery spanPhrase = SpanNearQuery.newOrderedNearQuery(FIELD)
.addClause(new SpanTermQuery(new Term(FIELD, "time")))
.addClause(new SpanTermQuery(new Term(FIELD, "men")))
.setSlop(1)
.build();
final SpanQuery unorderedNear = SpanNearQuery.newUnorderedNearQuery(FIELD)
.addClause(spanPhrase)
.addClause(new SpanTermQuery(new Term(FIELD, "all")))
.setSlop(5)
.build();
final SpanQuery orderedNear = SpanNearQuery.newOrderedNearQuery(FIELD)
.addClause(new SpanTermQuery(new Term(FIELD, "the")))
.addClause(unorderedNear)
.setSlop(10)
.build();
final Query innerConjunct = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "is")), BooleanClause.Occur.MUST)
.add(orderedNear, BooleanClause.Occur.MUST)
.build();
final Query disjunct = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "now")), BooleanClause.Occur.SHOULD)
.add(innerConjunct, BooleanClause.Occur.SHOULD)
.build();
final Query outerConjunct = new BooleanQuery.Builder()
.add(disjunct, BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(FIELD, "good")), BooleanClause.Occur.MUST)
.build();
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", outerConjunct));
Document doc = buildDoc("now is the time for all good men");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertEquals(2, m.getHitCount());
assertTrue(m.getFields().contains(FIELD));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 0, 3)));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(6, 24, 6, 28)));
}
}
public void testMinShouldMatchQuery() throws Exception {
final Query minq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "x")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(FIELD, "y")), BooleanClause.Occur.SHOULD)
.add(new TermQuery(new Term(FIELD, "z")), BooleanClause.Occur.SHOULD)
.setMinimumNumberShouldMatch(2)
.build();
final Query bq = new BooleanQuery.Builder()
.add(new TermQuery(new Term(FIELD, "a")), BooleanClause.Occur.MUST)
.add(new TermQuery(new Term(FIELD, "b")), BooleanClause.Occur.MUST)
.add(minq, BooleanClause.Occur.SHOULD)
.build();
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", bq));
Document doc = buildDoc("a b x");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(2, m.getHitCount());
assertTrue(m.getFields().contains(FIELD));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 0, 1)));
assertTrue(m.getHits(FIELD).contains(new HighlightsMatch.Hit(1, 2, 1, 3)));
}
}
public void testComplexPhraseQueryParser() throws Exception {
ComplexPhraseQueryParser cpqp = new ComplexPhraseQueryParser(FIELD, new StandardAnalyzer());
Query query = cpqp.parse("\"x b\"");
try (Monitor monitor = newMonitor()) {
monitor.register(new MonitorQuery("1", query));
Document doc = buildDoc("x b c");
MatchingQueries<HighlightsMatch> matches = monitor.match(doc, HighlightsMatch.MATCHER);
HighlightsMatch m = matches.matches("1");
assertNotNull(m);
assertEquals(2, m.getHitCount());
assertTrue(m.getFields().contains(FIELD));
}
}
public void testHighlightBatches() throws Exception {
String query = "\"cell biology\"";
try (Monitor monitor = newMonitor(WHITESPACE)) {
monitor.register(new MonitorQuery("query0", parse("non matching query")));
monitor.register(new MonitorQuery("query1", parse(query)));
monitor.register(new MonitorQuery("query2", parse("biology")));
Document doc1 = new Document();
doc1.add(newTextField(FIELD, "the cell biology count", Field.Store.NO)); // matches
Document doc2 = new Document();
doc2.add(newTextField(FIELD, "nope", Field.Store.NO));
Document doc3 = new Document();
doc3.add(newTextField(FIELD, "biology text", Field.Store.NO));
MultiMatchingQueries<HighlightsMatch> matches = monitor.match(new Document[]{doc1, doc2, doc3}, HighlightsMatch.MATCHER);
assertEquals(2, matches.getMatchCount(0));
assertEquals(0, matches.getMatchCount(1));
assertEquals(1, matches.getMatchCount(2));
HighlightsMatch m1 = matches.matches("query1", 0);
assertTrue(m1.getHits(FIELD).contains(new HighlightsMatch.Hit(1, 4, 2, 16)));
HighlightsMatch m2 = matches.matches("query2", 2);
assertTrue(m2.getHits(FIELD).contains(new HighlightsMatch.Hit(0, 0, 0, 7)));
}
}
}
