Java Code Examples for org.apache.lucene.search.suggest.Lookup.LookupResult

The following examples show how to use org.apache.lucene.search.suggest.Lookup.LookupResult. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: newblog   Source File: BlogServiceImpl.java    License: Apache License 2.0 6 votes vote down vote up
/**
     * lookup
     *
     * @param suggester
     * @param keyword
     * @throws IOException
     */
    private static List<String> lookup(AnalyzingInfixSuggester suggester, String keyword
    ) throws IOException {
        //先以contexts为过滤条件进行过滤,再以title为关键字进行筛选,根据weight值排序返回前2条
        //第3个布尔值即是否每个Term都要匹配,第4个参数表示是否需要关键字高亮
        List<LookupResult> results = suggester.lookup(keyword, 20, true, true);
        List<String> list = new ArrayList<>();
        for (LookupResult result : results) {
            list.add(result.key.toString());
            //从payload中反序列化出Blog对象
//            BytesRef bytesRef = result.payload;
//            InputStream is = Tools.bytes2InputStream(bytesRef.bytes);
//            Blog blog = (Blog) Tools.deSerialize(is);
//            System.out.println("blog-Name:" + blog.getTitle());
//            System.out.println("blog-Content:" + blog.getContent());
//            System.out.println("blog-image:" + blog.getImageurl());
//            System.out.println("blog-numberSold:" + blog.getHits());
        }
        return list;
    }
 
Example 2
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testAfterLoad() throws Exception {
  Input keys[] = new Input[] {
    new Input("lend me your ear", 8, new BytesRef("foobar")),
    new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
  };

  Path tempDir = createTempDir("AnalyzingInfixSuggesterTest");

  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  suggester.build(new InputArrayIterator(keys));
  assertEquals(2, suggester.getCount());
  suggester.close();

  suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
  assertEquals(2, results.size());
  assertEquals("a penny saved is a penny earned", results.get(0).key);
  assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).highlightKey);
  assertEquals(10, results.get(0).value);
  assertEquals(new BytesRef("foobaz"), results.get(0).payload);
  assertEquals(2, suggester.getCount());
  suggester.close();
  a.close();
}
 
Example 3
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testBothExactAndPrefix() throws Exception {
  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false);
  suggester.build(new InputArrayIterator(new Input[0]));
  suggester.add(new BytesRef("the pen is pretty"), null, 10, new BytesRef("foobaz"));
  suggester.refresh();

  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("pen p", random()), 10, true, true);
  assertEquals(1, results.size());
  assertEquals("the pen is pretty", results.get(0).key);
  assertEquals("the <b>pen</b> is <b>p</b>retty", results.get(0).highlightKey);
  assertEquals(10, results.get(0).value);
  assertEquals(new BytesRef("foobaz"), results.get(0).payload);
  suggester.close();
  a.close();
}
 
Example 4
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testRandomEdits() throws IOException {
  List<Input> keys = new ArrayList<>();
  int numTerms = atLeast(100);
  for (int i = 0; i < numTerms; i++) {
    keys.add(new Input("boo" + TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
  }
  keys.add(new Input("foo bar boo far", 12));
  MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
                                                0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
  suggester.build(new InputArrayIterator(keys));
  int numIters = atLeast(10);
  for (int i = 0; i < numIters; i++) {
    String addRandomEdit = addRandomEdit("foo bar boo", FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX);
    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence(addRandomEdit, random()), false, 2);
    assertEquals(addRandomEdit, 1, results.size());
    assertEquals("foo bar boo far", results.get(0).key.toString());
    assertEquals(12, results.get(0).value, 0.01F);  
  }
  IOUtils.close(analyzer, tempDir);
}
 
Example 5
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testNonLatinRandomEdits() throws IOException {
  List<Input> keys = new ArrayList<>();
  int numTerms = atLeast(100);
  for (int i = 0; i < numTerms; i++) {
    keys.add(new Input("буу" + TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
  }
  keys.add(new Input("фуу бар буу фар", 12));
  MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
      0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
  suggester.build(new InputArrayIterator(keys));
  int numIters = atLeast(10);
  for (int i = 0; i < numIters; i++) {
    String addRandomEdit = addRandomEdit("фуу бар буу", 0);
    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence(addRandomEdit, random()), false, 2);
    assertEquals(addRandomEdit, 1, results.size());
    assertEquals("фуу бар буу фар", results.get(0).key.toString());
    assertEquals(12, results.get(0).value, 0.01F);
  }
  IOUtils.close(analyzer, tempDir);
}
 
Example 6
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoSeps() throws Exception {
  Input[] keys = new Input[] {
    new Input("ab cd", 0),
    new Input("abcd", 1),
  };

  int options = 0;

  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",a, a, options, 256, -1, true, 1, true, 1, 3, false);
  suggester.build(new InputArrayIterator(keys));
  // TODO: would be nice if "ab " would allow the test to
  // pass, and more generally if the analyzer can know
  // that the user's current query has ended at a word, 
  // but, analyzers don't produce SEP tokens!
  List<LookupResult> r = suggester.lookup(TestUtil.stringToCharSequence("ab c", random()), false, 2);
  assertEquals(2, r.size());

  // With no PRESERVE_SEPS specified, "ab c" should also
  // complete to "abcd", which has higher weight so should
  // appear first:
  assertEquals("abcd", r.get(0).key.toString());
  IOUtils.close(a, tempDir);
}
 
Example 7
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, 0, 2, -1, true, 1, true, 1, 3, false);

  List<Input> keys = Arrays.asList(new Input[] {
      new Input("a", 40),
      new Input("a ", 50),
      new Input(" a", 60),
    });

  Collections.shuffle(keys, random());
  suggester.build(new InputArrayIterator(keys));

  List<LookupResult> results = suggester.lookup("a", false, 5);
  assertEquals(2, results.size());
  assertEquals(" a", results.get(0).key);
  assertEquals(60, results.get(0).value);
  assertEquals("a ", results.get(1).key);
  assertEquals(50, results.get(1).value);
  IOUtils.close(a, tempDir);
}
 
Example 8
Source Project: lucene-solr   Source File: AnalyzingSuggesterTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testNoSeps() throws Exception {
  Input[] keys = new Input[] {
    new Input("ab cd", 0),
    new Input("abcd", 1),
  };

  int options = 0;

  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, options, 256, -1, true);
  suggester.build(new InputArrayIterator(keys));
  // TODO: would be nice if "ab " would allow the test to
  // pass, and more generally if the analyzer can know
  // that the user's current query has ended at a word, 
  // but, analyzers don't produce SEP tokens!
  List<LookupResult> r = suggester.lookup(TestUtil.stringToCharSequence("ab c", random()), false, 2);
  assertEquals(2, r.size());

  // With no PRESERVE_SEPS specified, "ab c" should also
  // complete to "abcd", which has higher weight so should
  // appear first:
  assertEquals("abcd", r.get(0).key.toString());
  IOUtils.close(a, tempDir);
}
 
Example 9
Source Project: lucene-solr   Source File: WFSTCompletionTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testExactFirst() throws Exception {

    Directory tempDir = getDirectory();
    WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", true);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 20),
          new Input("x", 2),
        }));

    for(int topN=1;topN<4;topN++) {
      List<LookupResult> results = suggester.lookup("x", false, topN);

      assertEquals(Math.min(topN, 2), results.size());

      assertEquals("x", results.get(0).key);
      assertEquals(2, results.get(0).value);

      if (topN > 1) {
        assertEquals("x y", results.get(1).key);
        assertEquals(20, results.get(1).value);
      }
    }
    tempDir.close();
  }
 
Example 10
Source Project: lucene-solr   Source File: WFSTCompletionTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testNonExactFirst() throws Exception {

    Directory tempDir = getDirectory();
    WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 20),
          new Input("x", 2),
        }));

    for(int topN=1;topN<4;topN++) {
      List<LookupResult> results = suggester.lookup("x", false, topN);

      assertEquals(Math.min(topN, 2), results.size());

      assertEquals("x y", results.get(0).key);
      assertEquals(20, results.get(0).value);

      if (topN > 1) {
        assertEquals("x", results.get(1).key);
        assertEquals(2, results.get(1).value);
      }
    }
    tempDir.close();
  }
 
Example 11
Source Project: lucene-solr   Source File: FSTCompletionTest.java    License: Apache License 2.0 6 votes vote down vote up
@Slow
public void testMultilingualInput() throws Exception {
  List<Input> input = LookupBenchmarkTest.readTop50KWiki();

  Directory tempDir = getDirectory();
  FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
  lookup.build(new InputArrayIterator(input));
  assertEquals(input.size(), lookup.getCount());
  for (Input tf : input) {
    assertNotNull("Not found: " + tf.term.toString(), lookup.get(TestUtil.bytesToCharSequence(tf.term, random())));
    assertEquals(tf.term.utf8ToString(), lookup.lookup(TestUtil.bytesToCharSequence(tf.term, random()), true, 1).get(0).key.toString());
  }

  List<LookupResult> result = lookup.lookup(stringToCharSequence("wit"), true, 5);
  assertEquals(5, result.size());
  assertTrue(result.get(0).key.toString().equals("wit"));  // exact match.
  assertTrue(result.get(1).key.toString().equals("with")); // highest count.
  tempDir.close();
}
 
Example 12
Source Project: lucene-solr   Source File: FSTCompletionTest.java    License: Apache License 2.0 6 votes vote down vote up
public void testRandom() throws Exception {
  List<Input> freqs = new ArrayList<>();
  Random rnd = random();
  for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
    int weight = rnd.nextInt(100); 
    freqs.add(new Input("" + rnd.nextLong(), weight));
  }

  Directory tempDir = getDirectory();
  FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
  lookup.build(new InputArrayIterator(freqs.toArray(new Input[freqs.size()])));

  for (Input tf : freqs) {
    final String term = tf.term.utf8ToString();
    for (int i = 1; i < term.length(); i++) {
      String prefix = term.substring(0, i);
      for (LookupResult lr : lookup.lookup(stringToCharSequence(prefix), true, 10)) {
        assertTrue(lr.key.toString().startsWith(prefix));
      }
    }
  }
  tempDir.close();
}
 
Example 13
/** Convert {@link SuggesterResult} to NamedList for constructing responses */
private void toNamedList(SuggesterResult suggesterResult, Map<String, SimpleOrderedMap<NamedList<Object>>> resultObj) {
  for(String suggesterName : suggesterResult.getSuggesterNames()) {
    SimpleOrderedMap<NamedList<Object>> results = new SimpleOrderedMap<>();
    for (String token : suggesterResult.getTokens(suggesterName)) {
      SimpleOrderedMap<Object> suggestionBody = new SimpleOrderedMap<>();
      List<LookupResult> lookupResults = suggesterResult.getLookupResult(suggesterName, token);
      suggestionBody.add(SuggesterResultLabels.SUGGESTION_NUM_FOUND, lookupResults.size());
      List<SimpleOrderedMap<Object>> suggestEntriesNamedList = new ArrayList<>();
      for (LookupResult lookupResult : lookupResults) {
        String suggestionString = lookupResult.key.toString();
        long weight = lookupResult.value;
        String payload = (lookupResult.payload != null) ? 
            lookupResult.payload.utf8ToString()
            : "";
        
        SimpleOrderedMap<Object> suggestEntryNamedList = new SimpleOrderedMap<>();
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_TERM, suggestionString);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_WEIGHT, weight);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_PAYLOAD, payload);
        suggestEntriesNamedList.add(suggestEntryNamedList);
        
      }
      suggestionBody.add(SuggesterResultLabels.SUGGESTIONS, suggestEntriesNamedList);
      results.add(token, suggestionBody);
    }
    resultObj.put(suggesterName, results);
  }
}
 
Example 14
/** Convert NamedList (suggester response) to {@link SuggesterResult} */
private SuggesterResult toSuggesterResult(Map<String, SimpleOrderedMap<NamedList<Object>>> suggestionsMap) {
  SuggesterResult result = new SuggesterResult();
  if (suggestionsMap == null) {
    return result;
  }
  // for each token
  for(Map.Entry<String, SimpleOrderedMap<NamedList<Object>>> entry : suggestionsMap.entrySet()) {
    String suggesterName = entry.getKey();
    for (Iterator<Map.Entry<String, NamedList<Object>>> suggestionsIter = entry.getValue().iterator(); suggestionsIter.hasNext();) {
      Map.Entry<String, NamedList<Object>> suggestions = suggestionsIter.next(); 
      String tokenString = suggestions.getKey();
      List<LookupResult> lookupResults = new ArrayList<>();
      NamedList<Object> suggestion = suggestions.getValue();
      // for each suggestion
      for (int j = 0; j < suggestion.size(); j++) {
        String property = suggestion.getName(j);
        if (property.equals(SuggesterResultLabels.SUGGESTIONS)) {
          @SuppressWarnings("unchecked")
          List<NamedList<Object>> suggestionEntries = (List<NamedList<Object>>) suggestion.getVal(j);
          for(NamedList<Object> suggestionEntry : suggestionEntries) {
            String term = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_TERM);
            Long weight = (Long) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_WEIGHT);
            String payload = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_PAYLOAD);
            LookupResult res = new LookupResult(new CharsRef(term), weight, new BytesRef(payload));
            lookupResults.add(res);
          }
        }
        result.add(suggesterName, tokenString, lookupResults);
      }
    }
  }
  return result;
}
 
Example 15
Source Project: lucene-solr   Source File: TestFreeTextSuggester.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compare(LookupResult a, LookupResult b) {
  if (a.value > b.value) {
    return -1;
  } else if (a.value < b.value) {
    return 1;
  } else {
    // Tie break by UTF16 sort order:
    return ((String) a.key).compareTo((String) b.key);
  }
}
 
Example 16
Source Project: lucene-solr   Source File: TestFreeTextSuggester.java    License: Apache License 2.0 5 votes vote down vote up
private static String toString(List<LookupResult> results) {
  StringBuilder b = new StringBuilder();
  for(LookupResult result : results) {
    b.append(' ');
    b.append(result.key);
    b.append('/');
    b.append(String.format(Locale.ROOT, "%.2f", ((double) result.value)/Long.MAX_VALUE));
  }
  return b.toString().trim();
}
 
Example 17
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
private void testConstructorDefaults(AnalyzingInfixSuggester suggester, Input[] keys, Analyzer a, 
    boolean allTermsRequired, boolean highlight) throws IOException {
  AnalyzingInfixSuggester suggester2 = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false, allTermsRequired, highlight);
  suggester2.build(new InputArrayIterator(keys));
  
  CharSequence key = TestUtil.stringToCharSequence("penny ea", random());
  
  List<LookupResult> results1 = suggester.lookup(key, 10, allTermsRequired, highlight);
  List<LookupResult> results2 = suggester2.lookup(key, false, 10);
  assertEquals(results1.size(), results2.size());
  assertEquals(results1.get(0).key, results2.get(0).key);
  assertEquals(results1.get(0).highlightKey, results2.get(0).highlightKey);
  
  suggester2.close();
}
 
Example 18
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testHighlight() throws Exception {
  Input keys[] = new Input[] {
    new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
  };

  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false);
  suggester.build(new InputArrayIterator(keys));
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("penn", random()), 10, true, true);
  assertEquals(1, results.size());
  assertEquals("a penny saved is a penny earned", results.get(0).key);
  assertEquals("a <b>penn</b>y saved is a <b>penn</b>y earned", results.get(0).highlightKey);
  suggester.close();
  a.close();
}
 
Example 19
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testHighlightCaseChange() throws Exception {
  Input keys[] = new Input[] {
    new Input("a Penny saved is a penny earned", 10, new BytesRef("foobaz")),
  };

  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false);
  suggester.build(new InputArrayIterator(keys));
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("penn", random()), 10, true, true);
  assertEquals(1, results.size());
  assertEquals("a Penny saved is a penny earned", results.get(0).key);
  assertEquals("a <b>Penn</b>y saved is a <b>penn</b>y earned", results.get(0).highlightKey);
  suggester.close();

  // Try again, but overriding addPrefixMatch to highlight
  // the entire hit:
  suggester = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false) {
      @Override
      protected void addPrefixMatch(StringBuilder sb, String surface, String analyzed, String prefixToken) {
        sb.append("<b>");
        sb.append(surface);
        sb.append("</b>");
      }
    };
  suggester.build(new InputArrayIterator(keys));
  results = suggester.lookup(TestUtil.stringToCharSequence("penn", random()), 10, true, true);
  assertEquals(1, results.size());
  assertEquals("a Penny saved is a penny earned", results.get(0).key);
  assertEquals("a <b>Penny</b> saved is a <b>penny</b> earned", results.get(0).highlightKey);
  suggester.close();
  a.close();
}
 
Example 20
Source Project: lucene-solr   Source File: AnalyzingInfixSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNRTWithParallelAdds() throws IOException, InterruptedException {
  String[] keys = new String[] {"python", "java", "c", "scala", "ruby", "clojure", "erlang", "go", "swift", "lisp"};
  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  Path tempDir = createTempDir("AIS_NRT_PERSIST_TEST");
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  Thread[] multiAddThreads = new Thread[10];
  // Cannot call refresh on an suggester when no docs are added to the index
  expectThrows(IllegalStateException.class, () -> {
    suggester.refresh();
  });

  for(int i=0; i<10; i++) {
    multiAddThreads[i] = new Thread(new IndexDocument(suggester, keys[i]));
  }
  for(int i=0; i<10; i++) {
    multiAddThreads[i].start();
  }
  //Make sure all threads have completed indexing
  for(int i=0; i<10; i++) {
    multiAddThreads[i].join();
  }

  suggester.refresh();
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("python", random()), 10, true, false);
  assertEquals(1, results.size());
  assertEquals("python", results.get(0).key);

  //Test if the index is getting persisted correctly and can be reopened.
  suggester.commit();
  suggester.close();

  AnalyzingInfixSuggester suggester2 = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  results = suggester2.lookup(TestUtil.stringToCharSequence("python", random()), 10, true, false);
  assertEquals(1, results.size());
  assertEquals("python", results.get(0).key);

  suggester2.close();
  a.close();
}
 
Example 21
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
/**
 * basic "standardanalyzer" test with stopword removal
 */
public void testStandard() throws Exception {
  Input keys[] = new Input[] {
      new Input("the ghost of christmas past", 50),
  };
  
  Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", standard, standard, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, false, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
      FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
  suggester.build(new InputArrayIterator(keys));
  
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("the ghost of chris", random()), false, 1);
  assertEquals(1, results.size());
  assertEquals("the ghost of christmas past", results.get(0).key.toString());
  assertEquals(50, results.get(0).value, 0.01F);

  // omit the 'the' since it's a stopword, it's suggested anyway
  results = suggester.lookup(TestUtil.stringToCharSequence("ghost of chris", random()), false, 1);
  assertEquals(1, results.size());
  assertEquals("the ghost of christmas past", results.get(0).key.toString());
  assertEquals(50, results.get(0).value, 0.01F);

  // omit the 'the' and 'of' since they are stopwords, it's suggested anyway
  results = suggester.lookup(TestUtil.stringToCharSequence("ghost chris", random()), false, 1);
  assertEquals(1, results.size());
  assertEquals("the ghost of christmas past", results.get(0).key.toString());
  assertEquals(50, results.get(0).value, 0.01F);
  
  IOUtils.close(standard, tempDir);
}
 
Example 22
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testGraphDups() throws Exception {

    final Analyzer analyzer = new AnalyzingSuggesterTest.MultiCannedAnalyzer(
        new CannedTokenStream(
            token("wifi", 1, 1),
            token("hotspot", 0, 2),
            token("network", 1, 1),
            token("is", 1, 1),
            token("slow", 1, 1)),
        new CannedTokenStream(
            token("wi", 1, 1),
            token("hotspot", 0, 3),
            token("fi", 1, 1),
            token("network", 1, 1),
            token("is", 1, 1),
            token("fast", 1, 1)),
        new CannedTokenStream(
            token("wifi", 1, 1),
            token("hotspot",0,2),
            token("network",1,1)));

    Input keys[] = new Input[] {
        new Input("wifi network is slow", 50),
        new Input("wi fi network is fast", 10),
    };
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
    suggester.build(new InputArrayIterator(keys));
    
    List<LookupResult> results = suggester.lookup("wifi network", false, 10);
    if (VERBOSE) {
      System.out.println("Results: " + results);
    }
    assertEquals(2, results.size());
    assertEquals("wifi network is slow", results.get(0).key);
    assertEquals(50, results.get(0).value);
    assertEquals("wi fi network is fast", results.get(1).key);
    assertEquals(10, results.get(1).value);
    IOUtils.close(tempDir, analyzer);
  }
 
Example 23
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
  suggester.build(new InputArrayIterator(new Input[0]));

  List<LookupResult> result = suggester.lookup("a", false, 20);
  assertTrue(result.isEmpty());
  IOUtils.close(analyzer, tempDir);
}
 
Example 24
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testInputPathRequired() throws Exception {

    //  SynonymMap.Builder b = new SynonymMap.Builder(false);
    //  b.add(new CharsRef("ab"), new CharsRef("ba"), true);
    //  final SynonymMap map = b.build();

    //  The Analyzer below mimics the functionality of the SynonymAnalyzer
    //  using the above map, so that the suggest module does not need a dependency on the 
    //  synonym module 

    final Analyzer analyzer = new AnalyzingSuggesterTest.MultiCannedAnalyzer(
        new CannedTokenStream(
            token("ab", 1, 1),
            token("ba", 0, 1),
            token("xc", 1, 1)),
        new CannedTokenStream(
            token("ba", 1, 1),
            token("xd", 1, 1)),
        new CannedTokenStream(
            token("ab", 1, 1),
            token("ba", 0, 1),
            token("x", 1, 1)));

    Input keys[] = new Input[] {
        new Input("ab xc", 50),
        new Input("ba xd", 50),
    };
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("ab x", false, 1);
    assertTrue(results.size() == 1);
    IOUtils.close(analyzer, tempDir);
  }
 
Example 25
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testExactFirst() throws Exception {

    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 1),
          new Input("x y z", 3),
          new Input("x", 2),
          new Input("z z z", 20),
        }));

    //System.out.println("ALL: " + suggester.lookup("x y", false, 6));

    for(int topN=1;topN<6;topN++) {
      List<LookupResult> results = suggester.lookup("x y", false, topN);
      //System.out.println("topN=" + topN + " " + results);

      assertEquals(Math.min(topN, 4), results.size());

      assertEquals("x y", results.get(0).key);
      assertEquals(1, results.get(0).value);

      if (topN > 1) {
        assertEquals("z z z", results.get(1).key);
        assertEquals(20, results.get(1).value);

        if (topN > 2) {
          assertEquals("x y z", results.get(2).key);
          assertEquals(3, results.get(2).value);

          if (topN > 3) {
            assertEquals("x", results.get(3).key);
            assertEquals(2, results.get(3).value);
          }
        }
      }
    }
    IOUtils.close(a, tempDir);
  }
 
Example 26
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testNonExactFirst() throws Exception {

    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 1),
          new Input("x y z", 3),
          new Input("x", 2),
          new Input("z z z", 20),
        }));

    for(int topN=1;topN<6;topN++) {
      List<LookupResult> results = suggester.lookup("p", false, topN);

      assertEquals(Math.min(topN, 4), results.size());

      assertEquals("z z z", results.get(0).key);
      assertEquals(20, results.get(0).value);

      if (topN > 1) {
        assertEquals("x y z", results.get(1).key);
        assertEquals(3, results.get(1).value);

        if (topN > 2) {
          assertEquals("x", results.get(2).key);
          assertEquals(2, results.get(2).value);
          
          if (topN > 3) {
            assertEquals("x y", results.get(3).key);
            assertEquals(1, results.get(3).value);
          }
        }
      }
    }
    IOUtils.close(a, tempDir);
  }
 
Example 27
Source Project: lucene-solr   Source File: FuzzySuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
@Override
public int compare(LookupResult a, LookupResult b) {
  if (a.value > b.value) {
    return -1;
  } else if (a.value < b.value) {
    return 1;
  } else {
    final int c = CHARSEQUENCE_COMPARATOR.compare(a.key, b.key);
    assert c != 0: "term=" + a.key;
    return c;
  }
}
 
Example 28
Source Project: lucene-solr   Source File: AnalyzingSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", standard);
  suggester.build(new InputArrayIterator(new Input[0]));

  List<LookupResult> result = suggester.lookup("a", false, 20);
  assertTrue(result.isEmpty());
  IOUtils.close(standard, tempDir);
}
 
Example 29
Source Project: lucene-solr   Source File: AnalyzingSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testGraphDups() throws Exception {

    final Analyzer analyzer = new MultiCannedAnalyzer(
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1),
            token("is",1,1),
            token("slow",1,1)),
        new CannedTokenStream(
            token("wi",1,1),
            token("hotspot",0,3),
            token("fi",1,1),
            token("network",1,1),
            token("is",1,1),
            token("fast",1,1)),
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1)));

    Input keys[] = new Input[] {
        new Input("wifi network is slow", 50),
        new Input("wi fi network is fast", 10),
    };
    //AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("wifi network", false, 10);
    if (VERBOSE) {
      System.out.println("Results: " + results);
    }
    assertEquals(2, results.size());
    assertEquals("wifi network is slow", results.get(0).key);
    assertEquals(50, results.get(0).value);
    assertEquals("wi fi network is fast", results.get(1).key);
    assertEquals(10, results.get(1).value);
    IOUtils.close(analyzer, tempDir);
  }
 
Example 30
Source Project: lucene-solr   Source File: AnalyzingSuggesterTest.java    License: Apache License 2.0 5 votes vote down vote up
public void testInputPathRequired() throws Exception {

    //  SynonymMap.Builder b = new SynonymMap.Builder(false);
    //  b.add(new CharsRef("ab"), new CharsRef("ba"), true);
    //  final SynonymMap map = b.build();

    //  The Analyzer below mimics the functionality of the SynonymAnalyzer
    //  using the above map, so that the suggest module does not need a dependency on the 
    //  synonym module

    final Analyzer analyzer = new MultiCannedAnalyzer(
        new CannedTokenStream(
            token("ab", 1, 1),
            token("ba", 0, 1),
            token("xc", 1, 1)),
        new CannedTokenStream(
            token("ba", 1, 1),
            token("xd", 1, 1)),
        new CannedTokenStream(
            token("ab",1,1),
            token("ba",0,1),
            token("x",1,1)));

    Input keys[] = new Input[] {
        new Input("ab xc", 50),
        new Input("ba xd", 50),
    };
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("ab x", false, 1);
    assertEquals(1, results.size());
    IOUtils.close(analyzer, tempDir);
  }