org.apache.lucene.search.suggest.Lookup.LookupResult Java Examples

The following examples show how to use org.apache.lucene.search.suggest.Lookup.LookupResult. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: BlogServiceImpl.java    From newblog with Apache License 2.0 6 votes vote down vote up
/**
     * lookup
     *
     * @param suggester
     * @param keyword
     * @throws IOException
     */
    private static List<String> lookup(AnalyzingInfixSuggester suggester, String keyword
    ) throws IOException {
        //先以contexts为过滤条件进行过滤,再以title为关键字进行筛选,根据weight值排序返回前2条
        //第3个布尔值即是否每个Term都要匹配,第4个参数表示是否需要关键字高亮
        List<LookupResult> results = suggester.lookup(keyword, 20, true, true);
        List<String> list = new ArrayList<>();
        for (LookupResult result : results) {
            list.add(result.key.toString());
            //从payload中反序列化出Blog对象
//            BytesRef bytesRef = result.payload;
//            InputStream is = Tools.bytes2InputStream(bytesRef.bytes);
//            Blog blog = (Blog) Tools.deSerialize(is);
//            System.out.println("blog-Name:" + blog.getTitle());
//            System.out.println("blog-Content:" + blog.getContent());
//            System.out.println("blog-image:" + blog.getImageurl());
//            System.out.println("blog-numberSold:" + blog.getHits());
        }
        return list;
    }
 
Example #2
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNoSeps() throws Exception {
  Input[] keys = new Input[] {
    new Input("ab cd", 0),
    new Input("abcd", 1),
  };

  int options = 0;

  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, options, 256, -1, true);
  suggester.build(new InputArrayIterator(keys));
  // TODO: would be nice if "ab " would allow the test to
  // pass, and more generally if the analyzer can know
  // that the user's current query has ended at a word, 
  // but, analyzers don't produce SEP tokens!
  List<LookupResult> r = suggester.lookup(TestUtil.stringToCharSequence("ab c", random()), false, 2);
  assertEquals(2, r.size());

  // With no PRESERVE_SEPS specified, "ab c" should also
  // complete to "abcd", which has higher weight so should
  // appear first:
  assertEquals("abcd", r.get(0).key.toString());
  IOUtils.close(a, tempDir);
}
 
Example #3
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testRandomEdits() throws IOException {
  List<Input> keys = new ArrayList<>();
  int numTerms = atLeast(100);
  for (int i = 0; i < numTerms; i++) {
    keys.add(new Input("boo" + TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
  }
  keys.add(new Input("foo bar boo far", 12));
  MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
                                                0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE);
  suggester.build(new InputArrayIterator(keys));
  int numIters = atLeast(10);
  for (int i = 0; i < numIters; i++) {
    String addRandomEdit = addRandomEdit("foo bar boo", FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX);
    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence(addRandomEdit, random()), false, 2);
    assertEquals(addRandomEdit, 1, results.size());
    assertEquals("foo bar boo far", results.get(0).key.toString());
    assertEquals(12, results.get(0).value, 0.01F);  
  }
  IOUtils.close(analyzer, tempDir);
}
 
Example #4
Source File: AnalyzingInfixSuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testBothExactAndPrefix() throws Exception {
  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newDirectory(), a, a, 3, false);
  suggester.build(new InputArrayIterator(new Input[0]));
  suggester.add(new BytesRef("the pen is pretty"), null, 10, new BytesRef("foobaz"));
  suggester.refresh();

  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("pen p", random()), 10, true, true);
  assertEquals(1, results.size());
  assertEquals("the pen is pretty", results.get(0).key);
  assertEquals("the <b>pen</b> is <b>p</b>retty", results.get(0).highlightKey);
  assertEquals(10, results.get(0).value);
  assertEquals(new BytesRef("foobaz"), results.get(0).payload);
  suggester.close();
  a.close();
}
 
Example #5
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNonLatinRandomEdits() throws IOException {
  List<Input> keys = new ArrayList<>();
  int numTerms = atLeast(100);
  for (int i = 0; i < numTerms; i++) {
    keys.add(new Input("буу" + TestUtil.randomSimpleString(random()), 1 + random().nextInt(100)));
  }
  keys.add(new Input("фуу бар буу фар", 12));
  MockAnalyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",analyzer, analyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, true, FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS,
      0, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, true);
  suggester.build(new InputArrayIterator(keys));
  int numIters = atLeast(10);
  for (int i = 0; i < numIters; i++) {
    String addRandomEdit = addRandomEdit("фуу бар буу", 0);
    List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence(addRandomEdit, random()), false, 2);
    assertEquals(addRandomEdit, 1, results.size());
    assertEquals("фуу бар буу фар", results.get(0).key.toString());
    assertEquals(12, results.get(0).value, 0.01F);
  }
  IOUtils.close(analyzer, tempDir);
}
 
Example #6
Source File: WFSTCompletionTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testExactFirst() throws Exception {

    Directory tempDir = getDirectory();
    WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", true);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 20),
          new Input("x", 2),
        }));

    for(int topN=1;topN<4;topN++) {
      List<LookupResult> results = suggester.lookup("x", false, topN);

      assertEquals(Math.min(topN, 2), results.size());

      assertEquals("x", results.get(0).key);
      assertEquals(2, results.get(0).value);

      if (topN > 1) {
        assertEquals("x y", results.get(1).key);
        assertEquals(20, results.get(1).value);
      }
    }
    tempDir.close();
  }
 
Example #7
Source File: WFSTCompletionTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNonExactFirst() throws Exception {

    Directory tempDir = getDirectory();
    WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 20),
          new Input("x", 2),
        }));

    for(int topN=1;topN<4;topN++) {
      List<LookupResult> results = suggester.lookup("x", false, topN);

      assertEquals(Math.min(topN, 2), results.size());

      assertEquals("x y", results.get(0).key);
      assertEquals(20, results.get(0).value);

      if (topN > 1) {
        assertEquals("x", results.get(1).key);
        assertEquals(2, results.get(1).value);
      }
    }
    tempDir.close();
  }
 
Example #8
Source File: AnalyzingInfixSuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testAfterLoad() throws Exception {
  Input keys[] = new Input[] {
    new Input("lend me your ear", 8, new BytesRef("foobar")),
    new Input("a penny saved is a penny earned", 10, new BytesRef("foobaz")),
  };

  Path tempDir = createTempDir("AnalyzingInfixSuggesterTest");

  Analyzer a = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false);
  AnalyzingInfixSuggester suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  suggester.build(new InputArrayIterator(keys));
  assertEquals(2, suggester.getCount());
  suggester.close();

  suggester = new AnalyzingInfixSuggester(newFSDirectory(tempDir), a, a, 3, false);
  List<LookupResult> results = suggester.lookup(TestUtil.stringToCharSequence("ear", random()), 10, true, true);
  assertEquals(2, results.size());
  assertEquals("a penny saved is a penny earned", results.get(0).key);
  assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).highlightKey);
  assertEquals(10, results.get(0).value);
  assertEquals(new BytesRef("foobaz"), results.get(0).payload);
  assertEquals(2, suggester.getCount());
  suggester.close();
  a.close();
}
 
Example #9
Source File: FSTCompletionTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
@Slow
public void testMultilingualInput() throws Exception {
  List<Input> input = LookupBenchmarkTest.readTop50KWiki();

  Directory tempDir = getDirectory();
  FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
  lookup.build(new InputArrayIterator(input));
  assertEquals(input.size(), lookup.getCount());
  for (Input tf : input) {
    assertNotNull("Not found: " + tf.term.toString(), lookup.get(TestUtil.bytesToCharSequence(tf.term, random())));
    assertEquals(tf.term.utf8ToString(), lookup.lookup(TestUtil.bytesToCharSequence(tf.term, random()), true, 1).get(0).key.toString());
  }

  List<LookupResult> result = lookup.lookup(stringToCharSequence("wit"), true, 5);
  assertEquals(5, result.size());
  assertTrue(result.get(0).key.toString().equals("wit"));  // exact match.
  assertTrue(result.get(1).key.toString().equals("with")); // highest count.
  tempDir.close();
}
 
Example #10
Source File: FSTCompletionTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testRandom() throws Exception {
  List<Input> freqs = new ArrayList<>();
  Random rnd = random();
  for (int i = 0; i < 2500 + rnd.nextInt(2500); i++) {
    int weight = rnd.nextInt(100); 
    freqs.add(new Input("" + rnd.nextLong(), weight));
  }

  Directory tempDir = getDirectory();
  FSTCompletionLookup lookup = new FSTCompletionLookup(tempDir, "fst");
  lookup.build(new InputArrayIterator(freqs.toArray(new Input[freqs.size()])));

  for (Input tf : freqs) {
    final String term = tf.term.utf8ToString();
    for (int i = 1; i < term.length(); i++) {
      String prefix = term.substring(0, i);
      for (LookupResult lr : lookup.lookup(stringToCharSequence(prefix), true, 10)) {
        assertTrue(lr.key.toString().startsWith(prefix));
      }
    }
  }
  tempDir.close();
}
 
Example #11
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testNoSeps() throws Exception {
  Input[] keys = new Input[] {
    new Input("ab cd", 0),
    new Input("abcd", 1),
  };

  int options = 0;

  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy",a, a, options, 256, -1, true, 1, true, 1, 3, false);
  suggester.build(new InputArrayIterator(keys));
  // TODO: would be nice if "ab " would allow the test to
  // pass, and more generally if the analyzer can know
  // that the user's current query has ended at a word, 
  // but, analyzers don't produce SEP tokens!
  List<LookupResult> r = suggester.lookup(TestUtil.stringToCharSequence("ab c", random()), false, 2);
  assertEquals(2, r.size());

  // With no PRESERVE_SEPS specified, "ab c" should also
  // complete to "abcd", which has higher weight so should
  // appear first:
  assertEquals("abcd", r.get(0).key.toString());
  IOUtils.close(a, tempDir);
}
 
Example #12
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 6 votes vote down vote up
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, 0, 2, -1, true, 1, true, 1, 3, false);

  List<Input> keys = Arrays.asList(new Input[] {
      new Input("a", 40),
      new Input("a ", 50),
      new Input(" a", 60),
    });

  Collections.shuffle(keys, random());
  suggester.build(new InputArrayIterator(keys));

  List<LookupResult> results = suggester.lookup("a", false, 5);
  assertEquals(2, results.size());
  assertEquals(" a", results.get(0).key);
  assertEquals(60, results.get(0).value);
  assertEquals("a ", results.get(1).key);
  assertEquals(50, results.get(1).value);
  IOUtils.close(a, tempDir);
}
 
Example #13
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public int compare(LookupResult a, LookupResult b) {
  if (a.value > b.value) {
    return -1;
  } else if (a.value < b.value) {
    return 1;
  } else {
    final int c = CHARSEQUENCE_COMPARATOR.compare(a.key, b.key);
    assert c != 0: "term=" + a.key;
    return c;
  }
}
 
Example #14
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Analyzer standard = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true, MockTokenFilter.ENGLISH_STOPSET);
  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", standard);
  suggester.build(new InputArrayIterator(new Input[0]));

  List<LookupResult> result = suggester.lookup("a", false, 20);
  assertTrue(result.isEmpty());
  IOUtils.close(standard, tempDir);
}
 
Example #15
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testNonExactFirst() throws Exception {

    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 1),
          new Input("x y z", 3),
          new Input("x", 2),
          new Input("z z z", 20),
        }));

    for(int topN=1;topN<6;topN++) {
      List<LookupResult> results = suggester.lookup("p", false, topN);

      assertEquals(Math.min(topN, 4), results.size());

      assertEquals("z z z", results.get(0).key);
      assertEquals(20, results.get(0).value);

      if (topN > 1) {
        assertEquals("x y z", results.get(1).key);
        assertEquals(3, results.get(1).value);

        if (topN > 2) {
          assertEquals("x", results.get(2).key);
          assertEquals(2, results.get(2).value);
          
          if (topN > 3) {
            assertEquals("x y", results.get(3).key);
            assertEquals(1, results.get(3).value);
          }
        }
      }
    }
    IOUtils.close(a, tempDir);
  }
 
Example #16
Source File: Suggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
@Override
public SpellingResult getSuggestions(SpellingOptions options) throws IOException {
  log.debug("getSuggestions: {}", options.tokens);
  if (lookup == null) {
    log.info("Lookup is null - invoke spellchecker.build first");
    return EMPTY_RESULT;
  }
  SpellingResult res = new SpellingResult();
  CharsRef scratch = new CharsRef();
  for (Token t : options.tokens) {
    scratch.chars = t.buffer();
    scratch.offset = 0;
    scratch.length = t.length();
    boolean onlyMorePopular = (options.suggestMode == SuggestMode.SUGGEST_MORE_POPULAR) &&
      !(lookup instanceof WFSTCompletionLookup) &&
      !(lookup instanceof AnalyzingSuggester);
    List<LookupResult> suggestions = lookup.lookup(scratch, onlyMorePopular, options.count);
    if (suggestions == null) {
      continue;
    }
    if (options.suggestMode != SuggestMode.SUGGEST_MORE_POPULAR) {
      Collections.sort(suggestions);
    }
    for (LookupResult lr : suggestions) {
      res.add(t, lr.key.toString(), (int)lr.value);
    }
  }
  return res;
}
 
Example #17
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testExactFirst() throws Exception {

    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", a, a, AnalyzingSuggester.EXACT_FIRST | AnalyzingSuggester.PRESERVE_SEP, 256, -1, true, 1, true, 1, 3, false);
    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 1),
          new Input("x y z", 3),
          new Input("x", 2),
          new Input("z z z", 20),
        }));

    //System.out.println("ALL: " + suggester.lookup("x y", false, 6));

    for(int topN=1;topN<6;topN++) {
      List<LookupResult> results = suggester.lookup("x y", false, topN);
      //System.out.println("topN=" + topN + " " + results);

      assertEquals(Math.min(topN, 4), results.size());

      assertEquals("x y", results.get(0).key);
      assertEquals(1, results.get(0).value);

      if (topN > 1) {
        assertEquals("z z z", results.get(1).key);
        assertEquals(20, results.get(1).value);

        if (topN > 2) {
          assertEquals("x y z", results.get(2).key);
          assertEquals(3, results.get(2).value);

          if (topN > 3) {
            assertEquals("x", results.get(3).key);
            assertEquals(2, results.get(3).value);
          }
        }
      }
    }
    IOUtils.close(a, tempDir);
  }
 
Example #18
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testGraphDups() throws Exception {

    final Analyzer analyzer = new MultiCannedAnalyzer(
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1),
            token("is",1,1),
            token("slow",1,1)),
        new CannedTokenStream(
            token("wi",1,1),
            token("hotspot",0,3),
            token("fi",1,1),
            token("network",1,1),
            token("is",1,1),
            token("fast",1,1)),
        new CannedTokenStream(
            token("wifi",1,1),
            token("hotspot",0,2),
            token("network",1,1)));

    Input keys[] = new Input[] {
        new Input("wifi network is slow", 50),
        new Input("wi fi network is fast", 10),
    };
    //AnalyzingSuggester suggester = new AnalyzingSuggester(analyzer, AnalyzingSuggester.EXACT_FIRST, 256, -1);
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("wifi network", false, 10);
    if (VERBOSE) {
      System.out.println("Results: " + results);
    }
    assertEquals(2, results.size());
    assertEquals("wifi network is slow", results.get(0).key);
    assertEquals(50, results.get(0).value);
    assertEquals("wi fi network is fast", results.get(1).key);
    assertEquals(10, results.get(1).value);
    IOUtils.close(analyzer, tempDir);
  }
 
Example #19
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testInputPathRequired() throws Exception {

    //  SynonymMap.Builder b = new SynonymMap.Builder(false);
    //  b.add(new CharsRef("ab"), new CharsRef("ba"), true);
    //  final SynonymMap map = b.build();

    //  The Analyzer below mimics the functionality of the SynonymAnalyzer
    //  using the above map, so that the suggest module does not need a dependency on the 
    //  synonym module

    final Analyzer analyzer = new MultiCannedAnalyzer(
        new CannedTokenStream(
            token("ab", 1, 1),
            token("ba", 0, 1),
            token("xc", 1, 1)),
        new CannedTokenStream(
            token("ba", 1, 1),
            token("xd", 1, 1)),
        new CannedTokenStream(
            token("ab",1,1),
            token("ba",0,1),
            token("x",1,1)));

    Input keys[] = new Input[] {
        new Input("ab xc", 50),
        new Input("ba xd", 50),
    };
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", analyzer);
    suggester.build(new InputArrayIterator(keys));
    List<LookupResult> results = suggester.lookup("ab x", false, 1);
    assertEquals(1, results.size());
    IOUtils.close(analyzer, tempDir);
  }
 
Example #20
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testNonExactFirst() throws Exception {

    Analyzer a = getUnusualAnalyzer();
    Directory tempDir = getDirectory();
    AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, AnalyzingSuggester.PRESERVE_SEP, 256, -1, true);

    suggester.build(new InputArrayIterator(new Input[] {
          new Input("x y", 1),
          new Input("x y z", 3),
          new Input("x", 2),
          new Input("z z z", 20),
        }));

    for(int topN=1;topN<6;topN++) {
      List<LookupResult> results = suggester.lookup("p", false, topN);

      assertEquals(Math.min(topN, 4), results.size());

      assertEquals("z z z", results.get(0).key);
      assertEquals(20, results.get(0).value);

      if (topN > 1) {
        assertEquals("x y z", results.get(1).key);
        assertEquals(3, results.get(1).value);

        if (topN > 2) {
          assertEquals("x", results.get(2).key);
          assertEquals(2, results.get(2).value);
          
          if (topN > 3) {
            assertEquals("x y", results.get(3).key);
            assertEquals(1, results.get(3).value);
          }
        }
      }
    }
    IOUtils.close(a, tempDir);
  }
 
Example #21
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testMaxSurfaceFormsPerAnalyzedForm() throws Exception {
  Analyzer a = new MockAnalyzer(random());
  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 2, -1, true);
  suggester.build(new InputArrayIterator(shuffle(new Input("a", 40),
      new Input("a ", 50), new Input(" a", 60))));

  List<LookupResult> results = suggester.lookup("a", false, 5);
  assertEquals(2, results.size());
  assertEquals(" a", results.get(0).key);
  assertEquals(60, results.get(0).value);
  assertEquals("a ", results.get(1).key);
  assertEquals(50, results.get(1).value);
  IOUtils.close(a, tempDir);
}
 
Example #22
Source File: AnalyzingSuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/**
 * Adds 50 random keys, that all analyze to the same thing (dog), with the same cost,
 * and checks that they come back in surface-form order.
 */
public void testTieBreakOnSurfaceForm() throws Exception {
  Analyzer a = new MultiCannedAnalyzer(new CannedTokenStream(token("dog", 1, 1)));

  Directory tempDir = getDirectory();
  AnalyzingSuggester suggester = new AnalyzingSuggester(tempDir, "suggest", a, a, 0, 256, -1, true);

  // make 50 inputs all with the same cost of 1, random strings
  Input[] inputs = new Input[100];
  for (int i = 0; i < inputs.length; i++) {
    inputs[i] = new Input(TestUtil.randomSimpleString(random()), 1);
  }

  suggester.build(new InputArrayIterator(inputs));

  // Try to save/load:
  Path tmpDir = createTempDir("AnalyzingSuggesterTest");
  Path path = tmpDir.resolve("suggester");

  OutputStream os = Files.newOutputStream(path);
  suggester.store(os);
  os.close();

  InputStream is = Files.newInputStream(path);
  suggester.load(is);
  is.close();

  // now suggest everything, and check that stuff comes back in order
  List<LookupResult> results = suggester.lookup("", false, 50);
  assertEquals(50, results.size());
  for (int i = 1; i < 50; i++) {
    String previous = results.get(i-1).toString();
    String current = results.get(i).toString();
    assertTrue("surface forms out of order: previous=" + previous + ",current=" + current,
               current.compareTo(previous) >= 0);
  }

  IOUtils.close(a, tempDir);
}
 
Example #23
Source File: WFSTCompletionTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Directory tempDir = getDirectory();
  WFSTCompletionLookup suggester = new WFSTCompletionLookup(tempDir, "wfst", false);

  suggester.build(new InputArrayIterator(new Input[0]));
  assertEquals(0, suggester.getCount());
  List<LookupResult> result = suggester.lookup("a", false, 20);
  assertTrue(result.isEmpty());
  tempDir.close();
}
 
Example #24
Source File: SuggestComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Convert {@link SuggesterResult} to NamedList for constructing responses */
private void toNamedList(SuggesterResult suggesterResult, Map<String, SimpleOrderedMap<NamedList<Object>>> resultObj) {
  for(String suggesterName : suggesterResult.getSuggesterNames()) {
    SimpleOrderedMap<NamedList<Object>> results = new SimpleOrderedMap<>();
    for (String token : suggesterResult.getTokens(suggesterName)) {
      SimpleOrderedMap<Object> suggestionBody = new SimpleOrderedMap<>();
      List<LookupResult> lookupResults = suggesterResult.getLookupResult(suggesterName, token);
      suggestionBody.add(SuggesterResultLabels.SUGGESTION_NUM_FOUND, lookupResults.size());
      List<SimpleOrderedMap<Object>> suggestEntriesNamedList = new ArrayList<>();
      for (LookupResult lookupResult : lookupResults) {
        String suggestionString = lookupResult.key.toString();
        long weight = lookupResult.value;
        String payload = (lookupResult.payload != null) ? 
            lookupResult.payload.utf8ToString()
            : "";
        
        SimpleOrderedMap<Object> suggestEntryNamedList = new SimpleOrderedMap<>();
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_TERM, suggestionString);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_WEIGHT, weight);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_PAYLOAD, payload);
        suggestEntriesNamedList.add(suggestEntryNamedList);
        
      }
      suggestionBody.add(SuggesterResultLabels.SUGGESTIONS, suggestEntriesNamedList);
      results.add(token, suggestionBody);
    }
    resultObj.put(suggesterName, results);
  }
}
 
Example #25
Source File: SuggestComponent.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Convert NamedList (suggester response) to {@link SuggesterResult} */
private SuggesterResult toSuggesterResult(Map<String, SimpleOrderedMap<NamedList<Object>>> suggestionsMap) {
  SuggesterResult result = new SuggesterResult();
  if (suggestionsMap == null) {
    return result;
  }
  // for each token
  for(Map.Entry<String, SimpleOrderedMap<NamedList<Object>>> entry : suggestionsMap.entrySet()) {
    String suggesterName = entry.getKey();
    for (Iterator<Map.Entry<String, NamedList<Object>>> suggestionsIter = entry.getValue().iterator(); suggestionsIter.hasNext();) {
      Map.Entry<String, NamedList<Object>> suggestions = suggestionsIter.next(); 
      String tokenString = suggestions.getKey();
      List<LookupResult> lookupResults = new ArrayList<>();
      NamedList<Object> suggestion = suggestions.getValue();
      // for each suggestion
      for (int j = 0; j < suggestion.size(); j++) {
        String property = suggestion.getName(j);
        if (property.equals(SuggesterResultLabels.SUGGESTIONS)) {
          @SuppressWarnings("unchecked")
          List<NamedList<Object>> suggestionEntries = (List<NamedList<Object>>) suggestion.getVal(j);
          for(NamedList<Object> suggestionEntry : suggestionEntries) {
            String term = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_TERM);
            Long weight = (Long) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_WEIGHT);
            String payload = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_PAYLOAD);
            LookupResult res = new LookupResult(new CharsRef(term), weight, new BytesRef(payload));
            lookupResults.add(res);
          }
        }
        result.add(suggesterName, tokenString, lookupResults);
      }
    }
  }
  return result;
}
 
Example #26
Source File: SuggesterResult.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Add suggestion results for <code>token</code> */
public void add(String suggesterName, String token, List<LookupResult> results) {
  Map<String, List<LookupResult>> suggesterRes = this.suggestionsMap.get(suggesterName);
  if (suggesterRes == null) {
    this.suggestionsMap.put(suggesterName, new HashMap<String, List<LookupResult>>());
  }
  List<LookupResult> res = this.suggestionsMap.get(suggesterName).get(token);
  if (res == null) {
    res = results;
    this.suggestionsMap.get(suggesterName).put(token, res);
  }
}
 
Example #27
Source File: SolrSuggester.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
/** Returns suggestions based on the {@link SuggesterOptions} passed */
public SuggesterResult getSuggestions(SuggesterOptions options) throws IOException {
  if (log.isDebugEnabled()) {
    log.debug("getSuggestions: {}", options.token);
  }
  if (lookup == null) {
    log.info("Lookup is null - invoke suggest.build first");
    return EMPTY_RESULT;
  }
  
  SuggesterResult res = new SuggesterResult();
  List<LookupResult> suggestions;
  if(options.contextFilterQuery == null){
    //TODO: this path needs to be fixed to accept query params to override configs such as allTermsRequired, highlight
    suggestions = lookup.lookup(options.token, false, options.count);
  } else {
    BooleanQuery query = parseContextFilterQuery(options.contextFilterQuery);
    suggestions = lookup.lookup(options.token, query, options.count, options.allTermsRequired, options.highlight);
    if(suggestions == null){
      // Context filtering not supported/configured by lookup
      // Silently ignore filtering and serve a result by querying without context filtering
      if (log.isDebugEnabled()) {
        log.debug("Context Filtering Query not supported by {}", lookup.getClass());
      }
      suggestions = lookup.lookup(options.token, false, options.count);
    }
  }
  res.add(getName(), options.token.toString(), suggestions);
  return res;
}
 
Example #28
Source File: FuzzySuggesterTest.java    From lucene-solr with Apache License 2.0 5 votes vote down vote up
public void testEmpty() throws Exception {
  Analyzer analyzer = new MockAnalyzer(random(), MockTokenizer.KEYWORD, false);
  Directory tempDir = getDirectory();
  FuzzySuggester suggester = new FuzzySuggester(tempDir, "fuzzy", analyzer);
  suggester.build(new InputArrayIterator(new Input[0]));

  List<LookupResult> result = suggester.lookup("a", false, 20);
  assertTrue(result.isEmpty());
  IOUtils.close(analyzer, tempDir);
}
 
Example #29
Source File: AsyncBuildSuggestComponent.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
/** Convert {@link SuggesterResult} to NamedList for constructing responses */
private void toNamedList(SuggesterResult suggesterResult, Map<String, SimpleOrderedMap<NamedList<Object>>> resultObj) {
  for(String suggesterName : suggesterResult.getSuggesterNames()) {
    SimpleOrderedMap<NamedList<Object>> results = new SimpleOrderedMap<>();
    for (String token : suggesterResult.getTokens(suggesterName)) {
      SimpleOrderedMap<Object> suggestionBody = new SimpleOrderedMap<>();
      List<LookupResult> lookupResults = suggesterResult.getLookupResult(suggesterName, token);
      suggestionBody.add(SuggesterResultLabels.SUGGESTION_NUM_FOUND, lookupResults.size());
      List<SimpleOrderedMap<Object>> suggestEntriesNamedList = new ArrayList<>();
      for (LookupResult lookupResult : lookupResults) {
        String suggestionString = lookupResult.key.toString();
        long weight = lookupResult.value;
        String payload = (lookupResult.payload != null) ? 
            lookupResult.payload.utf8ToString()
            : "";
        
        SimpleOrderedMap<Object> suggestEntryNamedList = new SimpleOrderedMap<>();
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_TERM, suggestionString);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_WEIGHT, weight);
        suggestEntryNamedList.add(SuggesterResultLabels.SUGGESTION_PAYLOAD, payload);
        suggestEntriesNamedList.add(suggestEntryNamedList);
        
      }
      suggestionBody.add(SuggesterResultLabels.SUGGESTIONS, suggestEntriesNamedList);
      results.add(token, suggestionBody);
    }
    resultObj.put(suggesterName, results);
  }
}
 
Example #30
Source File: AsyncBuildSuggestComponent.java    From SearchServices with GNU Lesser General Public License v3.0 5 votes vote down vote up
/** Convert NamedList (suggester response) to {@link SuggesterResult} */
private SuggesterResult toSuggesterResult(Map<String, SimpleOrderedMap<NamedList<Object>>> suggestionsMap) {
  SuggesterResult result = new SuggesterResult();
  if (suggestionsMap == null) {
    return result;
  }
  // for each token
  for(Map.Entry<String, SimpleOrderedMap<NamedList<Object>>> entry : suggestionsMap.entrySet()) {
    String suggesterName = entry.getKey();
    for (Iterator<Map.Entry<String, NamedList<Object>>> suggestionsIter = entry.getValue().iterator(); suggestionsIter.hasNext();) {
      Map.Entry<String, NamedList<Object>> suggestions = suggestionsIter.next(); 
      String tokenString = suggestions.getKey();
      List<LookupResult> lookupResults = new ArrayList<>();
      NamedList<Object> suggestion = suggestions.getValue();
      // for each suggestion
      for (int j = 0; j < suggestion.size(); j++) {
        String property = suggestion.getName(j);
        if (property.equals(SuggesterResultLabels.SUGGESTIONS)) {
          @SuppressWarnings("unchecked")
          List<NamedList<Object>> suggestionEntries = (List<NamedList<Object>>) suggestion.getVal(j);
          for(NamedList<Object> suggestionEntry : suggestionEntries) {
            String term = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_TERM);
            Long weight = (Long) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_WEIGHT);
            String payload = (String) suggestionEntry.get(SuggesterResultLabels.SUGGESTION_PAYLOAD);
            LookupResult res = new LookupResult(new CharsRef(term), weight, new BytesRef(payload));
            lookupResults.add(res);
          }
        }
        result.add(suggesterName, tokenString, lookupResults);
      }
    }
  }
  return result;
}