Java Code Examples for org.apache.lucene.analysis.BaseTokenStreamTestCase#assertAnalyzesTo()

The following examples show how to use org.apache.lucene.analysis.BaseTokenStreamTestCase#assertAnalyzesTo() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

6 votes

public void testMailtoSchemeEmails () throws Exception {
  // See LUCENE-3880
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "mailto:[email protected]",
      new String[] {"mailto", "[email protected]"},
      new String[] { "<ALPHANUM>", "<EMAIL>" });

  // TODO: Support full mailto: scheme URIs. See RFC 6068: http://tools.ietf.org/html/rfc6068
  BaseTokenStreamTestCase.assertAnalyzesTo
      (a,  "mailto:[email protected],[email protected][email protected]"
         + "&subject=Subjectivity&body=Corpusivity%20or%20something%20like%20that",
       new String[] { "mailto",
                      "[email protected]",
                      // TODO: recognize ',' address delimiter. Also, see examples of ';' delimiter use at: http://www.mailto.co.uk/
                      ",[email protected]",
                      "[email protected]", // TODO: split field keys/values
                      "subject", "Subjectivity",
                      "body", "Corpusivity", "20or", "20something","20like", "20that" }, // TODO: Hex decoding + re-tokenization
       new String[] { "<ALPHANUM>",
                      "<EMAIL>",
                      "<EMAIL>",
                      "<EMAIL>",
                      "<ALPHANUM>", "<ALPHANUM>",
                      "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>" });
}

Example 2

Source File: TestStandardAnalyzer.java From lucene-solr with Apache License 2.0

6 votes

/** variation sequence */
public void testEmojiVariationSequence() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "#️⃣",
      new String[] { "#️⃣" },
      new String[] { "<EMOJI>" });
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "3️⃣",
      new String[] { "3️⃣",},
      new String[] { "<EMOJI>" });

  // text presentation sequences
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "#\uFE0E",
      new String[] { },
      new String[] { });
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "3\uFE0E",  // \uFE0E is included in \p{WB:Extend}
      new String[] { "3\uFE0E",},
      new String[] { "<NUM>" });
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "\u2B55\uFE0E",     // \u2B55 = HEAVY BLACK CIRCLE
      new String[] { "\u2B55",},
      new String[] { "<EMOJI>" });
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "\u2B55\uFE0E\u200D\u2B55\uFE0E",
      new String[] { "\u2B55", "\u200D\u2B55"},
      new String[] { "<EMOJI>", "<EMOJI>" });
}

Example 3

Source File: TestStandardAnalyzer.java From lucene-solr with Apache License 2.0

5 votes

public void testVariousTextSA() throws Exception {
  // various
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"C", "embedded", "developers", "wanted"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "FOO", "BAR"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new String[]{"foo", "bar", "FOO", "BAR"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"QUOTED", "word"});
}

Example 4

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

5 votes

public void testLUCENE1545() throws Exception {
  /*
   * Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTRE E.
   * The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.
   * Expected result is only on token "moͤchte".
   */
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "moͤchte", new String[] { "moͤchte" }); 
}

Example 5

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

5 votes

public void testVariousTextSA() throws Exception {
  // various
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"C", "embedded", "developers", "wanted"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "FOO", "BAR"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new String[]{"foo", "bar", "FOO", "BAR"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"QUOTED", "word"});
}

Example 6

Source File: TestICUTokenizer.java From lucene-solr with Apache License 2.0

5 votes

public void testEmojiFromTheFuture() throws Exception {
  // pick an unassigned character with extended_pictographic
  int ch = new UnicodeSet("[[:Extended_Pictographic:]&[:Unassigned:]]").getRangeStart(0);
  String value = new String(Character.toChars(ch));
  // should analyze to emoji type
  BaseTokenStreamTestCase.assertAnalyzesTo(a, value,
      new String[] { value },
      new String[] { "<EMOJI>" });
  // shouldn't break in a sequence
  BaseTokenStreamTestCase.assertAnalyzesTo(a, value + '\u200D' + value,
      new String[] { value + '\u200D' + value  },
      new String[] { "<EMOJI>" });
}

Example 7

Source File: TestUAX29URLEmailAnalyzer.java From lucene-solr with Apache License 2.0

5 votes

public void testLUCENE1545() throws Exception {
  /*
   * Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTER E.
   * The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.
   * Expected result is only one token "moͤchte".
   */
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "moͤchte", new String[] { "moͤchte" }); 
}

Example 8

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testBengali() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
      new String[] { "এই", "বিশ্বকোষ", "পরিচালনা", "করে", "উইকিমিডিয়া", "ফাউন্ডেশন", "একটি", "অলাভজনক", "সংস্থা", "উইকিপিডিয়ার",
      "শুরু", "১৫", "জানুয়ারি", "২০০১", "সালে", "এখন", "পর্যন্ত", "২০০টিরও", "বেশী", "ভাষায়", "উইকিপিডিয়া", "রয়েছে" });
}

Example 9

Source File: TestUAX29URLEmailAnalyzer.java From lucene-solr with Apache License 2.0

4 votes

public void testBengali() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
      new String[] { "এই", "বিশ্বকোষ", "পরিচালনা", "করে", "উইকিমিডিয়া", "ফাউন্ডেশন", "একটি", "অলাভজনক", "সংস্থা", "উইকিপিডিয়ার",
      "শুরু", "১৫", "জানুয়ারি", "২০০১", "সালে", "এখন", "পর্যন্ত", "২০০টিরও", "বেশী", "ভাষায়", "উইকিপিডিয়া", "রয়েছে" });
}

Example 10

Source File: TestStandardAnalyzer.java From lucene-solr with Apache License 2.0

4 votes

public void testDelimitersSA() throws Exception {
  // other delimiters: "-", "/", ","
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "some-dashed-phrase", new String[]{"some", "dashed", "phrase"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "dogs,chase,cats", new String[]{"dogs", "chase", "cats"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "ac/dc", new String[]{"ac", "dc"});
}

Example 11

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testKorean() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "훈민정음",
      new String[] { "훈민정음" },
      new String[] { "<HANGUL>" });
}

Example 12

Source File: TestUAX29URLEmailAnalyzer.java From lucene-solr with Apache License 2.0

4 votes

public void testAlphanumericSA() throws Exception {
  // alphanumeric tokens
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "B2B", new String[]{"b2b"});
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "2B", new String[]{"2b"});
}

Example 13

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testFarsi() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",
      new String[] { "ویکی", "پدیای", "انگلیسی", "در", "تاریخ", "۲۵", "دی", "۱۳۷۹", "به", "صورت", "مکملی",
      "برای", "دانشنامهٔ", "تخصصی", "نوپدیا", "نوشته", "شد" });
}

Example 14

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testAramaic() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",
      new String[] { "ܘܝܩܝܦܕܝܐ", "ܐܢܓܠܝܐ", "Wikipedia", "ܗܘ", "ܐܝܢܣܩܠܘܦܕܝܐ", "ܚܐܪܬܐ", "ܕܐܢܛܪܢܛ", "ܒܠܫܢ̈ܐ", "ܣܓܝܐ̈ܐ", "ܫܡܗ",
      "ܐܬܐ", "ܡܢ", "ܡ̈ܠܬܐ", "ܕ", "ܘܝܩܝ", "ܘ", "ܐܝܢܣܩܠܘܦܕܝܐ"});
}

Example 15

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testTextWithNumbersSA() throws Exception {
  // numbers
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "David has 5000 bones", new String[]{"David", "has", "5000", "bones"});
}

Example 16

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testAmharic() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
      new String[] { "ዊኪፔድያ", "የባለ", "ብዙ", "ቋንቋ", "የተሟላ", "ትክክለኛና", "ነጻ", "መዝገበ", "ዕውቀት", "ኢንሳይክሎፒዲያ", "ነው", "ማንኛውም" } );
}

Example 17

Source File: TestStandardAnalyzer.java From lucene-solr with Apache License 2.0

4 votes

public void testAmharic() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
      new String[] { "ዊኪፔድያ", "የባለ", "ብዙ", "ቋንቋ", "የተሟላ", "ትክክለኛና", "ነጻ", "መዝገበ", "ዕውቀት", "ኢንሳይክሎፒዲያ", "ነው", "ማንኛውም" } );
}

Example 18

Source File: TestICUTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testJapanese() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "仮名遣い カタカナ",
      new String[] { "仮", "名", "遣", "い", "カタカナ" },
      new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });
}

Example 19

Source File: TestStandardAnalyzer.java From lucene-solr with Apache License 2.0

4 votes

public void testJapanese() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "仮名遣い カタカナ",
      new String[] { "仮", "名", "遣", "い", "カタカナ" },
      new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });
}

Example 20

Source File: TestUAX29URLEmailTokenizer.java From lucene-solr with Apache License 2.0

4 votes

public void testJapanese() throws Exception {
  BaseTokenStreamTestCase.assertAnalyzesTo(a, "仮名遣い カタカナ",
      new String[] { "仮", "名", "遣", "い", "カタカナ" },
      new String[] { "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<IDEOGRAPHIC>", "<HIRAGANA>", "<KATAKANA>" });
}