package org.airsonic.player.service.search;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.junit.Test;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import static java.util.Arrays.asList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
/**
* Test case for Analyzer.
* These cases have the purpose of observing the current situation
* and observing the impact of upgrading Lucene.
*/
public class AnalyzerFactoryTestCase {
private AnalyzerFactory analyzerFactory = new AnalyzerFactory();
/**
* Test for the number of character separators per field.
*/
@Test
public void testTokenCounts() {
/*
* Analyzer used in legacy uses the same Tokenizer for all fields.
* (Some fields are converted to their own input string for integrity.)
* As a result, specifications for strings are scattered and difficult to understand.
* Using PerFieldAnalyzerWrapper,
* it is possible to use different Analyzer (Tokenizer/Filter) for each field.
* This allows consistent management of parsing definitions.
* It is also possible to apply definitions such as "id3 delimiters Tokenizer" to specific fields.
*/
// The number of words excluding articles is 7.
String query = "The quick brown fox jumps over the lazy dog.";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> terms = toTermString(n, query);
switch (n) {
/*
* In the legacy, these field divide input into 7. It is not necessary to delimit
* this field originally.
*/
case FieldNames.FOLDER:
case FieldNames.MEDIA_TYPE:
case FieldNames.GENRE:
assertEquals("oneTokenFields : " + n, 7, terms.size());
break;
/*
* These should be divided into 7.
*/
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("multiTokenFields : " + n, 7, terms.size());
break;
/*
* ID, FOLDER_ID, YEAR
* This is not a problem because the input value does not contain a delimiter.
*/
default:
assertEquals("oneTokenFields : " + n, 7, terms.size());
break;
}
});
}
/**
* Detailed tests on Punctuation.
* In addition to the common delimiters, there are many delimiters.
*/
@Test
public void testPunctuation1() {
String query = "B︴C";
String expected = "b︴c";
/*
* XXX 3.x -> 8.x :
* The definition of punctuation has changed.
*/
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> terms = toTermString(n, query);
switch (n) {
/*
* In the legacy, these field divide input into 2.
* It is not necessary to delimit
* this field originally.
*/
case FieldNames.FOLDER:
case FieldNames.GENRE:
case FieldNames.MEDIA_TYPE:
assertEquals("tokenized : " + n, 1, terms.size());
assertEquals("tokenized : " + n, expected, terms.get(0));
break;
/*
* What should the fields of this be?
* Generally discarded.
*/
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("tokenized : " + n, 1, terms.size());
assertEquals("tokenized : " + n, expected, terms.get(0));
break;
/*
* ID, FOLDER_ID, YEAR
* This is not a problem because the input value does not contain a delimiter.
*/
default:
assertEquals("tokenized : " + n, 2, terms.size());
break;
}
});
}
/*
* Detailed tests on Punctuation.
* Many of the symbols are delimiters or target to be removed.
*/
@Test
public void testPunctuation2() {
String query = "{'“『【【】】[︴○◎@ $〒→+]";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> terms = toTermString(n, query);
switch (n) {
case FieldNames.FOLDER:
case FieldNames.MEDIA_TYPE:
case FieldNames.GENRE:
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("removed : " + n, 0, terms.size());
break;
default:
assertEquals("removed : " + n, 0, terms.size());
}
});
}
/**
* Detailed tests on Stopward.
*
* @see org.apache.lucene.analysis.core.StopAnalyzer#ENGLISH_STOP_WORDS_SET
*/
@Test
public void testStopward() {
/*
* Legacy behavior is to remove ENGLISH_STOP_WORDS_SET from the Token stream.
* (Putting whether or not it matches the specification of the music search.)
*/
/*
* article.
* This is included in ENGLISH_STOP_WORDS_SET.
*/
String queryArticle = "a an the";
/*
* The default set as index stop word.
* But these are not included in ENGLISH_STOP_WORDS_SET.
*/
String queryArticle4Index = "el la los las le les";
/*
* Non-article in the ENGLISH_STOP_WORDS_SET.
* Stopwords are essential for newspapers and documents,
* but offten they are over-processed for song titles.
* For example, "we will rock you" can not be searched by "will".
*/
String queryStop = "and are as at be but by for if in into is it no not of on " //
+ "or such that their then there these they this to was will with";
/*
* Unique conjunctions often used in artist fields.
*/
String stopwordsForArtist = "by cv feat vs with";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> articleTerms = toTermString(n, queryArticle);
List<String> indexArticleTerms = toTermString(n, queryArticle4Index);
List<String> stopedTerms = toTermString(n, queryStop);
List<String> artistTerms = toTermString(n, stopwordsForArtist);
switch (n) {
case FieldNames.ALBUM:
case FieldNames.TITLE:
// Deleted because it is a stopword.
assertEquals("article : " + n, 0, articleTerms.size());
// "la los" is not deleted(#1235).
assertEquals("sonic server index article: " + n, 2, indexArticleTerms.size());
// Not deleted because it is not a stopword.
assertEquals("non-article stop words : " + n, 30, stopedTerms.size());
// Not deleted because it is not a stopword.
assertEquals("stop words for artsist : " + n, 5, artistTerms.size());
break;
case FieldNames.ARTIST:
// Deleted because it is a stopword.
assertEquals("article : " + n, 0, articleTerms.size());
// "la los" is not deleted(#1235).
assertEquals("sonic server index article: " + n, 2, indexArticleTerms.size());
// Not deleted because it is not a stopword(Except by and with).
assertEquals("non-article stop words : " + n, 28, stopedTerms.size());
// Deleted because it is a stopword.
assertEquals("stop words for artsist : " + n, 0, artistTerms.size());
break;
default:
fail(); // no analyze field is not applicable
break;
}
});
}
/**
* Simple test on FullWidth.
*/
@Test
public void testFullWidth() {
String query = "ＦＵＬＬ－ＷＩＤＴＨ";
List<String> terms = toTermString(query);
assertEquals(2, terms.size());
assertEquals("full", terms.get(0));
assertEquals("width", terms.get(1));
}
/**
* Combined case of Stop and full-width.
*/
@Test
public void testStopwardAndFullWidth() {
/*
* This and is not deleted because they are different from the default stopword.
*/
String queryHalfWidth = "THIS IS FULL-WIDTH SENTENCES.";
List<String> terms = toTermString(queryHalfWidth);
assertEquals(5, terms.size());
assertEquals("this", terms.get(0));
assertEquals("is", terms.get(1));
assertEquals("full", terms.get(2));
assertEquals("width", terms.get(3));
assertEquals("sentences", terms.get(4));
/*
* Legacy can avoid Stopward if it is full width.
* It is unclear whether it is a specification or not.
* (Problems due to a defect in filter application order?
* or
* Is it popular in English speaking countries?)
*/
String queryFullWidth = "ＴＨＩＳ ＩＳ ＦＵＬＬ－ＷＩＤＴＨ ＳＥＮＴＥＮＣＥＳ.";
terms = toTermString(queryFullWidth);
/*
* XXX 3.x -> 8.x :
*
* This is not a change due to the library but an intentional change.
* The filter order has been changed properly
* as it is probably not a deliberate specification.
*/
assertEquals(5, terms.size());
assertEquals("this", terms.get(0));
assertEquals("is", terms.get(1));
assertEquals("full", terms.get(2));
assertEquals("width", terms.get(3));
assertEquals("sentences", terms.get(4));
}
/**
* Tests on ligature and diacritical marks.
* In UAX#29, determination of non-practical word boundaries is not considered.
* Languages that use special strings require "practical word" sample.
* Unit testing with only ligature and diacritical marks is not possible.
*/
@Test
public void testAsciiFoldingStop() {
String queryLigature = "Cæsar";
String expectedLigature = "caesar";
String queryDiacritical = "Café";
String expectedDiacritical = "cafe";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> termsLigature = toTermString(n, queryLigature);
List<String> termsDiacritical = toTermString(n, queryDiacritical);
switch (n) {
/*
* It is decomposed into the expected string.
*/
case FieldNames.FOLDER:
case FieldNames.MEDIA_TYPE:
case FieldNames.GENRE:
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("Cæsar : " + n, 1, termsLigature.size());
assertEquals("Cæsar : " + n, expectedLigature, termsLigature.get(0));
assertEquals("Café : " + n, 1, termsDiacritical.size());
assertEquals("Café : " + n, expectedDiacritical, termsDiacritical.get(0));
break;
// Legacy has common behavior for all fields.
default:
assertEquals("Cæsar : " + n, 1, termsLigature.size());
assertEquals("Cæsar : " + n, expectedLigature, termsLigature.get(0));
assertEquals("Café : " + n, 1, termsDiacritical.size());
assertEquals("Café : " + n, expectedDiacritical, termsDiacritical.get(0));
break;
}
});
}
/**
* Detailed tests on LowerCase.
*/
@Test
public void testLowerCase() {
// Filter operation check only. Verify only some settings.
String query = "ABCDEFG";
String expected = "abcdefg";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> terms = toTermString(n, query);
switch (n) {
/*
* In legacy, it is converted to lower. (over-processed?)
*/
case FieldNames.FOLDER:
case FieldNames.MEDIA_TYPE:
assertEquals("lower : " + n, 1, terms.size());
assertEquals("lower : " + n, expected, terms.get(0));
break;
/*
* These are searchable fields in lower case.
*/
case FieldNames.GENRE:
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("lower : " + n, 1, terms.size());
assertEquals("lower : " + n, expected, terms.get(0));
break;
// Legacy has common behavior for all fields.
default:
assertEquals("lower : " + n, 1, terms.size());
assertEquals("lower : " + n, expected, terms.get(0));
break;
}
});
}
/**
* Detailed tests on EscapeRequires.
* The reserved string is discarded unless it is purposely Escape.
* This is fine as a search specification(if it is considered as a kind of reserved stop word).
* However, in the case of file path, it may be a problem.
*/
@Test
public void testLuceneEscapeRequires() {
String queryEscapeRequires = "+-&&||!(){}[]^\"~*?:\\/";
String queryFileUsable = "+-&&!(){}[]^~";
Arrays.stream(IndexType.values()).flatMap(i -> Arrays.stream(i.getFields())).forEach(n -> {
List<String> terms = toTermString(n, queryEscapeRequires);
switch (n) {
/*
* Will be removed. (Can not distinguish the directory of a particular pattern?)
*/
case FieldNames.FOLDER:
assertEquals("escape : " + n, 0, terms.size());
terms = toTermString(n, queryFileUsable);
assertEquals("escape : " + n, 0, terms.size());
break;
/*
* Will be removed.
*/
case FieldNames.MEDIA_TYPE:
case FieldNames.GENRE:
case FieldNames.ARTIST:
case FieldNames.ALBUM:
case FieldNames.TITLE:
assertEquals("escape : " + n, 0, terms.size());
break;
// Will be removed.
default:
assertEquals("escape : " + n, 0, terms.size());
break;
}
});
}
/**
* Create an example that makes UAX 29 differences easy to understand.
*/
@Test
public void testUax29() {
/*
* Case using test resource name
*/
// Semicolon, comma and hyphen.
String query = "Bach: Goldberg Variations, BWV 988 - Aria";
List<String> terms = toTermString(query);
assertEquals(6, terms.size());
assertEquals("bach", terms.get(0));
assertEquals("goldberg", terms.get(1));
assertEquals("variations", terms.get(2));
assertEquals("bwv", terms.get(3));
assertEquals("988", terms.get(4));
assertEquals("aria", terms.get(5));
// Underscores around words, ascii and semicolon.
query = "_ID3_ARTIST_ Céline Frisch: Café Zimmermann";
terms = toTermString(query);
assertEquals(5, terms.size());
/*
* XXX 3.x -> 8.x : _id3_artist_ in UAX#29.
* Since the effect is large, trim with Filter.
*/
assertEquals("id3_artist", terms.get(0));
assertEquals("celine", terms.get(1));
assertEquals("frisch", terms.get(2));
assertEquals("cafe", terms.get(3));
assertEquals("zimmermann", terms.get(4));
// Underscores around words and slashes.
query = "_ID3_ARTIST_ Sarah Walker/Nash Ensemble";
terms = toTermString(query);
assertEquals(5, terms.size());
/*
* XXX 3.x -> 8.x : _id3_artist_ in UAX#29.
* Since the effect is large, trim with Filter.
*/
assertEquals("id3_artist", terms.get(0));
assertEquals("sarah", terms.get(1));
assertEquals("walker", terms.get(2));
assertEquals("nash", terms.get(3));
assertEquals("ensemble", terms.get(4));
// Space
assertEquals(asList("abc", "def"), toTermString(" ABC DEF "));
assertEquals(asList("abc1", "def"), toTermString(" ABC1 DEF "));
// trim and delimiter
assertEquals(asList("abc", "def"), toTermString("+ABC+DEF+"));
assertEquals(asList("abc", "def"), toTermString("|ABC|DEF|"));
assertEquals(asList("abc", "def"), toTermString("!ABC!DEF!"));
assertEquals(asList("abc", "def"), toTermString("(ABC(DEF("));
assertEquals(asList("abc", "def"), toTermString(")ABC)DEF)"));
assertEquals(asList("abc", "def"), toTermString("{ABC{DEF{"));
assertEquals(asList("abc", "def"), toTermString("}ABC}DEF}"));
assertEquals(asList("abc", "def"), toTermString("[ABC[DEF["));
assertEquals(asList("abc", "def"), toTermString("]ABC]DEF]"));
assertEquals(asList("abc", "def"), toTermString("^ABC^DEF^"));
assertEquals(asList("abc", "def"), toTermString("\\ABC\\DEF\\"));
assertEquals(asList("abc", "def"), toTermString("\"ABC\"DEF\""));
assertEquals(asList("abc", "def"), toTermString("~ABC~DEF~"));
assertEquals(asList("abc", "def"), toTermString("*ABC*DEF*"));
assertEquals(asList("abc", "def"), toTermString("?ABC?DEF?"));
assertEquals(asList("abc:def"), toTermString(":ABC:DEF:")); // XXX 3.x -> 8.x : abc def -> abc:def
assertEquals(asList("abc", "def"), toTermString("-ABC-DEF-"));
assertEquals(asList("abc", "def"), toTermString("/ABC/DEF/"));
/*
* XXX 3.x -> 8.x : _abc_def_ in UAX#29.
* Since the effect is large, trim with Filter.
*/
assertEquals(asList("abc_def"), toTermString("_ABC_DEF_")); // XXX 3.x -> 8.x : abc def -> abc_def
assertEquals(asList("abc", "def"), toTermString(",ABC,DEF,"));
assertEquals(asList("abc.def"), toTermString(".ABC.DEF."));
assertEquals(asList("abc", "def"), toTermString("&ABC&DEF&")); // XXX 3.x -> 8.x : abc&def -> abc def
assertEquals(asList("abc", "def"), toTermString("@ABC@DEF@")); // XXX 3.x -> 8.x : abc@def -> abc def
assertEquals(asList("abc'def"), toTermString("'ABC'DEF'"));
// trim and delimiter and number
assertEquals(asList("abc1", "def"), toTermString("+ABC1+DEF+"));
assertEquals(asList("abc1", "def"), toTermString("|ABC1|DEF|"));
assertEquals(asList("abc1", "def"), toTermString("!ABC1!DEF!"));
assertEquals(asList("abc1", "def"), toTermString("(ABC1(DEF("));
assertEquals(asList("abc1", "def"), toTermString(")ABC1)DEF)"));
assertEquals(asList("abc1", "def"), toTermString("{ABC1{DEF{"));
assertEquals(asList("abc1", "def"), toTermString("}ABC1}DEF}"));
assertEquals(asList("abc1", "def"), toTermString("[ABC1[DEF["));
assertEquals(asList("abc1", "def"), toTermString("]ABC1]DEF]"));
assertEquals(asList("abc1", "def"), toTermString("^ABC1^DEF^"));
assertEquals(asList("abc1", "def"), toTermString("\\ABC1\\DEF\\"));
assertEquals(asList("abc1", "def"), toTermString("\"ABC1\"DEF\""));
assertEquals(asList("abc1", "def"), toTermString("~ABC1~DEF~"));
assertEquals(asList("abc1", "def"), toTermString("*ABC1*DEF*"));
assertEquals(asList("abc1", "def"), toTermString("?ABC1?DEF?"));
assertEquals(asList("abc1", "def"), toTermString(":ABC1:DEF:"));
assertEquals(asList("abc1", "def"), toTermString(",ABC1,DEF,")); // XXX 3.x -> 8.x : abc1,def -> abc1 def
assertEquals(asList("abc1", "def"), toTermString("-ABC1-DEF-")); // XXX 3.x -> 8.x : abc1-def -> abc1 def
assertEquals(asList("abc1", "def"), toTermString("/ABC1/DEF/")); // XXX 3.x -> 8.x : abc1/def -> abc1 def
/*
* XXX 3.x -> 8.x : _abc1_def_ in UAX#29.
* Since the effect is large, trim with Filter.
*/
assertEquals(asList("abc1_def"), toTermString("_ABC1_DEF_"));
assertEquals(asList("abc1", "def"), toTermString(".ABC1.DEF.")); // XXX 3.x -> 8.x : abc1.def -> abc1 def
assertEquals(asList("abc1", "def"), toTermString("&ABC1&DEF&"));
assertEquals(asList("abc1", "def"), toTermString("@ABC1@DEF@"));
assertEquals(asList("abc1", "def"), toTermString("'ABC1'DEF'"));
}
/**
* Special handling of single quotes.
*/
@Test
public void testSingleQuotes() {
/*
* A somewhat cultural that seems to be related to a specific language.
*/
String query = "This is Airsonic's analysis.";
List<String> terms = toTermString(query);
assertEquals(4, terms.size());
assertEquals("this", terms.get(0));// Not deleted because it is not a stopword
assertEquals("is", terms.get(1));// Not deleted because it is not a stopword
assertEquals("airsonic", terms.get(2));
assertEquals("analysis", terms.get(3));
/*
* XXX 3.x -> 8.x :
* we ve -> we've
*/
query = "We’ve been here before.";
terms = toTermString(query);
assertEquals(4, terms.size());
assertEquals("we've", terms.get(0));
assertEquals("been", terms.get(1));
assertEquals("here", terms.get(2));
assertEquals("before", terms.get(3));
query = "LʼHomme";
terms = toTermString(query);
assertEquals(1, terms.size());
assertEquals("lʼhomme", terms.get(0));
query = "L'Homme";
terms = toTermString(query);
assertEquals(1, terms.size());
assertEquals("l'homme", terms.get(0));
query = "aujourd'hui";
terms = toTermString(query);
assertEquals(1, terms.size());
assertEquals("aujourd'hui", terms.get(0));
query = "fo'c'sle";
terms = toTermString(query);
assertEquals(1, terms.size());
assertEquals("fo'c'sle", terms.get(0));
}
/*
* There is also a filter that converts the tense to correspond to the search by the present
* tense.
*/
@Test
public void testPastParticiple() {
/*
* Confirming no conversion to present tense.
*/
String query = "This is formed with a form of the verb \"have\" and a past participl.";
List<String> terms = toTermString(query);
assertEquals(11, terms.size());
assertEquals("this", terms.get(0));// Not deleted because it is not a stopword
assertEquals("is", terms.get(1));// Not deleted because it is not a stopword
assertEquals("formed", terms.get(2));// leave passive / not "form"
assertEquals("with", terms.get(3));// Not deleted because it is not a stopword
assertEquals("form", terms.get(4));
assertEquals("of", terms.get(5));
assertEquals("verb", terms.get(6));
assertEquals("have", terms.get(7));
assertEquals("and", terms.get(8));// Not deleted because it is not a stopword
assertEquals("past", terms.get(9));
assertEquals("participl", terms.get(10));
}
/*
* There are also filters that convert plurals to singular.
*/
@Test
public void testNumeral() {
/*
* Confirming no conversion to singular.
*/
String query = "books boxes cities leaves men glasses";
List<String> terms = toTermString(query);
assertEquals(6, terms.size());
assertEquals("books", terms.get(0));// leave numeral / not singular
assertEquals("boxes", terms.get(1));
assertEquals("cities", terms.get(2));
assertEquals("leaves", terms.get(3));
assertEquals("men", terms.get(4));
assertEquals("glasses", terms.get(5));
}
@Test
public void testGenre() {
/*
* Confirming no conversion to singular.
*/
String query = "{}";
List<String> terms = toQueryTermString(FieldNames.GENRE, query);
assertEquals(1, terms.size());
assertEquals("{ }", terms.get(0));
}
private List<String> toTermString(String str) {
return toTermString(null, str);
}
private List<String> toTermString(String field, String str) {
List<String> result = new ArrayList<>();
try {
TokenStream stream = analyzerFactory.getAnalyzer().tokenStream(field,
new StringReader(str));
stream.reset();
while (stream.incrementToken()) {
result.add(stream.getAttribute(CharTermAttribute.class).toString()
.replaceAll("^term\\=", ""));
}
stream.close();
} catch (IOException e) {
LoggerFactory.getLogger(AnalyzerFactoryTestCase.class)
.error("Error during Token processing.", e);
}
return result;
}
/*
* Should be added in later versions.
*/
public void testWildCard() {
}
@SuppressWarnings("unused")
private List<String> toQueryTermString(String field, String str) {
List<String> result = new ArrayList<>();
try {
TokenStream stream = analyzerFactory.getQueryAnalyzer().tokenStream(field,
new StringReader(str));
stream.reset();
while (stream.incrementToken()) {
result.add(stream.getAttribute(CharTermAttribute.class).toString()
.replaceAll("^term\\=", ""));
}
stream.close();
} catch (IOException e) {
LoggerFactory.getLogger(AnalyzerFactoryTestCase.class)
.error("Error during Token processing.", e);
}
return result;
}
}
