There are 11 code examples for org.apache.lucene.analysis.TokenStream.

The API names are highlighted below. You can use button to vote the code example(s) you like. The best code example will be ranked first next time. Thanks a lot for your feedback.

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: SynonymFilter.java (Click to view .java file)

Method Code:

/** 
 * this allows to add synonyms at indexing time
 */
public Token next() throws IOException {
  if (synonymStack.size() > 0) {
    return synonymStack.pop();
  }
  Token token=input.next(new Token());
  if (token == null) {
    return null;
  }
  addAliasesToStack(token);
  return token;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: LemmaAnalyzer.java (Click to view .java file)

Method Code:

public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=new LemmaFilter(new StopFilter(new LowerCaseFilter(new StandardFilter(new StandardTokenizer(reader))),StandardAnalyzer.STOP_WORDS),engine);
  return result;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: SynonymAnalyzer.java (Click to view .java file)

Method Code:

public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=new SynonymFilter(new LemmaFilter(new StopFilter(new LowerCaseFilter(new StandardFilter(new StandardTokenizer(reader))),StandardAnalyzer.STOP_WORDS),engine),engine,ProcessQueryFlag);
  return result;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: CoLocationFilter.java (Click to view .java file)

Method Code:

public CoLocationFilter(TokenStream in,NaturalLanguageEngine engine){
  super(in);
  this.engine=engine;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: BufferdFilter.java (Click to view .java file)

Method Code:

protected boolean addTokenToBuffer() throws IOException {
  Token token=input.next();
  if (token != null) {
    buffer.add(token);
    return true;
  }
  return false;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: LemmaFilter.java (Click to view .java file)

Method Code:

/** 
 * this allows to add synonyms at indexing time
 */
public Token next() throws IOException {
  if (lemmaStack.size() > 0) {
    return lemmaStack.pop();
  }
  Token token=input.next(new Token());
  if (token == null) {
    return null;
  }
  addAliasesToStack(token);
  return token;
}

Project Name: hnlpengine-luceneanalyzer Package: analysis

Source Code: CoLocationAnalyzer.java (Click to view .java file)

Method Code:

public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=new CoLocationFilter(new LemmaFilter(new StopFilter(new LowerCaseFilter(new StandardFilter(new StandardTokenizer(reader))),StandardAnalyzer.STOP_WORDS),engine),engine);
  return result;
}

Project Name: hnlpengine-nlpengine Package: utils

Source Code: tokensUtils.java (Click to view .java file)

Method Code:

public static Token[] tokensFromAnalysis(Analyzer analyzer,String text) throws IOException {
  TokenStream stream=analyzer.tokenStream("contents",new StringReader(text));
  ArrayList<Token> tokenList=new ArrayList<Token>();
  while (true) {
    Token token=stream.next();
    if (token == null)     break;
    tokenList.add(token);
  }
  return (Token[])tokenList.toArray(new Token[0]);
}

Project Name: rssowl.core Package: org.rssowl.core.internal.persist.search

Source Code: Indexer.java (Click to view .java file)

Method Code:

@Override public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=super.tokenStream(fieldName,reader);
  result=new LowerCaseFilter(result);
  return result;
}

Project Name: rssowl.core Package: org.rssowl.core.internal.persist.search

Source Code: LowercaseWhitespaceAnalyzer.java (Click to view .java file)

Method Code:

@Override public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=new WhitespaceTokenizer(reader);
  result=new LowerCaseFilter(result);
  return result;
}

Project Name: rssowl.core Package: org.rssowl.core.internal.persist.search

Source Code: LowercaseDelimiterAnalyzer.java (Click to view .java file)

Method Code:

@Override public TokenStream tokenStream(String fieldName,Reader reader){
  TokenStream result=new CharTokenizer(reader){
    @Override protected boolean isTokenChar(    char c){
      return c != fDelim;
    }
  }
;
  result=new LowerCaseFilter(result);
  return result;
}