com.aliasi.tokenizer.IndoEuropeanTokenizerFactory Java Examples

public static void main(String[] args) throws IOException {
        File corpusFile = new File("inputfile.txt");// my annotated file
        File modelFile = new File("outputmodelfile.model"); 

        System.out.println("Setting up Chunker Estimator");
        TokenizerFactory factory
            = IndoEuropeanTokenizerFactory.INSTANCE;
        HmmCharLmEstimator hmmEstimator
            = new HmmCharLmEstimator(MAX_N_GRAM,NUM_CHARS,LM_INTERPOLATION);
        CharLmHmmChunker chunkerEstimator
            = new CharLmHmmChunker(factory,hmmEstimator);

        System.out.println("Setting up Data Parser");
//        Muc6ChunkParser parser = new Muc6ChunkParser();  
//        parser.setHandler( chunkerEstimator);

        System.out.println("Training with Data from File=" + corpusFile);
//        parser.parse(corpusFile);

        System.out.println("Compiling and Writing Model to File=" + modelFile);
        AbstractExternalizable.compileTo(chunkerEstimator,modelFile);
    }
 

private static void usingTheLingPipeStemmer() {
    String words[] = {"bank", "banking", "banks", "banker",
        "banked", "bankart"};
    TokenizerFactory tokenizerFactory
            = IndoEuropeanTokenizerFactory.INSTANCE;
    TokenizerFactory porterFactory
            = new PorterStemmerTokenizerFactory(tokenizerFactory);
    String[] stems = new String[words.length];
    for (int i = 0; i < words.length; i++) {
        com.aliasi.tokenizer.Tokenization tokenizer
                = new com.aliasi.tokenizer.Tokenization(words[i], porterFactory);
        stems = tokenizer.tokens();
        System.out.print("Word: " + words[i]);
        for (String stem : stems) {
            System.out.println("  Stem: " + stem);
        }
    }

}
 

@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  // initialize sentence chunker
  TokenizerFactory tokenizerFactory = UimaContextHelper.createObjectFromConfigParameter(context,
          "tokenizer-factory", "tokenizer-factory-params", IndoEuropeanTokenizerFactory.class,
          TokenizerFactory.class);
  SentenceModel sentenceModel = UimaContextHelper.createObjectFromConfigParameter(context,
          "sentence-model", "sentence-model-params", IndoEuropeanSentenceModel.class,
          SentenceModel.class);
  chunker = new SentenceChunker(tokenizerFactory, sentenceModel);
  // initialize hits
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 200);
  // initialize query analyzer, index writer config, and query parser
  analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  parser = new QueryParser("text", analyzer);
  // initialize query string constructor
  queryStringConstructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          BooleanBagOfPhraseQueryStringConstructor.class, QueryStringConstructor.class);
}
 

@Override
public void initialize(UimaContext context) throws ResourceInitializationException {
  super.initialize(context);
  TokenizerFactory tokenizerFactory = UimaContextHelper.createObjectFromConfigParameter(context,
          "tokenizer-factory", "tokenizer-factory-params", IndoEuropeanTokenizerFactory.class,
          TokenizerFactory.class);
  SentenceModel sentenceModel = UimaContextHelper.createObjectFromConfigParameter(context,
          "sentence-model", "sentence-model-params", IndoEuropeanSentenceModel.class,
          SentenceModel.class);
  chunker = new SentenceChunker(tokenizerFactory, sentenceModel);
  // initialize hits
  hits = UimaContextHelper.getConfigParameterIntValue(context, "hits", 200);
  // initialize query analyzer, index writer config, and query parser
  analyzer = UimaContextHelper.createObjectFromConfigParameter(context, "query-analyzer",
          "query-analyzer-params", StandardAnalyzer.class, Analyzer.class);
  parser = new QueryParser("text", analyzer);
  // initialize query string constructor
  queryStringConstructor = UimaContextHelper.createObjectFromConfigParameter(context,
          "query-string-constructor", "query-string-constructor-params",
          BagOfPhraseQueryStringConstructor.class, QueryStringConstructor.class);
  String parserProviderName = UimaContextHelper
          .getConfigParameterStringValue(context, "parser-provider");
  parserProvider = ProviderCache.getProvider(parserProviderName, ParserProvider.class);

  lemma = new StanfordLemmatizer();
}
 

private static void usingExactDictionaryChunker() {
    initializeDictionary();
    System.out.println("\nDICTIONARY\n" + dictionary);

    ExactDictionaryChunker dictionaryChunker
            = new ExactDictionaryChunker(dictionary,
                    IndoEuropeanTokenizerFactory.INSTANCE, true, false);

    for (String sentence : sentences) {
        System.out.println("\nTEXT=" + sentence);
        displayChunkSet(dictionaryChunker, sentence);
    }
}
 

private static void usingExactDictionaryChunker() {
    initializeDictionary();
    System.out.println("\nDICTIONARY\n" + dictionary);

    ExactDictionaryChunker dictionaryChunker
            = new ExactDictionaryChunker(dictionary,
                    IndoEuropeanTokenizerFactory.INSTANCE, true, false);

    for (String sentence : sentences) {
        System.out.println("\nTEXT=" + sentence);
        displayChunkSet(dictionaryChunker, sentence);
    }
}
 

private static void lingpipeExamples() {
    List<String> tokenList = new ArrayList<>();
    List<String> whiteList = new ArrayList<>();
    String text = "A sample sentence processed \nby \tthe "
            + "LingPipe tokenizer.";
    com.aliasi.tokenizer.Tokenizer tokenizer = IndoEuropeanTokenizerFactory.INSTANCE.
            tokenizer(text.toCharArray(), 0, text.length());
    tokenizer.tokenize(tokenList, whiteList);
    for (String element : tokenList) {
        System.out.print(element + " ");
    }
    System.out.println();

}
 

private static void usingLingPipeTokenizers() {
//        String paragraph = "sample text string";
        char text[] = paragraph.toCharArray();
        TokenizerFactory tokenizerFactory = IndoEuropeanTokenizerFactory.INSTANCE;
        com.aliasi.tokenizer.Tokenizer tokenizer = tokenizerFactory.tokenizer(
                text, 0, text.length);
        for (String token : tokenizer) {
            System.out.println(token);
        }
    }
 

public TweetHandler removeStopWords() {
    TokenizerFactory tokenizerFactory
            = IndoEuropeanTokenizerFactory.INSTANCE;
    tokenizerFactory = new EnglishStopTokenizerFactory(tokenizerFactory);
    Tokenizer tokens = tokenizerFactory.tokenizer(
            this.text.toCharArray(), 0, this.text.length());
    StringBuilder buffer = new StringBuilder();
    for (String word : tokens) {
        buffer.append(word + " ");
    }
    this.text = buffer.toString();
    return this;
}
 

public static void removeStopWithLing(String text){
	//******************EXAMPLE WITH ling pipe *******************************************************************************************
	//mention lower vs upper case
	out.println(text);
	text = text.toLowerCase().trim();
	TokenizerFactory fact = IndoEuropeanTokenizerFactory.INSTANCE;
	fact = new EnglishStopTokenizerFactory(fact);
	Tokenizer tok = fact.tokenizer(text.toCharArray(), 0, text.length());
	for(String word : tok){
		out.print(word + " ");
	}
}
 

public static void main(String[] args) {

        MapDictionary<String> dictionary = new MapDictionary<String>();
        dictionary.addEntry(new DictionaryEntry<String>("Joe","PERSON",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("Fred","PERSON",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("Boston","PLACE",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("pub","PLACE",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("Vermont","PLACE",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("IBM","ORGANIZATION",CHUNK_SCORE));
        dictionary.addEntry(new DictionaryEntry<String>("Sally","PERSON",CHUNK_SCORE));


        ExactDictionaryChunker dictionaryChunkerTT
            = new ExactDictionaryChunker(dictionary,
                                         IndoEuropeanTokenizerFactory.INSTANCE,
                                         true,true);

        ExactDictionaryChunker dictionaryChunkerTF
            = new ExactDictionaryChunker(dictionary,
                                         IndoEuropeanTokenizerFactory.INSTANCE,
                                         true,false);

        ExactDictionaryChunker dictionaryChunkerFT
            = new ExactDictionaryChunker(dictionary,
                                         IndoEuropeanTokenizerFactory.INSTANCE,
                                         false,true);

        ExactDictionaryChunker dictionaryChunkerFF
            = new ExactDictionaryChunker(dictionary,
                                         IndoEuropeanTokenizerFactory.INSTANCE,
                                         false,false);



        System.out.println("\nDICTIONARY\n" + dictionary);

        for (int i = 0; i < sentences.length; ++i) {
            String text = sentences[i];
            System.out.println("\n\nTEXT=" + text);

            chunk(dictionaryChunkerTT,text);
            chunk(dictionaryChunkerTF,text);
            chunk(dictionaryChunkerFT,text);
            chunk(dictionaryChunkerFF,text);
        }

    }