opennlp.tools.cmdline.parser.ParserTool Java Examples

public double scoreStructure(String ca, String q, String passage, boolean verbose) throws InvalidFormatException, IOException{
	POSTaggerME parserModel = new POSTaggerME(new POSModel(new FileInputStream(new File("en-pos-model.bin"))));
	Tokenizer tokenizer = new TokenizerME(new TokenizerModel(new FileInputStream(new File("en-token.bin"))));
	Parser parser = ParserFactory.create(new ParserModel(new FileInputStream(new File("en-parser.bin"))));
	double score = 0;
	
	Parse[] questionParse = ParserTool.parseLine(q, parser, 1);
	Parse[] passageParse = ParserTool.parseLine(q, parser, 1);
	
	if (passage.contains(ca)) {
		for (int i =0; i < questionParse.length; i++) {
			score += matchChildren(questionParse[i],passageParse[i]);
		}
	}
	
	return score;
}
 

public void parserTest1() throws IOException {
	if (!this.modelsAreInitialized) init();
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); 
	Parse[] results = ParserTool.parseLine("Jane Austen was very modest about her own genius ."+this.q,
			parser, 1);
	Parse[] qResults = ParserTool.parseLine(this.q,parser, 1);
	Parse[] rChn = (results[0].getChildren())[0].getChildren();
	
	results[0].expandTopNode(results[0]);
	for (int i = 0; i < results.length; i++) {
		results[i].show();
	}
	for (int i = 0; i < qResults.length; i++) {
		qResults[i].show();
	}
	System.out.print("\n\n");
	for (int i = 0; i < rChn.length; i++) {
		rChn[i].show();
		System.out.print("\n");
	}
}
 

public static Parse[] parsePassageText(String p) throws InvalidFormatException{
	
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(sentenceModel);
	Parser parser = ParserFactory.create(
			parserModel,
			20, // beam size
			0.95); // advance percentage
 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);


		String sent= StringUtils.join(tks," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

public void parserTest1() throws IOException {
	if (!this.modelsAreInitialized) init();
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); 
	Parse[] results = ParserTool.parseLine("Jane Austen was very modest about her own genius ."+this.q,
			parser, 1);
	Parse[] qResults = ParserTool.parseLine(this.q,parser, 1);
	Parse[] rChn = (results[0].getChildren())[0].getChildren();
	
	results[0].expandTopNode(results[0]);
	for (int i = 0; i < results.length; i++) {
		results[i].show();
	}
	for (int i = 0; i < qResults.length; i++) {
		qResults[i].show();
	}
	System.out.print("\n\n");
	for (int i = 0; i < rChn.length; i++) {
		rChn[i].show();
		System.out.print("\n");
	}
}
 

public Parse[] parsePassageText(String p) throws InvalidFormatException{
	if (!modelsAreInitialized)init();
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(this.sentenceModel);
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); // advance percentage
	//find sentences, tokenize each, parse each, return top parse for each 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best

		String sent= StringUtils.join(tks," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

private static void usingOpenNLP() {
        String fileLocation = getModelDir() + "/en-parser-chunking.bin";
        System.out.println(fileLocation);
        try (InputStream modelInputStream = new FileInputStream(fileLocation);) {
            ParserModel model = new ParserModel(modelInputStream);
            Parser parser = ParserFactory.create(model);
            String sentence = "The cow jumped over the moon";
            // Used to demonstrate difference between NER and Parser
            sentence = "He was the last person to see Fred.";

            Parse parses[] = ParserTool.parseLine(sentence, parser, 3);
            for (Parse parse : parses) {
                // First display
                parse.show();
                // Second display
//                parse.showCodeTree();
                // Third display
//                System.out.println("Children");
//                Parse children[] = parse.getChildren();
//                for (Parse parseElement : children) {
//                    System.out.println(parseElement);
//                    System.out.println(parseElement.getText());
//                    System.out.println(parseElement.getType());
//                    Parse tags[] = parseElement.getTagNodes();
//                    System.out.println("Tags");
//                    for (Parse tag : tags) {
//                        System.out.println("[" + tag + "]" + " type: " + tag.getType()
//                                + "  Probability: " + tag.getProb()
//                                + "  Label: " + tag.getLabel());
//                    }
//                }
            }
        } catch (IOException ex) {
            ex.printStackTrace();
        }
    }
 

public static void main(String args[]) throws IOException
{
	String wordnetDir = System.getProperty("wordnet.dir");
	//wordnetDir="WordNet-3.0/dict/";
	String question="Who is Abraham Lincoln?";
	AnswerTypeContextGenerator atcg=new AnswerTypeContextGenerator(new File(wordnetDir));
	String q=null;
    String modelsDirProp = System.getProperty("model.dir");
   // modelsDirProp="opennlp-models/";
    File modelsDir = new File(modelsDirProp);
    InputStream chunkerStream = new FileInputStream(
        new File(modelsDir,"en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
        new File(modelsDir,"en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger =  new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    
    Parse query = ParserTool.parseLine(question,parser,1)[0];
	String[] context=atcg.getContext(query);
	for(int i=0;i<context.length;i++)
	{
		if(context[i].startsWith("hw=") || context[i].startsWith("mw="))
		{
			System.out.println(context[i].substring(3));
		}
	}
}
 

public String[] getFocusNoun(String question) throws IOException
{
	String wordnetDir = System.getProperty("wordnet.dir");
	wordnetDir="WordNet-3.0/dict/";
	AnswerTypeContextGenerator atcg=new AnswerTypeContextGenerator(new File(wordnetDir));
	String q=null;
    String modelsDirProp = System.getProperty("model.dir");
    modelsDirProp="opennlp-models/";
    File modelsDir = new File(modelsDirProp);
    InputStream chunkerStream = new FileInputStream(
        new File(modelsDir,"en-chunker.bin"));
    ChunkerModel chunkerModel = new ChunkerModel(chunkerStream);
    ChunkerME chunker = new ChunkerME(chunkerModel);
    InputStream posStream = new FileInputStream(
        new File(modelsDir,"en-pos-maxent.bin"));
    POSModel posModel = new POSModel(posStream);
    POSTaggerME tagger =  new POSTaggerME(posModel);
    Parser parser = new ChunkParser(chunker, tagger);
    
    Parse query = ParserTool.parseLine(question,parser,1)[0];
	String[] context=atcg.getContext(query);
	String[] focus=new String[2];
	int p=0;
	for(int i=0;i<context.length;i++)
	{
		if(context[i].startsWith("hw=") || context[i].startsWith("mw="))
		{
			//System.out.println(context[i].substring(3));
			focus[p++]=context[i].substring(3);
		}
	}
	return focus;
}
 

public Event next() {
    int split = line.indexOf(' ');
    String outcome = line.substring(0, split);
    String question = line.substring(split + 1);
    Parse query = ParserTool.parseLine(question, parser, 1)[0];
    return (new Event(outcome, atcg.getContext(query)));
}
 

public Parse parseSentence(String sentence){
	Parse topParses[] = ParserTool.parseLine(sentence, parser, 1);
	if (topParses.length == 0)
		return null;
	else 
		return topParses[0];
}
 

/** Turn one tokenized sentence into one top-ranked parse tree. */
public Parse parseSentence(List<String> tokens) {
	//StringTokenizer st = new StringTokenizer(tks[i]); 
	//There are several tokenizers available. SimpleTokenizer works best
	System.out.print(";");
	String sent= StringUtils.join(tokens," ");
	return ParserTool.parseLine(sent,parser, 1)[0];
}
 

public Parse[] parsePassageText(String p) throws InvalidFormatException{
	if (!modelsAreInitialized)init();
	//initialize 	 
	SentenceDetectorME sentenceDetector = new SentenceDetectorME(this.sentenceModel);
	NameFinderME nameFinder = new NameFinderME(this.nerModel);
	Parser parser = ParserFactory.create(
			this.parserModel,
			20, // beam size
			0.95); // advance percentage
	//find sentences, tokenize each, parse each, return top parse for each 	 	 
	String[] sentences = sentenceDetector.sentDetect(p);
	Parse[] results = new Parse[sentences.length];
	for (int i=0;i<sentences.length;i++){
		//String[] tks = SimpleTokenizer.INSTANCE.tokenize(sentences[i]);
		
		//StringTokenizer st = new StringTokenizer(tks[i]); 
		//There are several tokenizers available. SimpleTokenizer works best
		Tokenizer tokenizer = SimpleTokenizer.INSTANCE;
		for (int si = 0; si < sentences.length; si++) {
	        Span[] tokenSpans = tokenizer.tokenizePos(sentences[si]);
	        String[] tokens = Span.spansToStrings(tokenSpans, sentences[si]);
	        Span[] names = nameFinder.find(tokens);
	        for (int ni = 0; ni < names.length; ni++) {
	            Span startSpan = tokenSpans[names[ni].getStart()];
	            int nameStart = startSpan.getStart();
	            Span endSpan = tokenSpans[names[ni].getEnd() - 1];
	            int nameEnd = endSpan.getEnd();
	            String name = sentences[si].substring(nameStart, nameEnd);
	            System.out.println(name);
	        }
	    }
		String sent= StringUtils.join(tokenizer," ");
		System.out.println("Found sentence " + sent);
		Parse[] sentResults = ParserTool.parseLine(sent,parser, 1);
		results[i]=sentResults[0];
	}
	return results;
}
 

@Override
public double scorePassage(Phrase q, Answer a, Passage p) {
	
	int countOfQuestionNPsInPassage = 0;
	try {
		//prep NLP tools
		if (!this.modelsAreInitialized) init();
		Parser parser = ParserFactory.create(this.parserModel, 20, 0.95);

		//create question parse
		Parse[] questionParse = ParserTool.parseLine(q.text, parser, 1);

		//create passage parses (one for each sentence)
		String[] passageSentences = this.DivideIntoSentences(p);
		Parse[] passageParses = new Parse[passageSentences.length];
		Parse[] tempParse;
		for (int i=0; i < passageSentences.length; i++) {
			tempParse = ParserTool.parseLine(passageSentences[i], parser, 1);
			passageParses[i] = tempParse[0];
		}
		
		//retrieve NPs from the question parse
		navigateTree(questionParse, 0, questionNPs);

		//retrieve NPs from the passage parse
		for (int i=0; i < passageParses.length; i++) {
			navigateTree(passageParses, i, passageNPs);				
		}
		
		//count the number of question NPs that are in the passage NP set (A)
		for (String qNP: questionNPs) {
			for (String pNP: passageNPs) {
				//System.out.println("comparing " + qNP + " with " + pNP);
				if (qNP.equals(pNP)) {
					//System.out.println("match found");
					countOfQuestionNPsInPassage++;
				}
			}
		}
		//System.out.println(countOfQuestionNPsInPassage);
		
		//count the number of all NPs that are in the passage NP set (B)
		//passageNPs.size();
		
	} catch (InvalidFormatException e) {
		e.printStackTrace();
	}

	//calculate A/B and return as the score
	//System.out.print("******** score:  " + (double)countOfQuestionNPsInPassage/passageNPs.size() + "  *******");
	//System.out.println(" count:  " + passageNPs.size() + "  *******");
	if (passageNPs.size() == 0)
		return 0;
	else
		return (double)countOfQuestionNPsInPassage/passageNPs.size();
}