Java Code Examples for edu.stanford.nlp.ie.AbstractSequenceClassifier#classifyToCharacterOffsets()

The following examples show how to use edu.stanford.nlp.ie.AbstractSequenceClassifier#classifyToCharacterOffsets() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: StanfordExtractorTest.java    From CLAVIN-NERD with GNU General Public License v2.0 6 votes vote down vote up
/**
 * Checks conversion of Stanford NER output format into
 * {@link com.bericotech.clavin.resolver.ClavinLocationResolver}
 * input format.
 *
 * @throws IOException
 */
@Test
public void testConvertNERtoCLAVIN() throws IOException {
    InputStream mpis = this.getClass().getClassLoader().getResourceAsStream("models/english.all.3class.distsim.prop");
    Properties mp = new Properties();
    mp.load(mpis);
    AbstractSequenceClassifier<CoreMap> namedEntityRecognizer =
            CRFClassifier.getJarClassifier("/models/english.all.3class.distsim.crf.ser.gz", mp);

    String text = "I was born in Springfield and grew up in Boston.";
    List<Triple<String, Integer, Integer>> entitiesFromNER = namedEntityRecognizer.classifyToCharacterOffsets(text);

    List<LocationOccurrence> locationsForCLAVIN = convertNERtoCLAVIN(entitiesFromNER, text);
    assertEquals("wrong number of entities", 2, locationsForCLAVIN.size());
    assertEquals("wrong text for first entity", "Springfield", locationsForCLAVIN.get(0).getText());
    assertEquals("wrong position for first entity", 14, locationsForCLAVIN.get(0).getPosition());
    assertEquals("wrong text for second entity", "Boston", locationsForCLAVIN.get(1).getText());
    assertEquals("wrong position for second entity", 41, locationsForCLAVIN.get(1).getPosition());
}
 
Example 2
Source File: StanfordNamedEntityExtractor.java    From CLIFF with Apache License 2.0 4 votes vote down vote up
/**
 * Get extracted locations from a plain-text body.
 * 
 * @param textToParse                      Text content to perform extraction on.
 * @param manuallyReplaceDemonyms   Can slow down performance quite a bit
 * @param language   What language to parse in
 * @return          All the entities mentioned
 */
@Override
public ExtractedEntities extractEntities(String textToParse, boolean manuallyReplaceDemonyms, String language) {
    ExtractedEntities entities = new ExtractedEntities();

    if (textToParse==null || textToParse.length()==0){
        logger.warn("input to extractEntities was null or zero!");
        return entities; 
    }

    String text = textToParse;
    if(manuallyReplaceDemonyms){    // this is a noticeable performance hit
        logger.debug("Replacing all demonyms by hand");
        text = demonyms.replaceAll(textToParse);
    }
    
    AbstractSequenceClassifier<CoreMap> recognizer = recognizerByLanguage.get(language);
    
    // extract entities as <Entity Type, Start Index, Stop Index>
    List<Triple<String, Integer, Integer>> extractedEntities = 
    		recognizer.classifyToCharacterOffsets(text);

    if (extractedEntities != null) {
        for (Triple<String, Integer, Integer> extractedEntity : extractedEntities) {
            String entityName = text.substring(extractedEntity.second(), extractedEntity.third());
            int position = extractedEntity.second();
        	switch(extractedEntity.first){
            case "PERS":       // spanish
            case "I-PER":      // german
            case "PERSON":      // english
                if(personToPlaceSubstitutions.contains(entityName)){
                    entities.addLocation( getLocationOccurrence(personToPlaceSubstitutions.getSubstitution(entityName), position) );
                    logger.debug("Changed person "+entityName+" to a place");
                } else {
                    PersonOccurrence person = new PersonOccurrence(entityName, position);
                    entities.addPerson( person );
                }
                break;
            case "LUG":
            case "I-LOC":      // german
            case "LOCATION":    // english
                if(!locationBlacklist.contains(entityName)){
                    entities.addLocation( getLocationOccurrence(entityName, position) );
                } else {
                   logger.debug("Ignored blacklisted location "+entityName);
                }
                break;
            case "ORG":            // spanish
            case "I-ORG":          // german
            case "ORGANIZATION":    // english
                OrganizationOccurrence organization = new OrganizationOccurrence(entityName, position);
                entities.addOrganization( organization );
                break;
            case "OTROS":	// spanish
            case "MISC":    // if you're using the slower 4class model
                if (demonyms.contains(entityName)) {
                    logger.debug("Found and adding a MISC demonym "+entityName);
                    entities.addLocation( getLocationOccurrence(entityName, position) );
                }
                break;
            default:
                logger.error("Unknown NER type :"+ extractedEntity.first);
            }
        }
    }

    return entities;
}
 
Example 3
Source File: StanfordNamedEntityExtractor.java    From CLIFF with Apache License 2.0 4 votes vote down vote up
@Override
@SuppressWarnings("rawtypes")
public ExtractedEntities extractEntitiesFromSentences(Map[] sentences, boolean manuallyReplaceDemonyms, String language) {
	ExtractedEntities entities = new ExtractedEntities();

    if (sentences.length==0){
        logger.warn("input to extractEntities was null or zero!");
        return entities; 
    }

    if(manuallyReplaceDemonyms){    // this is a noticeable performance hit
        logger.debug("Replacing all demonyms by hand");
    }
    
    AbstractSequenceClassifier<CoreMap> recognizer = recognizerByLanguage.get(language);
    
    for(Map s:sentences){
        String storySentencesId = s.get("story_sentences_id").toString();
        String text = s.get("sentence").toString();
        if(manuallyReplaceDemonyms){    // this is a noticeable performance hit
            text = demonyms.replaceAll(text);
        }
        // extract entities as <Entity Type, Start Index, Stop Index>
        List<Triple<String, Integer, Integer>> extractedEntities = 
            recognizer.classifyToCharacterOffsets(text);
        if (extractedEntities != null) {
            for (Triple<String, Integer, Integer> extractedEntity : extractedEntities) {
                String entityName = text.substring(extractedEntity.second(), extractedEntity.third());
                int position = extractedEntity.second();
                switch(extractedEntity.first){
                case "PERSON":
                    if(personToPlaceSubstitutions.contains(entityName)){
                        entities.addLocation( getLocationOccurrence(personToPlaceSubstitutions.getSubstitution(entityName), position) );
                        logger.debug("Changed person "+entityName+" to a place");
                    } else {
                        PersonOccurrence person = new PersonOccurrence(entityName, position);
                        entities.addPerson( person );
                    }
                    break;
                case "LOCATION":
                    if(!locationBlacklist.contains(entityName)){
                        LocationOccurrence loc = getLocationOccurrence(entityName, position);  
                        // save the sentence id here
                        entities.addLocation( new SentenceLocationOccurrence(loc.getText(), storySentencesId) );
                    } else {
                       logger.debug("Ignored blacklisted location "+entityName);
                    }
                    break;
                case "ORGANIZATION":
                    OrganizationOccurrence organization = new OrganizationOccurrence(entityName, position);
                    entities.addOrganization( organization );
                    break;
                case "MISC":    // if you're using the slower 4class model
                    if (demonyms.contains(entityName)) {
                        logger.debug("Found and adding a MISC demonym "+entityName);
                        entities.addLocation( getLocationOccurrence(entityName, position) );
                    }
                    break;
                default:
                    logger.error("Unknown NER type :"+ extractedEntity.first);
                }
            }
        }
    }

    return entities;
}
 
Example 4
Source File: WorkflowDemoNERD.java    From CLAVIN-NERD with GNU General Public License v2.0 4 votes vote down vote up
/**
 * Sometimes, you might already be using Stanford NER elsewhere in
 * your application, and you'd like to just pass the output from
 * Stanford NER directly into CLAVIN, without having to re-run the
 * input through Stanford NER just to use CLAVIN. This example
 * shows you how to very easily do exactly that.
 *
 * @throws IOException
 * @throws ClavinException
 */
private static void resolveStanfordEntities() throws IOException, ClavinException {

    /*#####################################################################
     *
     * Start with Stanford NER -- no need to get CLAVIN involved for now.
     *
     *###################################################################*/

    // instantiate Stanford NER entity extractor
    InputStream mpis = WorkflowDemoNERD.class.getClassLoader().getResourceAsStream("models/english.all.3class.distsim.prop");
    Properties mp = new Properties();
    mp.load(mpis);
    AbstractSequenceClassifier<CoreMap> namedEntityRecognizer =
            CRFClassifier.getJarClassifier("/models/english.all.3class.distsim.crf.ser.gz", mp);

    // Unstructured text file about Somalia to be geoparsed
    File inputFile = new File("src/test/resources/sample-docs/Somalia-doc.txt");

    // Grab the contents of the text file as a String
    String inputString = TextUtils.fileToString(inputFile);

    // extract entities from input text using Stanford NER
    List<Triple<String, Integer, Integer>> entitiesFromNER = namedEntityRecognizer.classifyToCharacterOffsets(inputString);

    /*#####################################################################
     *
     * Now, CLAVIN comes into play...
     *
     *###################################################################*/

    // convert Stanford NER output to ClavinLocationResolver input
    List<LocationOccurrence> locationsForCLAVIN = convertNERtoCLAVIN(entitiesFromNER, inputString);

    // instantiate the CLAVIN location resolver
    ClavinLocationResolver clavinLocationResolver = new ClavinLocationResolver(new LuceneGazetteer(new File("./IndexDirectory")));

    // resolve location entities extracted from input text
    List<ResolvedLocation> resolvedLocations = clavinLocationResolver.resolveLocations(locationsForCLAVIN, 1, 1, false);

    // Display the ResolvedLocations found for the location names
    for (ResolvedLocation resolvedLocation : resolvedLocations)
        System.out.println(resolvedLocation);
}