Java Code Examples for it.unimi.dsi.lang.MutableString#toLowerCase()

The following examples show how to use it.unimi.dsi.lang.MutableString#toLowerCase() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: tagme   File: WikipediaAnchorParser.java    License: Apache License 2.0 4 votes vote down vote up
/**
 * It normalizes the anchor text:
 * 1. ascii normalization
 * 2. delete brackets at the end (i.e. in titles)
 * 3. delete a pattern at the beginning of the text, see anchorStart
 * 4. delete all dots '.'
 * 5. replace all punctuations with whitespaces, except for {@link WikipediaAnchorParser#SPECIAL_PUNCTS}
 * 6. if the original contained any of {@link WikipediaAnchorParser#SPECIAL_PUNCTS},
 * 	  it returns 2 anchors (with those puncts replaced by whitespace, and with those puncts deleted)
 *    otherwise, the normalization at 5.
 * If the normalization process doesn't produce a valid anchor, an empty array is returned
 * @param original
 * @param anchorStart A pattern that identifies some common articles or preposition to be deleted if they occur at the beginning of the anchor
 * @return
 */
public static CharSequence[] parseAnchor(CharSequence original, Pattern anchorStart)
{

	MutableString anchor = Chars.toNormalizedASCII(original);
	anchor.squeezeSpace();
	anchor.trim();

	if (anchor.length() < MIN_ANCHOR_LEN || !contaisText(anchor)) return Chars.EMPTY_STRINGS;

	anchor.loose();
	anchor.toLowerCase();

	Matcher m = P_FINAL_BRACKETS.matcher(anchor);
	if (m.find()) anchor.delete(m.start(), m.end());
	anchor.trim();
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	if (anchorStart != null)
	{
		Matcher m2 = anchorStart.matcher(anchor);
		if (m2.find()) anchor.delete(m2.start(), m2.end());
		anchor.trim();
		if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;
	}

	anchor = removeDots(anchor);
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	anchor = removePunctuations(anchor, SPECIAL_PUNCTS, false);
	if (anchor.length() < MIN_ANCHOR_LEN) return Chars.EMPTY_STRINGS;

	if (anchor.indexOfAnyOf(SPECIAL_PUNCTS_CHARS, 0)<0){
		if (!contaisText(anchor)) return Chars.EMPTY_STRINGS;
		else return new CharSequence[]{anchor};
	} else {
		MutableString anchorNoPuncts = new MutableString(anchor);
		anchor.replace(SPECIAL_PUNCTS_CHARS, SPECIAL_PUNCTS_CHAR_MAP);
		//			Chars.trimMultispace(anchor);
		anchor.squeezeSpace().trim();
		if (anchor.length() < MIN_ANCHOR_LEN || !contaisText(anchor)) return Chars.EMPTY_STRINGS;

		anchorNoPuncts.delete(SPECIAL_PUNCTS_CHARS);
		anchorNoPuncts.squeezeSpace().trim();
		if (anchorNoPuncts.length() < MIN_ANCHOR_LEN || !contaisText(anchorNoPuncts)) return new CharSequence[]{anchor};
		else return new CharSequence[]{anchor, anchorNoPuncts};
	}
}
 
Example 2
public boolean startElement( final Element element, final Map<Attribute,MutableString> attrMap ) {
	Object s;

	// TODO: what about IMG?
	
	if ( element == Element.A || element == Element.AREA || element == Element.LINK ) {
		s = attrMap.get( Attribute.HREF );
		if ( s != null )
			urls.add( s.toString() );
	}

	// IFRAME or FRAME + SRC
	if ( element == Element.IFRAME || element == Element.FRAME || element == Element.EMBED ) {
		s = attrMap.get( Attribute.SRC );
		if ( s != null )
			urls.add( s.toString() );
	}

	// BASE + HREF (change context!)
	if ( element == Element.BASE && base == null ) {
		s = attrMap.get( Attribute.HREF );
		if ( s != null )
			base = s.toString();
	}

	// META REFRESH/LOCATION
	if ( element == Element.META ) {
		final MutableString equiv = attrMap.get( Attribute.HTTP_EQUIV );
		final MutableString content = attrMap.get( Attribute.CONTENT );
		if ( equiv != null && content != null ) {
			equiv.toLowerCase();

			// http-equiv="refresh" content="0;URL=http://foo.bar/..."
			if ( equiv.equals( "refresh" ) && ( metaRefresh == null ) ) {

				final int pos = URLEQUAL_PATTERN.search( content );
				if ( pos != -1 )
					metaRefresh = content.substring( pos + URLEQUAL_PATTERN.length() ).toString();
			}

			// http-equiv="location" content="http://foo.bar/..."
			if ( equiv.equals( "location" ) && ( metaLocation == null ) )
				metaLocation = attrMap.get( Attribute.CONTENT ).toString();
		}
	}

	return true;
}