org.eclipse.jface.text.rules.WordRule Java Examples

/**
 * createScannerSpecificRules
 * 
 * @return
 */
protected Collection<? extends IRule> createScannerSpecificRules()
{
	List<IRule> rules = new ArrayList<IRule>();
	WordRule wordRule = new WordRule(new KeywordIdentifierDetector(), Token.UNDEFINED);
	wordRule.addWord("em", createToken(CSSTokenType.EMS));
	wordRule.addWord("ex", createToken(CSSTokenType.EXS));
	wordRule.addWord("px", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("cm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("mm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("in", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pt", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pc", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("deg", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("rad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("grad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("ms", createToken(CSSTokenType.TIME));
	wordRule.addWord("s", createToken(CSSTokenType.TIME));
	wordRule.addWord("hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("khz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("Hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("kHz", createToken(CSSTokenType.FREQUENCY));
	addWordsToRule(wordRule, FUNCTIONS, CSSTokenType.FUNCTION);
	rules.add(wordRule);
	return rules;
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexOptArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

public static WordRule buildRule(ImpexRules ruleType, IToken token) {
	WordRule result = null;
	switch (ruleType) {
	case KEYWORD:
		result = buildKeywordRule(token);
		break;
	case VARIABLE:
		result = buildVariableRule('$', token);
		break;
	case REFERENCE:
		result = buildVariableRule('&', token);
		break;
	case SEMICOLON:
		result = buildCharRule(';', token);
		break;
	case COMMA:
		result = buildCharRule(',', token);
		break;
	default:
		result = null;
		break;
	}
	return result;
}
 

/**
 * Create a new javadoc scanner for the given color provider.
 */
public JSDocScanner()
{
	super();

	List<IRule> list = new ArrayList<IRule>();

	// Add rule for tags.
	list.add(new SingleLineRule("<", ">", getToken("text.html.basic"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

	// Add rule for links.
	list.add(new SingleLineRule("{", "}", getToken("markup.underline.link"))); //$NON-NLS-2$ //$NON-NLS-3$ //$NON-NLS-1$

	// Add word rule for keywords.
	IToken keyword = getToken("meta.tag.documentation.js"); //$NON-NLS-1$
	WordRule wordRule = new WordRule(new JSDocWordDetector());
	for (String word : KEYWORDS)
	{
		wordRule.addWord(word, keyword);
	}
	list.add(wordRule);

	setDefaultReturnToken(getToken("comment.block.documentation.js")); //$NON-NLS-1$
	setRules(list.toArray(new IRule[list.size()]));
}
 

@Override
protected List<IRule> createRules() {
	List<IRule> rules = new ArrayList<IRule>();

	Token tagBorder = getToken(IJSXColorConstants.TAG_BORDER);
	Token tagName = getToken(IJSXColorConstants.TAG_NAME);
	Token tagAttributeName = getToken(IJSXColorConstants.TAG_ATTRIBUTE_NAME);
	Token tagAttributeEquals = getToken(IJSXColorConstants.TAG_ATTRIBUTE_EQUALS);
	Token tagAttributeValue = getToken(IJSXColorConstants.TAG_ATTRIBUTE_VALUE);

	rules.add(new SingleLineRule("\"", "\"", tagAttributeValue, '\\'));
	rules.add(new SingleLineRule("'", "'", tagAttributeValue, '\\'));
	rules.add(new SingleLineRule("{", "}", tagAttributeValue, '\\'));
	rules.add(new JSXTagRule(tagName, tagBorder));
	rules.add(new WordRule(new NameDetector(), tagAttributeName));

	// setDefaultReturnToken(token);
	return rules;
}
 

@Override
protected List<IRule> createRules() {
	setDefaultReturnToken(getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_VALUE));
	List<IRule> rules= new ArrayList<IRule>();

	// Add rule for arguments.
	IToken token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ARGUMENT);
	rules.add(new ArgumentRule(token));

	// Add word rule for assignment operator.
	token= getToken(PreferenceConstants.PROPERTIES_FILE_COLORING_ASSIGNMENT);
	WordRule wordRule= new WordRule(new AssignmentDetector(), token);
	rules.add(wordRule);

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new JavaWhitespaceDetector()));

	return rules;
}
 

/**
 * XMLTokenScanner
 */
public XMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));
	rules.add(new MultiLineRule("<!--", "-->", createToken(XMLTokenType.COMMENT))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new DocTypeRule(createToken(XMLTokenType.DOCTYPE), false));
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(XMLTokenType.CDATA))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new TagRule("?xml", createToken(XMLTokenType.DECLARATION))); //$NON-NLS-1$
	rules.add(new TagRule("/", createToken(XMLTokenType.END_TAG))); //$NON-NLS-1$
	rules.add(new TagRule(createToken(XMLTokenType.START_TAG)));

	// text
	IToken token = createToken(XMLTokenType.TEXT);
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

public HTMLDoctypeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule(
			"\"", "\"", createToken("string.quoted.double.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add a rule for single quotes
	rules.add(new MultiLineRule(
			"'", "'", createToken("string.quoted.single.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule wordRule = new WordRule(new WordDetector(), createToken(StringUtil.EMPTY), true);
	wordRule.addWord("DOCTYPE", createToken("entity.name.tag.doctype.html")); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(wordRule);

	CharacterMapRule rule = new CharacterMapRule();
	rule.add('>', createToken(HTMLTokenType.TAG_END));
	rule.add('=', createToken(HTMLTokenType.EQUAL));
	rules.add(rule);
	// Tag start <(/)?
	rules.add(new TagStartRule(new TagStartWordDetector(), createToken(HTMLTokenType.TAG_START), false));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(StringUtil.EMPTY));
}
 

public HTMLScanner()
{
	IRule[] rules = new IRule[4];
	rules[0] = new WhitespaceRule(new WhitespaceDetector());
	rules[1] = new EntityRule(createToken("constant.character.entity.html")); //$NON-NLS-1$
	// non-entity ampersands should be marked as invalid
	rules[2] = new SingleCharacterRule('&', createToken("invalid.illegal.bad-ampersand.html")); //$NON-NLS-1$
	rules[3] = new WordRule(new WordDetector(), createToken("text")); //$NON-NLS-1$
	setRules(rules);
	setDefaultReturnToken(createToken("text")); //$NON-NLS-1$
}
 

public HTMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();
	// generic whitespace rule
	rules.add(new WhitespaceRule(new WhitespaceDetector(), createToken(getTokenName(HTMLTokens.TEXT))));
	// comments
	rules.add(new PartitionerSwitchingIgnoreRule(new MultiLineRule(
			"<!--", "-->", createToken(getTokenName(HTMLTokens.COMMENT)), (char) 0, true))); //$NON-NLS-1$ //$NON-NLS-2$
	// DOCTYPE
	IToken token = createToken(getTokenName(HTMLTokens.DOCTYPE));
	rules.add(new MultiLineRule("<!DOCTYPE ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("<!doctype ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	// CDATA
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(getTokenName(HTMLTokens.CDATA)))); //$NON-NLS-1$ //$NON-NLS-2$
	// script
	rules.add(new TagRule("script", createToken(getTokenName(HTMLTokens.SCRIPT)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/script", createToken(getTokenName(HTMLTokens.SCRIPT_END)), true)); //$NON-NLS-1$
	// style
	rules.add(new TagRule("style", createToken(getTokenName(HTMLTokens.STYLE)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/style", createToken(getTokenName(HTMLTokens.STYLE_END)), true)); //$NON-NLS-1$
	// xml declaration
	rules.add(new TagRule("?xml", createToken(getTokenName(HTMLTokens.XML_DECL)))); //$NON-NLS-1$
	// tags
	rules.add(new TagRule("/", createToken(getTokenName(HTMLTokens.END_TAG)))); //$NON-NLS-1$
	rules.add(generalTagRule = new HTMLParserTagRule(createToken(getTokenName(HTMLTokens.START_TAG))));

	// text
	token = createToken(getTokenName(HTMLTokens.TEXT));
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

/**
 * createAtWordsRule
 * 
 * @return
 */
private WordRule createAtWordsRule()
{
	WordRule atRule = new WordRule(new IdentifierWithPrefixDetector('@'), createToken(CSSTokenType.AT_RULE), true);

	atRule.addWord("@import", createToken(CSSTokenType.IMPORT));
	atRule.addWord("@page", createToken(CSSTokenType.PAGE));
	atRule.addWord("@media", createToken(CSSTokenType.MEDIA_KEYWORD));
	atRule.addWord("@charset", createToken(CSSTokenType.CHARSET));
	atRule.addWord("@font-face", createToken(CSSTokenType.FONTFACE));
	atRule.addWord("@namespace", createToken(CSSTokenType.NAMESPACE));

	return atRule;
}
 

/**
 * createAtWordsRule
 * 
 * @return
 */
protected WordRule createAtWordsRule()
{
	WordRule atRule = new WordRule(new IdentifierWithPrefixDetector('@'), createToken(CSSTokenType.AT_RULE));

	atRule.addWord("@import", createToken(CSSTokenType.IMPORT));
	atRule.addWord("@page", createToken(CSSTokenType.PAGE));
	atRule.addWord(KEYWORD_MEDIA, createToken(CSSTokenType.MEDIA_KEYWORD));
	atRule.addWord("@charset", createToken(CSSTokenType.CHARSET));
	atRule.addWord("@font-face", createToken(CSSTokenType.FONTFACE));
	atRule.addWord("@namespace", createToken(CSSTokenType.NAMESPACE));

	return atRule;
}
 

/**
 * addWordsToRule
 * 
 * @param wordRule
 * @param words
 * @param tokenType
 */
private void addWordsToRule(WordRule wordRule, String[] words, CSSTokenType tokenType)
{
	IToken token = createToken(tokenType);

	for (String word : words)
	{
		wordRule.addWord(word, token);
	}
}
 

protected List<IRule> createRules()
{
	List<IRule> rules = new ArrayList<IRule>();
	WordRule wordRule = new WordRule(new WordDetector(), Token.UNDEFINED, !TaskTag.isCaseSensitive());
	IToken taskToken = new Token(TASK_TAG_SCOPE);
	for (TaskTag tag : TaskTag.getTaskTags())
	{
		wordRule.addWord(tag.getName(), taskToken);
	}
	rules.add(wordRule);
	return rules;
}
 

/**
 * XMLAttributeScanner
 */
XMLAttributeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// attribute values
	rules.add(new MultiLineRule("\"", "\"", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// attribute names
	rules.add(new XMLAttributeRule(createToken(XMLTokenType.ATTRIBUTE)));

	// tag name
	rules.add(new WordRule(new DTDNameDetector(), createToken(XMLTokenType.OTHER), true));

	// special characters
	CharacterMapRule characterRule = new CharacterMapRule();
	characterRule.add('<', createToken(XMLTokenType.OTHER));
	characterRule.add('>', createToken(XMLTokenType.OTHER));
	characterRule.add('=', createToken(XMLTokenType.OTHER));
	rules.add(characterRule);

	setRules(rules.toArray(new IRule[rules.size()]));

	setDefaultReturnToken(createToken(XMLTokenType.OTHER));
}
 

public XMLTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule("\"", "\"", doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("\"", "\"", QUOTED_STRING_BREAKS, doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add a rule for single quotes
	rules.add(new MultiLineRule("'", "'", singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("'", "'", QUOTED_STRING_BREAKS, singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule tagWordRule = new TagWordRule(new TagNameWordDetector(), createToken(XMLTokenType.TAG_NAME), true);
	rules.add(tagWordRule);

	// Attributes
	WordRule attributeWordRule = new WordRule(new AttributeNameWordDetector(), createToken(XMLTokenType.ATTRIBUTE),
			true);
	rules.add(attributeWordRule);

	rules.add(new MultiCharacterRule("</", createToken(XMLTokenType.START_TAG))); //$NON-NLS-1$
	rules.add(new MultiCharacterRule("/>", createToken(XMLTokenType.TAG_SELF_CLOSE))); //$NON-NLS-1$

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('<', createToken(XMLTokenType.START_TAG));
	charsRule.add('>', createToken(XMLTokenType.END_TAG));
	charsRule.add('=', equalToken);
	rules.add(charsRule);

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(XMLTokenType.TEXT));
}
 

public XMLScanner()
{
	IRule[] rules = new IRule[4];
	rules[0] = new WhitespaceRule(new WhitespaceDetector());
	rules[1] = new EntityRule(createToken(CONSTANT_CHARACTER_ENTITY_XML));
	// non-entity ampersands should be marked as invalid
	rules[2] = new SingleCharacterRule('&', createToken(INVALID_ILLEGAL_BAD_AMPERSAND_XML));
	rules[3] = new WordRule(new WordDetector(), createToken(TEXT));
	setRules(rules);
	setDefaultReturnToken(createToken(TEXT));
}
 

/**
 * Construct the rules
 */
public TLACodeScanner()
{
    TLAColorProvider provider = TLAEditorActivator.getDefault().getTLAColorProvider();

    IToken keyword = new Token(new TextAttribute(provider.getColor(TLAColorProvider.TLA_KEYWORD_KEY), null, SWT.BOLD));
    IToken value = new Token(new TextAttribute(provider.getColor(TLAColorProvider.TLA_VALUE_KEY)));
    IToken other = new Token(new TextAttribute(provider.getColor(TLAColorProvider.DEFAULT_TEXT_KEY)));

    List rules = new ArrayList();

    // Add generic whitespace rule.
    // rules.add(new WhitespaceRule(DocumentHelper.getDefaultWhitespaceDetector()));

    // Add word rule for standard words
    WordRule wordRule = new WordRule(DocumentHelper.getDefaultWordDetector(), other);
    
    // add values
    for (int i = 0; i < ITLAReserveredWords.ALL_VALUES_ARRAY.length; i++)
    {
        wordRule.addWord(ITLAReserveredWords.ALL_VALUES_ARRAY[i], value);
    }
    
    // add reserved words
    for (int i = 0; i < ITLAReserveredWords.ALL_WORDS_ARRAY.length; i++)
    {
        wordRule.addWord(ITLAReserveredWords.ALL_WORDS_ARRAY[i], keyword);
    }
    
    rules.add(wordRule);

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

@Override
protected List<IRule> createRules() {

	List<IRule> list= new ArrayList<IRule>();

	// Add rule for tags
	Token token= getToken(IJavaColorConstants.JAVADOC_TAG);
	list.add(new TagRule(token));


	// Add rule for HTML comments
	WordRule wordRule= new WordRule(new HTMLCommentDetector(), token);
	wordRule.addWord("<!--", token); //$NON-NLS-1$
	wordRule.addWord("--!>", token); //$NON-NLS-1$
	list.add(wordRule);


	// Add rules for links
	token= getToken(IJavaColorConstants.JAVADOC_LINK);
	list.add(new MultiLineRule("{@link", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@value", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@inheritDoc", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add rules for @code and @literals
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new MultiLineRule("{@code", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@literal", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add generic whitespace rule
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new WhitespaceRule(new JavaWhitespaceDetector(), token));


	list.addAll(super.createRules());
	return list;
}
 

public InstructionsRuleScanner(ColorProvider provider) {
	IToken stringToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_STRING_COLOR))));
	IToken commentToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_COMMENT_COLOR))));
	IToken instructToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_INSTRUCT_COLOR)), null, Font.ITALIC));
	IToken definitionsToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_DEF_COLOR)), null, Font.ITALIC));

	List<IRule> rules = Lists.newArrayList();
	// rule for Strings - may spanning multiple lines
	rules.add(new MultiLineRule("\"", "\"", stringToken));
	rules.add(new MultiLineRule("\'", "\'", stringToken));
	rules.add(new EndOfLineRule("#%", instructToken));
	// rule for comments - ended by a line delimiter
	rules.add(new EndOfLineRule("//", commentToken));

	WordRule wordRule = RuleFactory.buildRule(ImpexRules.KEYWORD, null);		
	// rule for instructions
	for (String word : Formatter.INSTRUCTION_CLASS_PROPOSALS) {
		wordRule.addWord(word, instructToken);
	}

	rules.add(wordRule);

	// rule for definitions
	wordRule = RuleFactory.buildRule(ImpexRules.VARIABLE, definitionsToken);
	rules.add(wordRule);
	IRule[] ruleArray = new IRule[rules.size()];
	setRules(rules.toArray(ruleArray));
}
 

@Override
protected List<IRule> createRules() {
	setDefaultReturnToken(getToken(PreferenceConstants.EDITOR_CONFIG_COLORING_PROPERTY_VALUE));
	List<IRule> rules = new ArrayList<IRule>();

	// Add word rule for assignment operator.
	IToken token = getToken(PreferenceConstants.EDITOR_CONFIG_COLORING_ASSIGNMENT);
	WordRule wordRule = new WordRule(new AssignmentDetector(), token);
	rules.add(wordRule);

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new EditorConfigWhitespaceDetector()));

	return rules;
}
 

protected void initialize(String[] keywords) {
    ColorRegistry registry = JFaceResources.getColorRegistry();

    IToken keyword = new Token(new TextAttribute(registry.get(KEYWORD_COLOR), null, SWT.BOLD));
    IToken string = new Token(new TextAttribute(registry.get(STRING_COLOR)));
    IToken number = new Token(new TextAttribute(registry.get(NUMBER_COLOR)));
    IToken annotation = new Token(new TextAttribute(registry.get(ANNOTATION_COLOR)));
    IToken defaultToken = new Token(new TextAttribute(registry.get(DEFAULT_COLOR)));

    List<IRule> rules = new ArrayList<IRule>();

    // strings
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));

    // annotations
    rules.add(new MultiLineRule("[", "]", annotation));

    // numbers
    rules.add(new NumberRule(number));

    // keywords and normal (default) text
    WordRule wordRule = new WordRule(new WordDetector(), defaultToken);
    for (int i = 0; i < keywords.length; i++) {
        wordRule.addWord(keywords[i], keyword);
    }
    rules.add(wordRule);

    setRules(rules.toArray(new IRule[rules.size()]));
}
 

protected void addWords( WordRule rule, String[] tokens, IToken token )
{
	for ( int i = 0; i < tokens.length; i++ )
	{
		rule.addWord( tokens[i], token );
	}

}
 

/**
 * A default constructor.
 * @param manager
 */
public TexScanner(ColorManager manager) {
    // A token that defines how to color numbers
    IToken numberToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_NUMBER),
            null,
            manager.getStyle(ColorManager.TEX_NUMBER_STYLE)));

    // A token that defines how to color command words (\command_word)
    IToken commandToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMAND),
            null,
            manager.getStyle(ColorManager.COMMAND_STYLE)));

    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    // A token that defines how to color special characters (\_, \&, \~ ...)
    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));
    
    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new TexSpecialCharRule(specialCharToken));
    rules.add(new WordRule(new TexWord(), commandToken));
    rules.add(new NumberRule(numberToken));
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    
    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/**
 * Creates a BibTeX document scanner
 * 
 * @param provider The color provider for syntax highlighting
 */
public BibCodeScanner(BibColorProvider provider) {

    IToken keyword = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.KEYWORD)));
    IToken comment = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.SINGLE_LINE_COMMENT)));
    IToken other = new Token(new TextAttribute(provider
            .getColor(BibColorProvider.DEFAULT)));

    List rules = new ArrayList();

    // Add rule for single line comments.
    rules.add(new EndOfLineRule("%", comment));

    rules.add(new BibCommandRule(keyword));

    // Add generic whitespace rule.
    rules.add(new WhitespaceRule(new WhitespaceDetector()));

    // Add word rule for keywords, types, and constants.
    WordRule wordRule = new WordRule(new BibWordDetector(), other);
    rules.add(wordRule);

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

@Override
protected void initRules(ArrayList2<IRule> rules) {
	
	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new LangWhitespaceDetector()));
	
	
	final IToken tkDefault = getToken(GoUIPreferenceConstants.DEFAULT);
	
	WordRule wordRule = new WordRule(new JavaWordDetector(), tkDefault);
	
	final IToken keyword         = getToken(GoUIPreferenceConstants.KEYWORD);
	final IToken primitive       = getToken(GoUIPreferenceConstants.KW_NATIVE_TYPES);
	final IToken builtinFunction = getToken(GoUIPreferenceConstants.BUILTIN_FUNCTION);
	final IToken textToken       = getToken(GoUIPreferenceConstants.DEFAULT);
	setDefaultReturnToken(textToken);
	
	// add tokens for each reserved word
	wordRule.addWord("break",       keyword);
	wordRule.addWord("default",     keyword);
	wordRule.addWord("func",        keyword);
	wordRule.addWord("interface",   keyword);
	wordRule.addWord("select",      keyword);
	wordRule.addWord("case",        keyword);
	wordRule.addWord("defer",       keyword);
	wordRule.addWord("go",          keyword);
	wordRule.addWord("map",         keyword);
	wordRule.addWord("struct",      keyword);
	wordRule.addWord("chan",        keyword);
	wordRule.addWord("else",        keyword);
	wordRule.addWord("goto",        keyword);
	wordRule.addWord("package",     keyword);
	wordRule.addWord("switch",      keyword);
	wordRule.addWord("const",       keyword);
	wordRule.addWord("fallthrough", keyword);
	wordRule.addWord("if",          keyword);
	wordRule.addWord("range",       keyword);
	wordRule.addWord("type",        keyword);
	wordRule.addWord("continue",    keyword);
	wordRule.addWord("for",         keyword);
	wordRule.addWord("import",      keyword);
	wordRule.addWord("return",      keyword);
	wordRule.addWord("var",         keyword);
	
	wordRule.addWord("iota",        keyword);
	
	wordRule.addWord("append",  builtinFunction);
	wordRule.addWord("cap",     builtinFunction);
	wordRule.addWord("close",   builtinFunction);
	wordRule.addWord("complex", builtinFunction);
	wordRule.addWord("copy",    builtinFunction);
	wordRule.addWord("delete",  builtinFunction);
	wordRule.addWord("imag",    builtinFunction);
	wordRule.addWord("len",     builtinFunction);
	wordRule.addWord("make",    builtinFunction);
	wordRule.addWord("new",     builtinFunction);
	wordRule.addWord("panic",   builtinFunction);
	wordRule.addWord("print",   builtinFunction);
	wordRule.addWord("println", builtinFunction);
	wordRule.addWord("real",    builtinFunction);
	wordRule.addWord("recover", builtinFunction);
	
	final IToken value = getToken(GoUIPreferenceConstants.KW_LITERAL);
	wordRule.addWord("nil",   value);
	wordRule.addWord("true",  value);
	wordRule.addWord("false", value);
	
	wordRule.addWord("uint8", primitive);
	wordRule.addWord("uint16", primitive);
	wordRule.addWord("uint32", primitive);
	wordRule.addWord("uint64", primitive);
	wordRule.addWord("int8", primitive);
	wordRule.addWord("int16", primitive);
	wordRule.addWord("int32", primitive);
	wordRule.addWord("int64", primitive);
	wordRule.addWord("float32", primitive);
	wordRule.addWord("float64", primitive);
	wordRule.addWord("complex64",  primitive);
	wordRule.addWord("complex128", primitive);
	wordRule.addWord("rune",       primitive);
	wordRule.addWord("byte",       primitive);
	wordRule.addWord("uint",       primitive);
	wordRule.addWord("int",        primitive);
	wordRule.addWord("uintptr",    primitive);
	
	wordRule.addWord("string", primitive);
	wordRule.addWord("bool",   primitive);
	wordRule.addWord("error",  primitive);
	
	rules.add(wordRule);
	
	rules.add(new LexingRule_RuleAdapter(new GoSubLexer(getToken(GoUIPreferenceConstants.NUMBER))));
	
	rules.add(new GoOperatorRule(getToken(GoUIPreferenceConstants.OPERATOR)));
	rules.add(new GoControlCharactersRule(getToken(GoUIPreferenceConstants.STRUCTURAL_SYMBOLS)));
}
 

/**
 * createRules
 * 
 * @return
 */
protected List<IRule> createRules()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// multi-line comments
	rules.add(new MultiLineRule("/*", "*/", createToken(CSSTokenType.COMMENT), (char) 0, true));

	// strings
	rules.addAll(createStringRules());

	// at-keywords
	rules.add(createAtWordsRule());

	// units
	rules.addAll(createUnitRules());

	// numbers
	rules.add(new CSSNumberRule(createToken(CSSTokenType.NUMBER)));

	// hex colors
	// TODO: we need separate scanners for inside and outside of rules. This will erroneously pick up some ids as
	// well
	rules.add(new CSSHexColorRule(createToken(CSSTokenType.RGB)));

	// classes;
	rules.add(new WordRule(new IdentifierWithPrefixDetector('.'), createToken(CSSTokenType.CLASS)));

	// ids
	rules.add(new WordRule(new IdentifierWithPrefixDetector('#'), createToken(CSSTokenType.ID)));

	// !important
	rules.add(new CSSImportantRule(createToken(CSSTokenType.IMPORTANT)));

	// url
	rules.add(new CSSURLRule(createToken(CSSTokenType.URL)));

	// TODO: functions

	// TODO: Unicode

	// identifiers
	rules.add(new CSSIdentifierRule(createToken(CSSTokenType.IDENTIFIER)));

	// single character punctuators
	rules.add(createPunctuatorsRule());

	// multi-character punctuators
	WordRule punctuatorRule2 = new WordRule(new EqualOperatorWordDetector(), Token.UNDEFINED);
	punctuatorRule2.addWord("~=", createToken(CSSTokenType.INCLUDES));
	punctuatorRule2.addWord("|=", createToken(CSSTokenType.DASHMATCH));
	punctuatorRule2.addWord("^=", createToken(CSSTokenType.BEGINS_WITH));
	punctuatorRule2.addWord("$=", createToken(CSSTokenType.ENDS_WITH));
	rules.add(punctuatorRule2);

	return rules;
}
 

/**
 * 
 */
public DTDTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// NOTE: There is no String, but we're using this to generalize pubid,
	// att value, entity value
	rules.add(new MultiLineRule("\"", "\"", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$

	WordRule operatorRule = new WordRule(new DTDOperatorWordDetector(), Token.UNDEFINED);
	operatorRule.addWord("<![", createToken(DTDTokenType.SECTION_START)); //$NON-NLS-1$
	operatorRule.addWord("]]>", createToken(DTDTokenType.SECTION_END)); //$NON-NLS-1$
	rules.add(operatorRule);

	WordRule wordRule = new WordRule(new DTDWordDetector(), Token.UNDEFINED);
	wordRule.addWord("<!ATTLIST", createToken(DTDTokenType.ATTLIST)); //$NON-NLS-1$
	wordRule.addWord("<!ELEMENT", createToken(DTDTokenType.ELEMENT)); //$NON-NLS-1$
	wordRule.addWord("<!ENTITY", createToken(DTDTokenType.ENTITY)); //$NON-NLS-1$
	wordRule.addWord("<!NOTATION", createToken(DTDTokenType.NOTATION)); //$NON-NLS-1$
	wordRule.addWord("#FIXED", createToken(DTDTokenType.FIXED)); //$NON-NLS-1$
	wordRule.addWord("#IMPLIED", createToken(DTDTokenType.IMPLIED)); //$NON-NLS-1$
	wordRule.addWord("#PCDATA", createToken(DTDTokenType.PCDATA)); //$NON-NLS-1$
	wordRule.addWord("#REQUIRED", createToken(DTDTokenType.REQUIRED)); //$NON-NLS-1$
	wordRule.addWord("ANY", createToken(DTDTokenType.ANY)); //$NON-NLS-1$
	wordRule.addWord("CDATA", createToken(DTDTokenType.CDATA_TYPE)); //$NON-NLS-1$
	wordRule.addWord("EMPTY", createToken(DTDTokenType.EMPTY)); //$NON-NLS-1$
	wordRule.addWord("ENTITY", createToken(DTDTokenType.ENTITY_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ENTITIES", createToken(DTDTokenType.ENTITIES_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ID", createToken(DTDTokenType.ID_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREF", createToken(DTDTokenType.IDREF_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREFS", createToken(DTDTokenType.IDREFS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IGNORE", createToken(DTDTokenType.IGNORE)); //$NON-NLS-1$
	wordRule.addWord("INCLUDE", createToken(DTDTokenType.INCLUDE)); //$NON-NLS-1$
	wordRule.addWord("NDATA", createToken(DTDTokenType.NDATA)); //$NON-NLS-1$
	wordRule.addWord("NMTOKEN", createToken(DTDTokenType.NMTOKEN_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NMTOKENS", createToken(DTDTokenType.NMTOKENS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NOTATION", createToken(DTDTokenType.NOTATION_TYPE)); //$NON-NLS-1$
	wordRule.addWord("PUBLIC", createToken(DTDTokenType.PUBLIC)); //$NON-NLS-1$
	wordRule.addWord("SYSTEM", createToken(DTDTokenType.SYSTEM)); //$NON-NLS-1$
	rules.add(wordRule);

	rules.add(new DTDEntityRule('%', createToken(DTDTokenType.PE_REF)));
	rules.add(new DTDEntityRule('&', createToken(DTDTokenType.PE_REF)));

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('>', createToken(DTDTokenType.GREATER_THAN));
	charsRule.add('(', createToken(DTDTokenType.LPAREN));
	charsRule.add('|', createToken(DTDTokenType.PIPE));
	charsRule.add(')', createToken(DTDTokenType.RPAREN));
	charsRule.add('?', createToken(DTDTokenType.QUESTION));
	charsRule.add('*', createToken(DTDTokenType.STAR));
	charsRule.add('+', createToken(DTDTokenType.PLUS));
	charsRule.add(',', createToken(DTDTokenType.COMMA));
	charsRule.add('%', createToken(DTDTokenType.PERCENT));
	charsRule.add('[', createToken(DTDTokenType.LBRACKET));
	rules.add(charsRule);

	rules.add(new WordRule(new DTDNameDetector(), createToken(DTDTokenType.NAME)));
	rules.add(new WordRule(new DTDNmtokenWordDetector(), createToken(DTDTokenType.NMTOKEN)));

	setRules(rules.toArray(new IRule[rules.size()]));
}
 

public JsniScanner(IColorManager colorManager) {
  // TODO: get these from GWT preference store instead of Java's

  // TODO: need to refresh colorManager when prefs change
  //
  IToken jsniJavaRef = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_JAVA_REF)));

  IToken jsniComment = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_COMMENT)));

  IToken jsniKeyword = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_KEYWORD)));

  IToken jsniDefault = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_DEFAULT)));

  IToken jsniString = new Token(new TextAttribute(
      colorManager.getColor(JsniColorConstants.JSNI_STRING)));

  ArrayList<IRule> rules = new ArrayList<IRule>();

  // Java references
  rules.add(new JavaRefRule(jsniJavaRef));

  // single line comments
  rules.add(new EndOfLineRule("//", jsniComment));

  // JS keywords
  WordRule keywordRules = new WordRule(new JsWordDetector(), jsniDefault);
  for (String keyword : JS_KEYWORDS) {
    keywordRules.addWord(keyword, jsniKeyword);
  }
  rules.add(keywordRules);

  // Strings
  rules.add(new SingleLineRule("\"", "\"", jsniString, '\\', true));
  rules.add(new SingleLineRule("'", "'", jsniString, '\\', true));

  // Add generic whitespace rule.
  rules.add(new WhitespaceRule(new WhitespaceDetector()));

  // convert to array and set them for RuleBasedScanner to use
  setRules(rules.toArray(new IRule[rules.size()]));
}