org.eclipse.jface.text.rules.MultiLineRule Java Examples

public XMLPartitionScanner() {

      IToken xmlComment = new Token(XML_COMMENT);
      IToken xmlPI = new Token(XML_PI);
      IToken startTag = new Token(XML_START_TAG);
      IToken endTag = new Token(XML_END_TAG);
      IToken docType = new Token(XML_DOCTYPE);
      IToken text = new Token(XML_TEXT);

      IPredicateRule[] rules = new IPredicateRule[7];

      rules[0] = new NonMatchingRule();
      rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
      rules[2] = new MultiLineRule("<?", "?>", xmlPI);
      rules[3] = new MultiLineRule("</", ">", endTag);
      rules[4] = new StartTagRule(startTag);
      rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
      rules[6] = new XMLTextPredicateRule(text);

      setPredicateRules(rules);
   }
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public SamplePartitionScanner() {
	IToken tkString = new Token(LANG_STRING);
	IToken tkRawString = new Token(LANG_RAW_STRING);
	IToken tkCharacter = new Token(LANG_CHARACTER);
	IToken tkSingleComment = new Token(LANG_SINGLE_COMMENT);
	IToken tkMultiComment = new Token(LANG_MULTI_COMMENT);
	
	List<IPredicateRule> rules = new ArrayList<IPredicateRule>();
	
	rules.add(new MultiLineRule("`", "`", tkRawString, NO_ESCAPE, true));
	rules.add(new MultiLineRule("\"", "\"", tkString, '\\', true));
	rules.add(new SingleLineRule("'", "'", tkCharacter, '\\', true));
	
	rules.add(new EndOfLineRule("//", tkSingleComment, NO_ESCAPE));
	
	rules.add(new MultiLineRule("/*", "*/", tkMultiComment, NO_ESCAPE, true));
	
	
	setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
}
 

public ModulaRuleBasedPartitionScanner() {
	super();
	
       // Create the list of rules that produce tokens
	inactiveCodeRule = new InactiveCodeRule();
	IPredicateRule[] rules = new IPredicateRule[] {
		inactiveCodeRule,
	    new EndOfLineRule("--", END_OF_LINE_COMMENT_TOKEN), //$NON-NLS-1$
	    new ModulaCommentRule(BLOCK_COMMENT_TOKEN),
                            
           new SingleLineRule("\"", "\"", DOUBLE_QUOTE_STRING_TOKEN), //$NON-NLS-1$ //$NON-NLS-2$
           new SingleLineRule("'", "'",   SINGLE_QUOTE_STRING_TOKEN), //$NON-NLS-1$ //$NON-NLS-2$

	    new MultiLineRule("<*", "*>", PRAGMA_TOKEN, (char)0, true) //$NON-NLS-1$ //$NON-NLS-2$
	};

	setPredicateRules(rules);
       setDefaultReturnToken(DEFAULT_TOKEN);
}
 

/**
 *  
 */
public SQLPartitionScanner( )
{
	super( );
	IToken sqlComment = new Token( COMMENT );
	IToken sqlQuoteString = new Token( QUOTE_STRING );

	
	ArrayList rules = new ArrayList( );
	rules.add( new MultiLineRule( "\"", "\"", sqlQuoteString, '\\' ) ); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add( new MultiLineRule( "\'", "\'", sqlQuoteString, '\\' ) ); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add( new EndOfLineRule( "//", sqlComment ) ); //$NON-NLS-1$
	rules.add( new EndOfLineRule( "--", sqlComment ) ); //$NON-NLS-1$
	rules.add( new MultiLineRule( "/*", "*/", sqlComment ) ); //$NON-NLS-1$ //$NON-NLS-2$
	
	setPredicateRules( (IPredicateRule[]) rules.toArray( new IPredicateRule[rules.size( )] ) );

}
 

/**
 * XMLTokenScanner
 */
public XMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));
	rules.add(new MultiLineRule("<!--", "-->", createToken(XMLTokenType.COMMENT))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new DocTypeRule(createToken(XMLTokenType.DOCTYPE), false));
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(XMLTokenType.CDATA))); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new TagRule("?xml", createToken(XMLTokenType.DECLARATION))); //$NON-NLS-1$
	rules.add(new TagRule("/", createToken(XMLTokenType.END_TAG))); //$NON-NLS-1$
	rules.add(new TagRule(createToken(XMLTokenType.START_TAG)));

	// text
	IToken token = createToken(XMLTokenType.TEXT);
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

private static IPartitionTokenScanner createDefaultScanner() {
    final IToken string = new Token(PatternExpressionViewer.GROOVY_EXPRESSION_CONTENT_TYPE);
    final RuleBasedPartitionScanner scanner = new RuleBasedPartitionScanner();
    scanner.setPredicateRules(new IPredicateRule[] {
            new MultiLineRule(GROOVY_START_TAG, GROOVY_END_TAG, string)
    });
    return scanner;
}
 

public XMLScanner( ColorManager manager) {
   IToken procInstr = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.PROC_INSTR)));
   IToken docType = new Token(new TextAttribute(manager.getColor(IXMLColorConstants.DOCTYPE)));

   IRule[] rules = new IRule[3];
   // Add rule for processing instructions and doctype
   rules[0] = new MultiLineRule("<?", "?>", procInstr);
   rules[1] = new MultiLineRule("<!DOCTYPE", ">", docType);
   // Add generic whitespace rule.
   rules[2] = new WhitespaceRule(new XMLWhitespaceDetector());

   setRules(rules);
}
 

@Override
protected void initPredicateRules(ArrayList2<IPredicateRule> rules) {
	addStandardRules(rules, 
		LangPartitionTypes.LINE_COMMENT.getId(), 
		LangPartitionTypes.BLOCK_COMMENT.getId(), 
		null, 
		null, 
		null
	);
	
	rules.add(new SingleLineRule("'", "'", new Token(LangPartitionTypes.CHARACTER.getId()), '\\'));
	rules.add(new MultiLineRule("`", "`", new Token(LangPartitionTypes.MULTILINE_STRING.getId())));
	rules.add(new SingleLineRule("\"", "\"", new Token(LangPartitionTypes.STRING.getId()), '\\'));
}
 

protected void initialize(String[] keywords) {
    ColorRegistry registry = JFaceResources.getColorRegistry();

    IToken keyword = new Token(new TextAttribute(registry.get(KEYWORD_COLOR), null, SWT.BOLD));
    IToken string = new Token(new TextAttribute(registry.get(STRING_COLOR)));
    IToken number = new Token(new TextAttribute(registry.get(NUMBER_COLOR)));
    IToken annotation = new Token(new TextAttribute(registry.get(ANNOTATION_COLOR)));
    IToken defaultToken = new Token(new TextAttribute(registry.get(DEFAULT_COLOR)));

    List<IRule> rules = new ArrayList<IRule>();

    // strings
    rules.add(new SingleLineRule("\"", "\"", string, '\\'));

    // annotations
    rules.add(new MultiLineRule("[", "]", annotation));

    // numbers
    rules.add(new NumberRule(number));

    // keywords and normal (default) text
    WordRule wordRule = new WordRule(new WordDetector(), defaultToken);
    for (int i = 0; i < keywords.length; i++) {
        wordRule.addWord(keywords[i], keyword);
    }
    rules.add(wordRule);

    setRules(rules.toArray(new IRule[rules.size()]));
}
 

public PartitionScanner() {
    super();
    final IToken comment = new Token(ContentTypes.COMMENT_CONTENT_TYPE);
    final IPredicateRule[] rules = new IPredicateRule[2];
    rules[0] = new MultiLineRule("/*", "*/", comment);
    rules[1] = new MultiLineRule("(*", "*)", comment);
    this.setPredicateRules(rules);
}
 

public CommentScanner() {
	List<IRule> rules = new ArrayList<IRule>();

	Token commentToken = new Token(new TextAttribute(new Color(
			Display.getCurrent(), new RGB(63, 127, 95))));
	rules.add(new EndOfLineRule("--", commentToken));
	rules.add(new EndOfLineRule("//", commentToken));
	rules.add(new MultiLineRule("/*", "*/", commentToken));

	setRules(rules.toArray(new IRule[rules.size()]));
}
 

public CalcitePartitionScanner() {
	IToken comment = new Token(SQL_COMMENT);
	IToken string = new Token(SQL_STRING);
	IToken quotedIdentifier = new Token(SQL_QUOTED_IDENTIFIER);

	setPredicateRules(new IPredicateRule[] {
			new EndOfLineRule("//", comment),
			new EndOfLineRule("--", comment),
			new MultiLineRule("/*", "*/", comment),
			new SingleLineRule("\"", "\"", quotedIdentifier),
			new MultiLineRule("'", "'", string),
	});
}
 

@Override
protected List<IRule> createRules() {

	List<IRule> list= new ArrayList<IRule>();

	// Add rule for tags
	Token token= getToken(IJavaColorConstants.JAVADOC_TAG);
	list.add(new TagRule(token));


	// Add rule for HTML comments
	WordRule wordRule= new WordRule(new HTMLCommentDetector(), token);
	wordRule.addWord("<!--", token); //$NON-NLS-1$
	wordRule.addWord("--!>", token); //$NON-NLS-1$
	list.add(wordRule);


	// Add rules for links
	token= getToken(IJavaColorConstants.JAVADOC_LINK);
	list.add(new MultiLineRule("{@link", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@value", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@inheritDoc", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add rules for @code and @literals
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new MultiLineRule("{@code", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$
	list.add(new MultiLineRule("{@literal", "}", token)); //$NON-NLS-2$ //$NON-NLS-1$

	// Add generic whitespace rule
	token= getToken(IJavaColorConstants.JAVADOC_DEFAULT);
	list.add(new WhitespaceRule(new JavaWhitespaceDetector(), token));


	list.addAll(super.createRules());
	return list;
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public JavaPartitionScanner() {
	super();

	IToken string= new Token(JAVA_STRING);
	IToken character= new Token(JAVA_CHARACTER);
	IToken javaDoc= new Token(JAVA_DOC);
	IToken multiLineComment= new Token(JAVA_MULTI_LINE_COMMENT);
	IToken singleLineComment= new Token(JAVA_SINGLE_LINE_COMMENT);

	List<IPredicateRule> rules= new ArrayList<IPredicateRule>();

	// Add rule for single line comments.
	rules.add(new EndOfLineRule("//", singleLineComment)); //$NON-NLS-1$

	// Add rule for strings.
	rules.add(new SingleLineRule("\"", "\"", string, '\\')); //$NON-NLS-2$ //$NON-NLS-1$

	// Add rule for character constants.
	rules.add(new SingleLineRule("'", "'", character, '\\')); //$NON-NLS-2$ //$NON-NLS-1$

	// Add special case word rule.
	EmptyCommentRule wordRule= new EmptyCommentRule(multiLineComment);
	rules.add(wordRule);

	// Add rules for multi-line comments and javadoc.
	rules.add(new MultiLineRule("/**", "*/", javaDoc)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("/*", "*/", multiLineComment)); //$NON-NLS-1$ //$NON-NLS-2$

	IPredicateRule[] result= new IPredicateRule[rules.size()];
	rules.toArray(result);
	setPredicateRules(result);
}
 

public HTMLTokenScanner()
{
	List<IRule> rules = new ArrayList<IRule>();
	// generic whitespace rule
	rules.add(new WhitespaceRule(new WhitespaceDetector(), createToken(getTokenName(HTMLTokens.TEXT))));
	// comments
	rules.add(new PartitionerSwitchingIgnoreRule(new MultiLineRule(
			"<!--", "-->", createToken(getTokenName(HTMLTokens.COMMENT)), (char) 0, true))); //$NON-NLS-1$ //$NON-NLS-2$
	// DOCTYPE
	IToken token = createToken(getTokenName(HTMLTokens.DOCTYPE));
	rules.add(new MultiLineRule("<!DOCTYPE ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("<!doctype ", ">", token)); //$NON-NLS-1$ //$NON-NLS-2$
	// CDATA
	rules.add(new MultiLineRule("<![CDATA[", "]]>", createToken(getTokenName(HTMLTokens.CDATA)))); //$NON-NLS-1$ //$NON-NLS-2$
	// script
	rules.add(new TagRule("script", createToken(getTokenName(HTMLTokens.SCRIPT)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/script", createToken(getTokenName(HTMLTokens.SCRIPT_END)), true)); //$NON-NLS-1$
	// style
	rules.add(new TagRule("style", createToken(getTokenName(HTMLTokens.STYLE)), true)); //$NON-NLS-1$
	rules.add(new TagRule("/style", createToken(getTokenName(HTMLTokens.STYLE_END)), true)); //$NON-NLS-1$
	// xml declaration
	rules.add(new TagRule("?xml", createToken(getTokenName(HTMLTokens.XML_DECL)))); //$NON-NLS-1$
	// tags
	rules.add(new TagRule("/", createToken(getTokenName(HTMLTokens.END_TAG)))); //$NON-NLS-1$
	rules.add(generalTagRule = new HTMLParserTagRule(createToken(getTokenName(HTMLTokens.START_TAG))));

	// text
	token = createToken(getTokenName(HTMLTokens.TEXT));
	rules.add(new WordRule(new WordDetector(), token));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(token);
}
 

public HTMLDoctypeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule(
			"\"", "\"", createToken("string.quoted.double.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add a rule for single quotes
	rules.add(new MultiLineRule(
			"'", "'", createToken("string.quoted.single.doctype.identifiers-and-DTDs.html"), '\\')); //$NON-NLS-1$ //$NON-NLS-2$ //$NON-NLS-3$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule wordRule = new WordRule(new WordDetector(), createToken(StringUtil.EMPTY), true);
	wordRule.addWord("DOCTYPE", createToken("entity.name.tag.doctype.html")); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(wordRule);

	CharacterMapRule rule = new CharacterMapRule();
	rule.add('>', createToken(HTMLTokenType.TAG_END));
	rule.add('=', createToken(HTMLTokenType.EQUAL));
	rules.add(rule);
	// Tag start <(/)?
	rules.add(new TagStartRule(new TagStartWordDetector(), createToken(HTMLTokenType.TAG_START), false));

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(StringUtil.EMPTY));
}
 

/**
 * 
 */
public DTDSourceScanner() {
	super();
	List<IRule> rules = new ArrayList<IRule>();
	
	// Already handled by partitioning, but we need this for the parser
	rules.add(new MultiLineRule("<!--", "-->", createToken(DTDTokenType.COMMENT), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$

	// TODO: This should require Name directly after the opening <? and it
	// should reject <?xml
	rules.add(new MultiLineRule("<?", "?>", createToken(DTDTokenType.PI), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$

	rules.addAll(Arrays.asList(fRules));
	setRules(rules.toArray(new IRule[rules.size()]));
}
 

/**
 * XMLAttributeScanner
 */
XMLAttributeScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// attribute values
	rules.add(new MultiLineRule("\"", "\"", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(XMLTokenType.VALUE), '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// attribute names
	rules.add(new XMLAttributeRule(createToken(XMLTokenType.ATTRIBUTE)));

	// tag name
	rules.add(new WordRule(new DTDNameDetector(), createToken(XMLTokenType.OTHER), true));

	// special characters
	CharacterMapRule characterRule = new CharacterMapRule();
	characterRule.add('<', createToken(XMLTokenType.OTHER));
	characterRule.add('>', createToken(XMLTokenType.OTHER));
	characterRule.add('=', createToken(XMLTokenType.OTHER));
	rules.add(characterRule);

	setRules(rules.toArray(new IRule[rules.size()]));

	setDefaultReturnToken(createToken(XMLTokenType.OTHER));
}
 

public XMLTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add rule for double quotes
	rules.add(new MultiLineRule("\"", "\"", doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("\"", "\"", QUOTED_STRING_BREAKS, doubleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add a rule for single quotes
	rules.add(new MultiLineRule("'", "'", singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new BreakingMultiLineRule("'", "'", QUOTED_STRING_BREAKS, singleQuotedStringToken, '\\')); //$NON-NLS-1$ //$NON-NLS-2$

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// Tags
	WordRule tagWordRule = new TagWordRule(new TagNameWordDetector(), createToken(XMLTokenType.TAG_NAME), true);
	rules.add(tagWordRule);

	// Attributes
	WordRule attributeWordRule = new WordRule(new AttributeNameWordDetector(), createToken(XMLTokenType.ATTRIBUTE),
			true);
	rules.add(attributeWordRule);

	rules.add(new MultiCharacterRule("</", createToken(XMLTokenType.START_TAG))); //$NON-NLS-1$
	rules.add(new MultiCharacterRule("/>", createToken(XMLTokenType.TAG_SELF_CLOSE))); //$NON-NLS-1$

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('<', createToken(XMLTokenType.START_TAG));
	charsRule.add('>', createToken(XMLTokenType.END_TAG));
	charsRule.add('=', equalToken);
	rules.add(charsRule);

	setRules(rules.toArray(new IRule[rules.size()]));
	setDefaultReturnToken(createToken(XMLTokenType.TEXT));
}
 

public GWTPartitionScanner() {
  IToken jsniMethod = new Token(GWTPartitions.JSNI_METHOD);
  MultiLineRule jsniRule = new MultiLineRule("/*-{", "}-*/", jsniMethod);

  IPredicateRule[] result = new IPredicateRule[] {jsniRule};
  setPredicateRules(result);
}
 

public InstructionsRuleScanner(ColorProvider provider) {
	IToken stringToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_STRING_COLOR))));
	IToken commentToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_COMMENT_COLOR))));
	IToken instructToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_INSTRUCT_COLOR)), null, Font.ITALIC));
	IToken definitionsToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_DEF_COLOR)), null, Font.ITALIC));

	List<IRule> rules = Lists.newArrayList();
	// rule for Strings - may spanning multiple lines
	rules.add(new MultiLineRule("\"", "\"", stringToken));
	rules.add(new MultiLineRule("\'", "\'", stringToken));
	rules.add(new EndOfLineRule("#%", instructToken));
	// rule for comments - ended by a line delimiter
	rules.add(new EndOfLineRule("//", commentToken));

	WordRule wordRule = RuleFactory.buildRule(ImpexRules.KEYWORD, null);		
	// rule for instructions
	for (String word : Formatter.INSTRUCTION_CLASS_PROPOSALS) {
		wordRule.addWord(word, instructToken);
	}

	rules.add(wordRule);

	// rule for definitions
	wordRule = RuleFactory.buildRule(ImpexRules.VARIABLE, definitionsToken);
	rules.add(wordRule);
	IRule[] ruleArray = new IRule[rules.size()];
	setRules(rules.toArray(ruleArray));
}
 

/**
 * createRules
 * 
 * @return
 */
protected List<IRule> createRules()
{
	List<IRule> rules = new ArrayList<IRule>();

	// Add generic whitespace rule.
	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// multi-line comments
	rules.add(new MultiLineRule("/*", "*/", createToken(CSSTokenType.COMMENT), (char) 0, true));

	// strings
	rules.addAll(createStringRules());

	// at-keywords
	rules.add(createAtWordsRule());

	// units
	rules.addAll(createUnitRules());

	// numbers
	rules.add(new CSSNumberRule(createToken(CSSTokenType.NUMBER)));

	// hex colors
	// TODO: we need separate scanners for inside and outside of rules. This will erroneously pick up some ids as
	// well
	rules.add(new CSSHexColorRule(createToken(CSSTokenType.RGB)));

	// classes;
	rules.add(new WordRule(new IdentifierWithPrefixDetector('.'), createToken(CSSTokenType.CLASS)));

	// ids
	rules.add(new WordRule(new IdentifierWithPrefixDetector('#'), createToken(CSSTokenType.ID)));

	// !important
	rules.add(new CSSImportantRule(createToken(CSSTokenType.IMPORTANT)));

	// url
	rules.add(new CSSURLRule(createToken(CSSTokenType.URL)));

	// TODO: functions

	// TODO: Unicode

	// identifiers
	rules.add(new CSSIdentifierRule(createToken(CSSTokenType.IDENTIFIER)));

	// single character punctuators
	rules.add(createPunctuatorsRule());

	// multi-character punctuators
	WordRule punctuatorRule2 = new WordRule(new EqualOperatorWordDetector(), Token.UNDEFINED);
	punctuatorRule2.addWord("~=", createToken(CSSTokenType.INCLUDES));
	punctuatorRule2.addWord("|=", createToken(CSSTokenType.DASHMATCH));
	punctuatorRule2.addWord("^=", createToken(CSSTokenType.BEGINS_WITH));
	punctuatorRule2.addWord("$=", createToken(CSSTokenType.ENDS_WITH));
	rules.add(punctuatorRule2);

	return rules;
}
 

/**
 * 
 */
public DTDTagScanner()
{
	List<IRule> rules = new ArrayList<IRule>();

	rules.add(new WhitespaceRule(new WhitespaceDetector()));

	// NOTE: There is no String, but we're using this to generalize pubid,
	// att value, entity value
	rules.add(new MultiLineRule("\"", "\"", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$
	rules.add(new MultiLineRule("'", "'", createToken(DTDTokenType.STRING), '\0', true)); //$NON-NLS-1$ //$NON-NLS-2$

	WordRule operatorRule = new WordRule(new DTDOperatorWordDetector(), Token.UNDEFINED);
	operatorRule.addWord("<![", createToken(DTDTokenType.SECTION_START)); //$NON-NLS-1$
	operatorRule.addWord("]]>", createToken(DTDTokenType.SECTION_END)); //$NON-NLS-1$
	rules.add(operatorRule);

	WordRule wordRule = new WordRule(new DTDWordDetector(), Token.UNDEFINED);
	wordRule.addWord("<!ATTLIST", createToken(DTDTokenType.ATTLIST)); //$NON-NLS-1$
	wordRule.addWord("<!ELEMENT", createToken(DTDTokenType.ELEMENT)); //$NON-NLS-1$
	wordRule.addWord("<!ENTITY", createToken(DTDTokenType.ENTITY)); //$NON-NLS-1$
	wordRule.addWord("<!NOTATION", createToken(DTDTokenType.NOTATION)); //$NON-NLS-1$
	wordRule.addWord("#FIXED", createToken(DTDTokenType.FIXED)); //$NON-NLS-1$
	wordRule.addWord("#IMPLIED", createToken(DTDTokenType.IMPLIED)); //$NON-NLS-1$
	wordRule.addWord("#PCDATA", createToken(DTDTokenType.PCDATA)); //$NON-NLS-1$
	wordRule.addWord("#REQUIRED", createToken(DTDTokenType.REQUIRED)); //$NON-NLS-1$
	wordRule.addWord("ANY", createToken(DTDTokenType.ANY)); //$NON-NLS-1$
	wordRule.addWord("CDATA", createToken(DTDTokenType.CDATA_TYPE)); //$NON-NLS-1$
	wordRule.addWord("EMPTY", createToken(DTDTokenType.EMPTY)); //$NON-NLS-1$
	wordRule.addWord("ENTITY", createToken(DTDTokenType.ENTITY_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ENTITIES", createToken(DTDTokenType.ENTITIES_TYPE)); //$NON-NLS-1$
	wordRule.addWord("ID", createToken(DTDTokenType.ID_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREF", createToken(DTDTokenType.IDREF_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IDREFS", createToken(DTDTokenType.IDREFS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("IGNORE", createToken(DTDTokenType.IGNORE)); //$NON-NLS-1$
	wordRule.addWord("INCLUDE", createToken(DTDTokenType.INCLUDE)); //$NON-NLS-1$
	wordRule.addWord("NDATA", createToken(DTDTokenType.NDATA)); //$NON-NLS-1$
	wordRule.addWord("NMTOKEN", createToken(DTDTokenType.NMTOKEN_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NMTOKENS", createToken(DTDTokenType.NMTOKENS_TYPE)); //$NON-NLS-1$
	wordRule.addWord("NOTATION", createToken(DTDTokenType.NOTATION_TYPE)); //$NON-NLS-1$
	wordRule.addWord("PUBLIC", createToken(DTDTokenType.PUBLIC)); //$NON-NLS-1$
	wordRule.addWord("SYSTEM", createToken(DTDTokenType.SYSTEM)); //$NON-NLS-1$
	rules.add(wordRule);

	rules.add(new DTDEntityRule('%', createToken(DTDTokenType.PE_REF)));
	rules.add(new DTDEntityRule('&', createToken(DTDTokenType.PE_REF)));

	CharacterMapRule charsRule = new CharacterMapRule();
	charsRule.add('>', createToken(DTDTokenType.GREATER_THAN));
	charsRule.add('(', createToken(DTDTokenType.LPAREN));
	charsRule.add('|', createToken(DTDTokenType.PIPE));
	charsRule.add(')', createToken(DTDTokenType.RPAREN));
	charsRule.add('?', createToken(DTDTokenType.QUESTION));
	charsRule.add('*', createToken(DTDTokenType.STAR));
	charsRule.add('+', createToken(DTDTokenType.PLUS));
	charsRule.add(',', createToken(DTDTokenType.COMMA));
	charsRule.add('%', createToken(DTDTokenType.PERCENT));
	charsRule.add('[', createToken(DTDTokenType.LBRACKET));
	rules.add(charsRule);

	rules.add(new WordRule(new DTDNameDetector(), createToken(DTDTokenType.NAME)));
	rules.add(new WordRule(new DTDNmtokenWordDetector(), createToken(DTDTokenType.NMTOKEN)));

	setRules(rules.toArray(new IRule[rules.size()]));
}
 

protected void init() {
    TextAttribute keyAttr = tokenAttribute(PreferenceConstants.COLOR_KEY, PreferenceConstants.BOLD_KEY,
            PreferenceConstants.ITALIC_KEY, PreferenceConstants.UNDERLINE_KEY);
    IToken keyToken = new YAMLToken(keyAttr, YAMLToken.KEY);

    TextAttribute pathKeyAttr = tokenAttribute(PreferenceConstants.COLOR_KEY, PreferenceConstants.BOLD_KEY,
            PreferenceConstants.ITALIC_KEY, PreferenceConstants.UNDERLINE_KEY);
    IToken pathKeyToken = new YAMLToken(pathKeyAttr, YAMLToken.KEY);

    TextAttribute scalarAttr = tokenAttribute(PreferenceConstants.COLOR_SCALAR, PreferenceConstants.BOLD_SCALAR,
            PreferenceConstants.ITALIC_SCALAR, PreferenceConstants.UNDERLINE_SCALAR);
    IToken scalarToken = new YAMLToken(scalarAttr, YAMLToken.SCALAR);

    TextAttribute commentAttr = tokenAttribute(PreferenceConstants.COLOR_COMMENT, PreferenceConstants.BOLD_COMMENT,
            PreferenceConstants.ITALIC_COMMENT, PreferenceConstants.UNDERLINE_COMMENT);
    IToken commentToken = new YAMLToken(commentAttr, YAMLToken.COMMENT);

    TextAttribute documentAttr = tokenAttribute(PreferenceConstants.COLOR_DOCUMENT,
            PreferenceConstants.BOLD_DOCUMENT, PreferenceConstants.ITALIC_DOCUMENT,
            PreferenceConstants.UNDERLINE_DOCUMENT);

    IToken documentStartToken = new YAMLToken(documentAttr, YAMLToken.DOCUMENT_START);
    IToken documentEndToken = new YAMLToken(documentAttr, YAMLToken.DOCUMENT_END);

    TextAttribute anchorAttr = tokenAttribute(PreferenceConstants.COLOR_ANCHOR, PreferenceConstants.BOLD_ANCHOR,
            PreferenceConstants.ITALIC_ANCHOR, PreferenceConstants.UNDERLINE_ANCHOR);
    IToken anchorToken = new YAMLToken(anchorAttr, YAMLToken.ANCHOR);

    TextAttribute aliasAttr = tokenAttribute(PreferenceConstants.COLOR_ALIAS, PreferenceConstants.BOLD_ALIAS,
            PreferenceConstants.ITALIC_ALIAS, PreferenceConstants.UNDERLINE_ALIAS);
    IToken aliasToken = new YAMLToken(aliasAttr, YAMLToken.ALIAS);

    IToken indicatorCharToken = new YAMLToken(new TextAttribute(null), YAMLToken.INDICATOR_CHARACTER);

    TextAttribute tagAttr = tokenAttribute(PreferenceConstants.COLOR_TAG_PROPERTY,
            PreferenceConstants.BOLD_TAG_PROPERTY, PreferenceConstants.ITALIC_TAG_PROPERTY,
            PreferenceConstants.UNDERLINE_TAG_PROPERTY);
    IToken tagPropToken = new YAMLToken(tagAttr, YAMLToken.TAG_PROPERTY);

    TextAttribute constantAttr = tokenAttribute(PreferenceConstants.COLOR_CONSTANT,
            PreferenceConstants.BOLD_CONSTANT, PreferenceConstants.ITALIC_CONSTANT,
            PreferenceConstants.UNDERLINE_CONSTANT);
    IToken predefinedValToken = new YAMLToken(constantAttr, YAMLToken.CONSTANT);

    IToken whitespaceToken = new YAMLToken(new TextAttribute(null), YAMLToken.WHITESPACE);

    IToken directiveToken = new YAMLToken(new TextAttribute(null), YAMLToken.DIRECTIVE);

    ArrayList<IRule> rules = new ArrayList<IRule>();

    rules.add(new KeyRule(keyToken));
    rules.add(new SingleQuotedKeyRule(keyToken));
    rules.add(new DoubleQuotedKeyRule(keyToken));
    rules.add(new PathRule(pathKeyToken));
    rules.add(new MultiLineRule("\"", "\"", scalarToken, '\\'));
    rules.add(new MultiLineRule("'", "'", scalarToken));
    rules.add(new EndOfLineRule("#", commentToken));
    rules.add(new EndOfLineRule("%TAG", directiveToken));
    rules.add(new EndOfLineRule("%YAML", directiveToken));
    rules.add(new DocumentStartAndEndRule("---", documentStartToken));
    rules.add(new DocumentStartAndEndRule("...", documentEndToken));
    rules.add(new IndicatorCharacterRule(indicatorCharToken));
    rules.add(new WhitespaceRule(whitespaceToken));
    rules.add(new WordPatternRule(new AnchorWordDetector(), "&", "", anchorToken));
    rules.add(new WordPatternRule(new AnchorWordDetector(), "*", "", aliasToken));
    rules.add(new WordPatternRule(new TagWordDetector(), "!", "", tagPropToken));

    rules.add(new PredefinedValueRule(predefinedValToken));

    rules.add(new ScalarRule(scalarToken));

    IRule[] rulesArray = new IRule[rules.size()];
    rules.toArray(rulesArray);
    setRules(rulesArray);
    setDefaultReturnToken(scalarToken);

}
 

public ImpexRuleScanner(ColorProvider provider) {
	IToken stringToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_STRING_COLOR))));
	IToken commentToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_COMMENT_COLOR))));
	IToken instructToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_INSTRUCT_COLOR)), null, Font.ITALIC));
	IToken modifiersToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_MODIF_COLOR))));
	IToken impexTagsToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_TAG_COLOR)), null, Font.BOLD));
	IToken keywordsValuesToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_KVAL_COLOR))));
	IToken referenceToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_REF_COLOR)), null, Font.BOLD));
	IToken definitionsToken = new Token(new TextAttribute(provider.getColor(PreferenceConverter.getColor(store, PreferenceConstants.P_DEF_COLOR)), null, Font.ITALIC));

	List<IRule> rules = Lists.newArrayList();
	// trap instruction within quotes first to prevent being picked up by next rules
	rules.add(new EndOfLineRule("\"#%", instructToken));
	
	// rule for Strings - may span multiple lines
	rules.add(new MultiLineRule("\"", "\"", stringToken));
	rules.add(new MultiLineRule("\'", "\'", stringToken));
	
	rules.add(new EndOfLineRule("#%", instructToken));
	// rule for comments - ended by a line delimiter
	rules.add(new EndOfLineRule("#", commentToken));

	WordRule wordRule = RuleFactory.buildRule(ImpexRules.KEYWORD, null);		
	// rule for modifiers
	for (String word : Formatter.IMPEX_KEYWORDS_ATTRIBUTES) {
		wordRule.addWord(word, modifiersToken);
	}

	// rule for impex tags
	for (String header : Formatter.HEADER_MODE_PROPOSALS) {
		wordRule.addWord(header, impexTagsToken);
	}
	
	// rule for keyword values
	for (String value : Formatter.KEYWORDS_VALUES) {
		wordRule.addWord(value, keywordsValuesToken);
	}

	rules.add(wordRule);

	// rule for definitions
	wordRule = RuleFactory.buildRule(ImpexRules.VARIABLE, definitionsToken);
	rules.add(wordRule);
	wordRule = RuleFactory.buildRule(ImpexRules.REFERENCE, referenceToken);
	rules.add(wordRule);
	
	wordRule = RuleFactory.buildRule(ImpexRules.SEMICOLON, modifiersToken);
	rules.add(wordRule);
	wordRule = RuleFactory.buildRule(ImpexRules.COMMA, modifiersToken);
	rules.add(wordRule);
	
	IRule[] ruleArray = new IRule[rules.size()];
	setRules(rules.toArray(ruleArray));
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexTikzScanner(ColorManager manager) {
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.COMMAND),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    IToken specialCharToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_SPECIAL),
            null,
            manager.getStyle(ColorManager.TEX_SPECIAL_STYLE)));

    IToken numberToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.TEX_NUMBER),
            null,
            manager.getStyle(ColorManager.TEX_NUMBER_STYLE)));

    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    IToken argToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.CURLY_BRACKETS_STYLE)));

    IToken optArgToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.SQUARE_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.SQUARE_BRACKETS_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new TexSpecialCharRule(specialCharToken));
    rules.add(new WordRule(new TexWord(), commandToken));
    rules.add(new NumberRule(numberToken));
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new MultiLineRule("{", "}", argToken, '\\'));
    rules.add(new MultiLineRule("[", "]", optArgToken, '\\'));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}