org.eclipse.jface.text.rules.Token Java Examples

@Override
protected List<IRule> createRules() {
	List<IRule> list= new ArrayList<IRule>();
	Token defaultToken= getToken(fDefaultTokenProperty);

	List<WordMatcher> matchers= createMatchers();
	if (matchers.size() > 0) {
		CombinedWordRule combinedWordRule= new CombinedWordRule(new AtJavaIdentifierDetector(), defaultToken);
		for (int i= 0, n= matchers.size(); i < n; i++)
			combinedWordRule.addWordMatcher(matchers.get(i));
		list.add(combinedWordRule);
	}

	setDefaultReturnToken(defaultToken);

	return list;
}
 

@Override
public IToken nextToken() {
	while (true) {
		lastScannedIndex++;
		if (lastScannedIndex >= concepts.size())
			return Token.EOF;

		SyntaxConcept concept = getLastConcept();
		TextAttribute attr = ClassificationFormat.getTextAttribute(concept);
		if (attr != null) {
			Logger.debug("token: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
			return new Token(attr);
		} else {
			Logger.debug("ignored: '" + concept.type + "' @" + concept.line + "," + concept.column + "; value: "
					+ concept.value);
		}
	}
}
 

/**
 * createLexemeProvider
 * 
 * @param document
 * @param offset
 * @return
 */
ILexemeProvider<XMLTokenType> createLexemeProvider(IDocument document, int offset)
{
	int documentLength = document.getLength();

	// account for last position returning an empty IDocument default partition
	int lexemeProviderOffset = (offset >= documentLength) ? documentLength - 1 : offset;

	return new XMLLexemeProvider(document, lexemeProviderOffset, new XMLTagScanner()
	{
		@Override
		protected IToken createToken(XMLTokenType type)
		{
			return new Token(type);
		}
	});
}
 

/**
 * createScannerSpecificRules
 * 
 * @return
 */
protected Collection<? extends IRule> createScannerSpecificRules()
{
	List<IRule> rules = new ArrayList<IRule>();
	WordRule wordRule = new WordRule(new KeywordIdentifierDetector(), Token.UNDEFINED);
	wordRule.addWord("em", createToken(CSSTokenType.EMS));
	wordRule.addWord("ex", createToken(CSSTokenType.EXS));
	wordRule.addWord("px", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("cm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("mm", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("in", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pt", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("pc", createToken(CSSTokenType.LENGTH));
	wordRule.addWord("deg", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("rad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("grad", createToken(CSSTokenType.ANGLE));
	wordRule.addWord("ms", createToken(CSSTokenType.TIME));
	wordRule.addWord("s", createToken(CSSTokenType.TIME));
	wordRule.addWord("hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("khz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("Hz", createToken(CSSTokenType.FREQUENCY));
	wordRule.addWord("kHz", createToken(CSSTokenType.FREQUENCY));
	addWordsToRule(wordRule, FUNCTIONS, CSSTokenType.FUNCTION);
	rules.add(wordRule);
	return rules;
}
 

public XMLPartitionScanner() {

      IToken xmlComment = new Token(XML_COMMENT);
      IToken xmlPI = new Token(XML_PI);
      IToken startTag = new Token(XML_START_TAG);
      IToken endTag = new Token(XML_END_TAG);
      IToken docType = new Token(XML_DOCTYPE);
      IToken text = new Token(XML_TEXT);

      IPredicateRule[] rules = new IPredicateRule[7];

      rules[0] = new NonMatchingRule();
      rules[1] = new MultiLineRule("<!--", "-->", xmlComment);
      rules[2] = new MultiLineRule("<?", "?>", xmlPI);
      rules[3] = new MultiLineRule("</", ">", endTag);
      rules[4] = new StartTagRule(startTag);
      rules[5] = new MultiLineRule("<!DOCTYPE", ">", docType);
      rules[6] = new XMLTextPredicateRule(text);

      setPredicateRules(rules);
   }
 

private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
	boolean eventValue = false;
	Object value = event.getNewValue();
	if (value instanceof Boolean)
		eventValue = ((Boolean) value).booleanValue();
	else if (IPreferenceStore.TRUE.equals(value))
		eventValue = true;

	Object data = token.getData();
	if (data instanceof TextAttribute) {
		TextAttribute oldAttr = (TextAttribute) data;
		boolean activeValue = (oldAttr.getStyle() & styleAttribute) == styleAttribute;
		if (activeValue != eventValue)
			token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(),
					eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
	}
}
 

@Override
public IToken evaluate(ICharacterScanner scanner, boolean resume) {
	if (scanner instanceof ITokenScanner) {
		if (isShowInactiveCode && !CollectionUtils.isEmpty(inactiveCodeRegions)) {
			ITokenScanner tokenScanner = (ITokenScanner) scanner;
			ITextRegion reg = findInactiveCodeRegion(tokenScanner.getTokenOffset());
			if (reg != null) {
				for (int i = tokenScanner.getTokenOffset(); i < reg.getOffset() + reg.getLength(); i++) {
					scanner.read();
				}
				return ModulaPartitionTokens.DISABLED_CODE;
			}
		}
	}
	return Token.UNDEFINED;
}
 

private void adaptToStyleChange(Token token, PropertyChangeEvent event, int styleAttribute) {
	boolean eventValue= false;
	Object value= event.getNewValue();
	if (value instanceof Boolean)
		eventValue= ((Boolean) value).booleanValue();
	else if (IPreferenceStore.TRUE.equals(value))
		eventValue= true;

	Object data= token.getData();
	if (data instanceof TextAttribute) {
		TextAttribute oldAttr= (TextAttribute) data;
		boolean activeValue= (oldAttr.getStyle() & styleAttribute) == styleAttribute;
		if (activeValue != eventValue)
			token.setData(new TextAttribute(oldAttr.getForeground(), oldAttr.getBackground(), eventValue ? oldAttr.getStyle() | styleAttribute : oldAttr.getStyle() & ~styleAttribute));
	}
}
 

/**
 * A default constructor.
 * @param manager
 */
public TexArgScanner(ColorManager manager) {
    IToken commentToken = new Token(new TextAttribute(manager
            .getColor(ColorManager.COMMENT),
            null,
            manager.getStyle(ColorManager.COMMENT_STYLE)));

    //Commands are colored in argument color with command styles 
    IToken commandToken = new Token(
            new TextAttribute(
                    manager.getColor(ColorManager.CURLY_BRACKETS),
                    null,
                    manager.getStyle(ColorManager.COMMAND_STYLE)));

    List<IRule> rules = new ArrayList<IRule>();
    rules.add(new EndOfLineRule("%", commentToken, '\\'));
    rules.add(new WhitespaceRule(new WhitespaceDetector()));
    rules.add(new WordRule(new TexWord(), commandToken));

    IRule[] result = new IRule[rules.size()];
    rules.toArray(result);
    setRules(result);
}
 

/***
 * Add some partition rules common to C-style languages.
 * All rules are optional, if an id is null, the rule will not be added.
 */
protected static void addStandardRules(ArrayList2<IPredicateRule> rules, 
		String lineCommentId, String blockCommentId, 
		String docLineCommentId, String docBlockCommentId,
		String stringId) {
	if(docLineCommentId != null) {
		rules.add(new PatternRule("///", null, new Token(docLineCommentId), NO_ESCAPE_CHAR, true, true));
	}
	if(docBlockCommentId != null) {
		rules.add(new PatternRule("/**", "*/", new Token(docBlockCommentId), NO_ESCAPE_CHAR, false, true));
	}
	
	if(lineCommentId != null) {
		rules.add(new PatternRule("//", null, new Token(lineCommentId), NO_ESCAPE_CHAR, true, true));
	}
	if(blockCommentId != null) {
		rules.add(new PatternRule("/*", "*/", new Token(blockCommentId), NO_ESCAPE_CHAR, false, true));
	}
	
	if(stringId != null) {
		rules.add(new PatternRule("\"", "\"", new Token(stringId), '\\', false, true));
	}
	
}
 

public IToken evaluate(ICharacterScanner scanner, boolean resume) {
    if (resume) {
        boolean inString = false;
        do {
            int c = scanner.read();
            if (((char) c) == ',' && !inString)
                break;
            else if (((char) c) == '@') {
                scanner.unread();
                return Token.UNDEFINED;
            } else if (((char) c) == '"' && !inString)
                inString = true;
            else if (((char) c) == '"' && inString)
                inString = false;
            else if (c == ICharacterScanner.EOF)
                return Token.UNDEFINED;
        } while (true);
    }
    return doEvaluate(scanner, 1);
}
 

/**
 * Creates the partitioner and sets up the appropriate rules.
 */
public SamplePartitionScanner() {
	IToken tkString = new Token(LANG_STRING);
	IToken tkRawString = new Token(LANG_RAW_STRING);
	IToken tkCharacter = new Token(LANG_CHARACTER);
	IToken tkSingleComment = new Token(LANG_SINGLE_COMMENT);
	IToken tkMultiComment = new Token(LANG_MULTI_COMMENT);
	
	List<IPredicateRule> rules = new ArrayList<IPredicateRule>();
	
	rules.add(new MultiLineRule("`", "`", tkRawString, NO_ESCAPE, true));
	rules.add(new MultiLineRule("\"", "\"", tkString, '\\', true));
	rules.add(new SingleLineRule("'", "'", tkCharacter, '\\', true));
	
	rules.add(new EndOfLineRule("//", tkSingleComment, NO_ESCAPE));
	
	rules.add(new MultiLineRule("/*", "*/", tkMultiComment, NO_ESCAPE, true));
	
	
	setPredicateRules(rules.toArray(new IPredicateRule[rules.size()]));
}
 

/**
 * 
 */
public CompositePartitionScanner(ISubPartitionScanner defaultPartitionScanner, ISubPartitionScanner primaryPartitionScanner,
		IPartitionerSwitchStrategy partitionerSwitchStrategy) {
	this.defaultPartitionScanner = defaultPartitionScanner;
	this.primaryPartitionScanner = primaryPartitionScanner;

	defaultPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getDefaultSwitchStrategy());
	primaryPartitionScanner.initCharacterScanner(this, partitionerSwitchStrategy.getPrimarySwitchStrategy());

	String[][] pairs = partitionerSwitchStrategy.getSwitchTagPairs();
	switchRules = new IPredicateRule[pairs.length][];
	for (int i = 0; i < pairs.length; ++i) {
		switchRules[i] = new IPredicateRule[] { new SingleTagRule(pairs[i][0], new Token(START_SWITCH_TAG)), new SingleTagRule(pairs[i][1], new Token(END_SWITCH_TAG)) };
	}

	currentPartitionScanner = defaultPartitionScanner;
	setDefaultReturnToken(new Token(IDocument.DEFAULT_CONTENT_TYPE));
}
 

/**
 * HTMLSubPartitionScanner
 */
public HTMLSubPartitionScanner()
{
	// @formatter:off
	super(
			new ISubPartitionScanner[] {
			new SubPartitionScanner(
					HTMLSourceConfiguration.getDefault().getPartitioningRules(),
					HTMLSourceConfiguration.CONTENT_TYPES,
					new Token(HTMLSourceConfiguration.DEFAULT)
					),
					JSSourceConfiguration.getDefault().createSubPartitionScanner(),
					CSSSourceConfiguration.getDefault().createSubPartitionScanner(),
					SVGSourceConfiguration.getDefault().createSubPartitionScanner()
			},
			new IPartitionScannerSwitchStrategy[] {
			new PartitionScannerSwitchStrategy(JS_SWITCH_SEQUENCES),
					new PartitionScannerSwitchStrategy(CSS_SWITCH_SEQUENCES),
					new PartitionScannerSwitchStrategy(SVG_SWITCH_SEQUENCES)
			}
	);
	// @formatter:on
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
}
 

public static void configure(TextLayout textLayout) {
	String text = textLayout.getText();
	Document doc = new Document(text);
	ITokenScanner scanner = getRecipeScanner(doc);
	scanner.setRange(doc, 0, doc.getLength());
	IToken token;
	while ((token = scanner.nextToken()) != Token.EOF) {
		int offset = scanner.getTokenOffset();
		int length = scanner.getTokenLength();
		Object data = token.getData();
		if (data != null && data instanceof TextStyle) {
			TextStyle textStyle = (TextStyle) data;
			textLayout.setStyle(textStyle, offset, offset + length - 1);
		}
	}
	scanner = null;
	doc = null;
}
 

@Override
public IToken evaluate(ICharacterScanner scanner) {
	int ch = scanner.read();
	if (ch == '<') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	} else if (ch == '>') {
		ch = scanner.read();
		if (ch == '/') {
			return token;
		}
		scanner.unread();
		return token;
	}
	scanner.unread();
	return Token.UNDEFINED;
}
 

@Test public void testWholePart() throws Exception {
	PartitionTokenScanner scanner = getPartitionTokenScanner(t(2, 3),t(4,3),t(2,1),t(34,3));
	scanner.setPartialRange(null, 0, 42, "3", 0);
	
	assertEquals("3",scanner.nextToken().getData());
	assertEquals(0,scanner.getTokenOffset());
	assertEquals(6,scanner.getTokenLength());
	
	assertEquals("1",scanner.nextToken().getData());
	assertEquals(6,scanner.getTokenOffset());
	assertEquals(2,scanner.getTokenLength());
	
	assertEquals("3",scanner.nextToken().getData());
	assertEquals(8,scanner.getTokenOffset());
	assertEquals(34,scanner.getTokenLength());
	
	assertSame(Token.EOF, scanner.nextToken());
}
 

protected XMLTagScanner getXMLTagScanner( )
{
	if ( tagScanner == null )
	{
		tagScanner = new XMLTagScanner( colorManager );
		tagScanner.setDefaultReturnToken( new Token( new TextAttribute( colorManager.getColor( IXMLColorConstants.TAG ) ) ) );
	}
	return tagScanner;
}
 

public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if ((c != ICharacterScanner.EOF) && Character.isDigit((char)c)) {
        readLength = 1;
        
        do {
            c = scanner.read();
            readLength++;
        } while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c));
        if (c == '.') {
            do {
                c = scanner.read();
            } while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c));
            if (c == 'E') {
                c = scanner.read();
                if ((c == '+') || (c == '-')) {
                    c = scanner.read();
                }
                while ((c != ICharacterScanner.EOF) && Character.isDigit((char)c)) {
                    c = scanner.read();
                }
            }
            scanner.unread();
            return successToken;
        }
        
        while (readLength > 1) {
            readLength--;
            scanner.unread();
        }
    }
    scanner.unread();
    return Token.UNDEFINED;
}
 

public IToken evaluate(ICharacterScanner scanner, boolean resume)
{
	// Should we try to match the first char first?
	if (firstChar != Character.MIN_VALUE)
	{
		int readChar = scanner.read();
		scanner.unread();
		if (readChar == ICharacterScanner.EOF)
		{
			return Token.EOF;
		}
		if (firstChar != readChar)
		{
			return Token.UNDEFINED;
		}
	}
	String line = readNextLine(scanner);
	if (line == null)
		return Token.EOF;
	Matcher matcher = regexp.matcher(line);
	if (matcher.find() && matcher.start() == 0)
	{
		// Unread back to end of regexp match!
		String match = matcher.group();
		if (match.length() < line.length())
		{
			int toUnread = line.length() - match.length();
			unread(scanner, toUnread);
		}
		return successToken;
	}
	unread(scanner, line.length());
	return Token.UNDEFINED;
}
 

@Test public void testRichStringBetween_01() {
	initializeScanner("��");
	assertNotSame(Token.EOF, scanner.nextToken());
	assertEquals(0, scanner.getTokenOffset());
	assertEquals(2, scanner.getTokenLength());
	assertSame(Token.EOF, scanner.nextToken());
}
 

public void createPresentation(TextPresentation presentation, ITypedRegion region) {
	if (fScanner == null) {
		// will be removed if deprecated constructor will be removed
		addRange(presentation, region.getOffset(), region.getLength(), fDefaultTextStyle);
		return;
	}

	int lastStart = region.getOffset();
	int length = 0;
	boolean firstToken = true;
	IToken lastToken = Token.UNDEFINED;
	TextStyle lastTextStyle = getTokenTextStyle(lastToken);

	fScanner.setRange(fDocument, lastStart, region.getLength());

	while (true) {
		IToken token = fScanner.nextToken();
		if (token.isEOF())
			break;

		TextStyle textStyle = getTokenTextStyle(token);
		if (lastTextStyle != null && lastTextStyle.equals(textStyle)) {
			length += fScanner.getTokenLength();
			firstToken = false;
		} else {
			if (!firstToken)
				addRange(presentation, lastStart, length, lastTextStyle);
			firstToken = false;
			lastToken = token;
			lastTextStyle = textStyle;
			lastStart = fScanner.getTokenOffset();
			length = fScanner.getTokenLength();
		}
	}

	addRange(presentation, lastStart, length, lastTextStyle);
}
 

/**
 * Defines a tikz environment or single line scanner and sets the default color for it
 * @return a scanner to detect argument partitions
 */
protected TexTikzScanner getTexTikzScanner() {
    if (tikzScanner == null) {
        tikzScanner = new TexTikzScanner(colorManager);
        tikzScanner.setDefaultReturnToken(
                new Token(
                        new TextAttribute(
                                colorManager.getColor(ColorManager.DEFAULT),
                                null,
                                colorManager.getStyle(ColorManager.DEFAULT_STYLE))));
    }
    return tikzScanner;
}
 

/**
 * Defines an optional argument (square bracket) scanner and sets the default color for it
 * @return a scanner to detect argument partitions
 */
protected TexOptArgScanner getTexOptArgScanner() {
    if (optArgumentScanner == null) {
        optArgumentScanner = new TexOptArgScanner(colorManager);
        optArgumentScanner.setDefaultReturnToken(
                new Token(
                        new TextAttribute(
                                colorManager.getColor(ColorManager.SQUARE_BRACKETS),
                                null,
                                colorManager.getStyle(ColorManager.SQUARE_BRACKETS_STYLE))));
    }
    return optArgumentScanner;
}
 

/**
 * Defines an argument (curly bracket) scanner and sets the default color for it
 * @return a scanner to detect argument partitions
 */
protected TexArgScanner getTexArgScanner() {
    if (argumentScanner == null) {
        argumentScanner = new TexArgScanner(colorManager);
        argumentScanner.setDefaultReturnToken(
                new Token(
                        new TextAttribute(
                                colorManager.getColor(ColorManager.CURLY_BRACKETS),
                                null,
                                colorManager.getStyle(ColorManager.CURLY_BRACKETS_STYLE))));
    }
    return argumentScanner;
}
 

protected IToken doEvaluate(ICharacterScanner scanner, boolean resume)
{
	int column = scanner.getColumn();
	IToken token = super.doEvaluate(scanner, resume);
	if (token.isUndefined())
		return token;

	// Make sure whole thing matches pattern
	int read = scanner.getColumn() - column;
	for (int i = 0; i < read; i++)
	{
		scanner.unread();
	}
	StringBuilder builder = new StringBuilder();
	for (int i = 0; i < read; i++)
	{
		builder.append((char) scanner.read());
	}
	String word = builder.toString();
	if (word.length() > 2 && ENTITY_PATTERN.matcher(word).find())
	{
		return token;
	}
	for (int i = 0; i < read; i++)
	{
		scanner.unread();
	}
	return Token.UNDEFINED;
}
 

/**
 * Defines a default partition scanner and sets the default
 * color for it
 * @return 	a scanner to find default partitions.
 */
protected TexScanner getTexScanner() {
    if (scanner == null) {
        scanner = new TexScanner(colorManager);
        scanner.setDefaultReturnToken(
                new Token(
                        new TextAttribute(
                                colorManager.getColor(ColorManager.DEFAULT),
                                null,
                                colorManager.getStyle(ColorManager.DEFAULT_STYLE))));
    }
    return scanner;
}
 

@Override
public void setPartialRange(IDocument document, int offset, int length, String contentType, int partitionOffset) {
	defaultTokenState = null;
	hasResume = false;
	resetRules(defaultPartitionScanner.getRules());
	resetRules(primaryPartitionScanner.getRules());
	currentPartitionScanner = defaultPartitionScanner;
	currentPartitionScanner.setLastToken(new Token(contentType));
	if (IDocument.DEFAULT_CONTENT_TYPE.equals(contentType) && partitioner != null) {
		TypedPosition partition = partitioner.findClosestPosition(offset);
		if (partition != null) {
			if (partition.overlapsWith(offset, length)) {
				partition = partitioner.findClosestPosition(offset - 1);
			}
		}
		if (partition != null) {
			String type = partition.getType();
			if (primaryPartitionScanner.hasContentType(type)) {
				currentPartitionScanner = primaryPartitionScanner;
			} else if (START_SWITCH_TAG.equals(type)) {
				hasSwitch = true;
			}
			currentPartitionScanner.setLastToken(new Token(type));
		}
	} else if (primaryPartitionScanner.hasContentType(contentType)) {
		currentPartitionScanner = primaryPartitionScanner;
	}
	super.setPartialRange(document, offset, length, contentType, partitionOffset);
}
 

public IToken evaluate(ICharacterScanner scanner) {
    int c = scanner.read();
    if (((char) c) == '@') {
            do {
                c = scanner.read();
            } while (Character.isLetter((char) c));
            scanner.unread();
            return fToken;
    }
    scanner.unread();
    return Token.UNDEFINED;
}