org.eclipse.xtext.ui.editor.model.ILexerTokenRegion Java Examples

@Override
public IToken nextToken() {
	if (currentTemplateTextToken != null) {
		if (currentTemplateTextToken.hasNext())
			return currentTemplateTextToken.nextToken();
		else
			currentTemplateTextToken = null;
	}
	if (!getIterator().hasNext())
		return Token.EOF;
	ILexerTokenRegion next = getIterator().next();
	int tokenType = next.getLexerTokenType();
	switch (tokenType) {
	case RULE_TEMPLATE_HEAD:
	case RULE_TEMPLATE_MIDDLE: {
		currentTemplateTextToken = createTemplateToken(tokenType, next);
		return currentTemplateTextToken.nextToken();
	}
	default:
		setCurrentToken(next);
		return createToken(next);
	}
}
 

public List<StyleRange> getRanges(String expression) {
	final List<StyleRange> ranges = Lists.newArrayList();
	DocumentEvent event = new DocumentEvent();
	event.fDocument = new DummyDocument(expression);
	DocumentTokenSource tokenSource = tokenSourceProvider.get();
	tokenSource.updateStructure(event);
	Iterator<ILexerTokenRegion> iterator = tokenSource.getTokenInfos().iterator();
	while (iterator.hasNext()) {
		ILexerTokenRegion next = iterator.next();
		TextAttribute attribute = attributeProvider.getAttribute(tokenTypeMapper.getId(next.getLexerTokenType()));
		StyleRange range = new StyleRange(next.getOffset(), next.getLength(), attribute.getForeground(),
				attribute.getBackground());
		range.font = attribute.getFont();
		range.fontStyle = attribute.getStyle();
		ranges.add(range);
	}
	return merge(ranges);
}
 

@Override
public IToken nextToken() {
	if (currentRichTextToken != null) {
		if (currentRichTextToken.hasNext())
			return currentRichTextToken.nextToken();
		else
			currentRichTextToken = null;
	}
	if (!getIterator().hasNext())
		return Token.EOF;
	ILexerTokenRegion next = getIterator().next();
	int tokenType = next.getLexerTokenType();
	if (tokenType >= 0 && allTokenTypesAsString[tokenType] != null) {
		currentRichTextToken = createRichTextToken(allTokenTypesAsString[tokenType], next);
		return currentRichTextToken.nextToken();
	} else {
		setCurrentToken(next);
		return createToken(next);	
	}
}
 

/**
 * Creates a new template token for the given token id. A template token is in fact a list of tokens for the
 * colorer, e.g a single terminal on the grammar level may yield various tokens for the syntax coloring.
 */
protected TemplateToken createTemplateToken(int id, ILexerTokenRegion token) {
	int tokenOffset = token.getOffset();
	int tokenLength = token.getLength();
	TemplateToken result = new TemplateToken();
	switch (id) {
	case RULE_TEMPLATE_HEAD:
		result.delimiter = false; // ranges defined by 'offsets' and 'lengths' do *not* start with a delimiter
		result.offsets = new int[] { tokenOffset, tokenOffset + tokenLength - 2 };
		result.lengths = new int[] { tokenLength - 2, 2 };
		break;
	case RULE_TEMPLATE_MIDDLE:
		if (tokenLength == 2) {
			result.delimiter = true; // ranges defined by 'offsets' and 'lengths' start with a delimiter
			result.offsets = new int[] { tokenOffset };
			result.lengths = new int[] { tokenLength };
		} else {
			result.delimiter = false; // ranges defined by 'offsets' and 'lengths' do *not* start with a delimiter
			result.offsets = new int[] { tokenOffset, tokenOffset + tokenLength - 2 };
			result.lengths = new int[] { tokenLength - 2, 2 };
		}
	}
	return result;
}
 

public void setRange(IDocument document, final int offset, final int length) {
	this.regionOffset = offset;
	overlapFilter = Regions.overlaps(offset, length);
	Iterable<ILexerTokenRegion> newIterable = getTokens(document);
	if (delegateIterable != null) {
		if (!delegateIterable.equals(newIterable)) {
			current = null;
		}
	}
	if (current == null) {
		delegate = newIterable.iterator();
		delegateIterable = newIterable;
		computedHasNext = false;
		hasNext = false;
	} else {
		if (current.getOffset() <= offset && current.getOffset() + current.getLength() > offset) {
			// offset is inside of current - return current as next
			computedHasNext = true;
			hasNext = true;
		} else {
			// restart - use a new delegate
			computedHasNext = false;
			hasNext = false;
			delegate = newIterable.iterator();
			delegateIterable = newIterable;
		}
	}
}
 

protected RichTextToken createRichTextToken(String id, ILexerTokenRegion token) {
	int tokenOffset = token.getOffset();
	int tokenLength = token.getLength();
	RichTextToken result = new RichTextToken();
	if ("RULE_RICH_TEXT_START".equals(id)) {
		result.delimiter = false;
		result.offsets = new int[] { tokenOffset, tokenOffset + tokenLength - 1 };
		result.lengths = new int[] { tokenLength - 1, 1 };
	} else if ("RULE_RICH_TEXT_END".equals(id)) {
		result.delimiter = true;
		result.offsets = new int[] { tokenOffset, tokenOffset + 1 };
		result.lengths = new int[] { 1, tokenLength - 1 };
	} else if ("RULE_RICH_TEXT_INBETWEEN".equals(id)) {
		result.delimiter = true;
		if (tokenLength == 2) {
			result.offsets = new int[] { tokenOffset };
			result.lengths = new int[] { tokenLength };
		} else {
			result.offsets = new int[] { tokenOffset, tokenOffset + 1, tokenOffset + tokenLength - 1 };
			result.lengths = new int[] { 1, tokenLength - 2, 1 };
		}
	} else if ("RULE_COMMENT_RICH_TEXT_END".equals(id)) {
		result.delimiter = true;
		result.offsets = new int[] { tokenOffset, tokenOffset + 2 };
		result.lengths = new int[] { 2, tokenLength - 2 };
	} else if ("RULE_COMMENT_RICH_TEXT_INBETWEEN".equals(id)) {
		result.delimiter = true;
		result.offsets = new int[] { tokenOffset, tokenOffset + 2, tokenOffset + tokenLength - 1 };
		result.lengths = new int[] { 2, tokenLength - 3, 1 };
	} else 
		throw new IllegalArgumentException(id);
	return result;
}
 

protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
	Iterable<ILexerTokenRegion> result = tokenSourceAccess.getTokens(document, true);
	if (result == null) {
		// User might have selected a non-xtext document in e.g. a compare operation.
		// Return an empty iterable, this will disable syntax highlighting
		// for the "non-xtext editor".
		return Collections.emptyList();
	}
	return result;
}
 

@Test public void testInsertComment() throws Exception {
	document.set("bar 345 grammar : so 'baz & so'");
	IRegion region = tokenSource.getLastDamagedRegion();
	ArrayList<ILexerTokenRegion> list = Lists.newArrayList(tokenSource.getTokenInfos());
	document.replace(8, 7, "/*grammar*/");
	IRegion region2 = tokenSource.getLastDamagedRegion();
	assertTrue(!region.equals(region2));
	ArrayList<ILexerTokenRegion> list2 = Lists.newArrayList(tokenSource.getTokenInfos());
	assertTrue(!list.equals(list2));
}
 

@Override
public ILexerTokenRegion next() {
	if (!computedHasNext) {
		// compute hasNext on demand to make sure the internal state is valid
		hasNext();
	}
	if (!hasNext) {
		throw new NoSuchElementException("You should check for #hasNext prior to calling #next");
	}
	ILexerTokenRegion result = current;
	current = null;
	computedHasNext = false;
	hasNext = false;
	return result;
}
 

private IRegion doComputeIntersection(ITypedRegion partition, DocumentEvent e, IDocument document) {
	Iterable<ILexerTokenRegion> tokensInPartition = Iterables.filter(tokenSourceAccess.getTokens(document, false), Regions.overlaps(partition.getOffset(), partition.getLength()));
	Iterator<ILexerTokenRegion> tokens = Iterables.filter(tokensInPartition, Regions.overlaps(e.getOffset(), e.getLength())).iterator();
	if (tokens.hasNext()) {
		ILexerTokenRegion first = tokens.next();
		ILexerTokenRegion last = first;
		while(tokens.hasNext())
			last = tokens.next();
		return new Region(first.getOffset(), last.getOffset()+last.getLength() -first.getOffset());
	}
	// this shouldn't happen, but just in case return the whole partition
	return partition;
}
 

@Override
protected IRegion findWord(IDocument document, int offset) {
	Iterable<ILexerTokenRegion> tokenIterable = tokenSourceAccess.getTokens(document, true);
	if (tokenIterable == null) {
		return super.findWord(document, offset);
	}
	Iterator<ILexerTokenRegion> tokenIterator = tokenIterable.iterator();
	ILexerTokenRegion leadingToken = null;
	ILexerTokenRegion trailingToken = null;
	while(tokenIterator.hasNext()) {
		ILexerTokenRegion token = tokenIterator.next();
		if (token.getOffset() <= offset && token.getOffset() + token.getLength() >= offset) {
			if (leadingToken != null)
				trailingToken = token;
			else
				leadingToken = token;
		}
		if (token.getOffset() > offset)
			break;
	}
	if (leadingToken != null) {
		try {
			if (leadingToken.getLength() > 1 && (trailingToken == null || !Character.isLetter(document.getChar(trailingToken.getOffset())))) {
				return new Region(leadingToken.getOffset(), leadingToken.getLength());
			} else if (trailingToken != null) {
				return new Region(trailingToken.getOffset(), trailingToken.getLength());
			}
		} catch(BadLocationException ignore) {}
	}
	return super.findWord(document, offset);
}
 

public TokenScanner getTokenScanner(int... lengths) throws Exception {
	final List<ILexerTokenRegion> tokens = Lists.newArrayList();
	int offset = 0;
	for (final int length : lengths) {
		final int currentOffset = offset;
		tokens.add(new ILexerTokenRegion() {

			@Override
			public int getLength() {
				return length;
			}

			@Override
			public int getLexerTokenType() {
				return 4711;
			}
			
			@Override
			public int getOffset() {
				return currentOffset;
			}
		});
		offset+=length;
	}
	TokenScanner tokenScanner = new TokenScanner() {
		@Override
		protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
			return tokens;
		}
		
		@Override
		protected IToken createToken(ILexerTokenRegion token) {
			return Token.UNDEFINED;
		}
	};
	return tokenScanner;

}
 

public PartitionTokenScanner getPartitionTokenScanner(ILexerTokenRegion... tokenDescs) throws Exception {
	final List<ILexerTokenRegion> tokens = Arrays.asList(tokenDescs);
	int offset = 0;
	for (ILexerTokenRegion token : tokens) {
		((LexerTokenRegion)token).setOffset(offset);
		offset+=token.getLength();
	}
	PartitionTokenScanner tokenScanner = new PartitionTokenScanner(){
		@Override
		protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
			return tokens;
		}
		@Override
		protected boolean shouldMergePartitions(String contentType) {
			return "3".equals(contentType);
		}
	};
	tokenScanner.setMapper(new ITokenTypeToPartitionTypeMapper() {
		
		@Override
		public String[] getSupportedPartitionTypes() {
			throw new UnsupportedOperationException();
		}
		
		@Override
		public String getPartitionType(int antlrTokenType) {
			return ""+antlrTokenType;
		}
	});
	return tokenScanner;

}
 

@Test public void testConcurrentModification() throws Exception {
	document.set("foo bar");
	Iterator<ILexerTokenRegion> iterator = tokenSource.getTokenInfos().iterator();
	ILexerTokenRegion region = iterator.next();
	region = iterator.next();
	document.set("barfoo");
	region = iterator.next();
	assertEquals(3,region.getLength());
}
 

public List<ILexerTokenRegion> getTokens() {
	return Collections.emptyList();
}
 

protected IToken createToken(ILexerTokenRegion currentToken) {
	String id = tokenIdMapper.getId(currentToken.getLexerTokenType());
	Token token = new Token(getAttribute(id));
	return token;
}
 

protected Iterator<ILexerTokenRegion> getIterator() {
	return reentrantIterator;
}
 

protected ILexerTokenRegion getCurrentToken() {
	return currentToken;
}
 

protected void setCurrentToken(ILexerTokenRegion currentToken) {
	this.currentToken = currentToken;
}
 

public List<ILexerTokenRegion> getTokens() {
	return Collections.emptyList();
}