Java Code Examples for com.intellij.lexer.Lexer#getTokenText()

The following examples show how to use com.intellij.lexer.Lexer#getTokenText() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: ConsoleViewUtil.java    From consulo with Apache License 2.0 6 votes vote down vote up 
public static void printWithHighlighting(@Nonnull ConsoleView console, @Nonnull String text, @Nonnull SyntaxHighlighter highlighter, Runnable doOnNewLine) {
  Lexer lexer = highlighter.getHighlightingLexer();
  lexer.start(text, 0, text.length(), 0);

  IElementType tokenType;
  while ((tokenType = lexer.getTokenType()) != null) {
    ConsoleViewContentType contentType = getContentTypeForToken(tokenType, highlighter);
    StringTokenizer eolTokenizer = new StringTokenizer(lexer.getTokenText(), "\n", true);
    while (eolTokenizer.hasMoreTokens()) {
      String tok = eolTokenizer.nextToken();
      console.print(tok, contentType);
      if (doOnNewLine != null && "\n".equals(tok)) {
        doOnNewLine.run();
      }
    }

    lexer.advance();
  }
}
Example 2
Source File: UnescapingPsiBuilder.java    From BashSupport with Apache License 2.0 4 votes vote down vote up 
private void initTokenListAndCharSequence(Lexer lexer) {
    lexer.start(processedText);

    myShrunkSequence = new ArrayList<MyShiftedToken>(512); //assume a larger token size by default
    StringBuilder charSequenceBuilder = new StringBuilder();

    int realPos = 0;
    int shrunkPos = 0;
    while (lexer.getTokenType() != null) {
        final IElementType tokenType = lexer.getTokenType();
        final String tokenText = lexer.getTokenText();

        int tokenStart = lexer.getTokenStart();
        int tokenEnd = lexer.getTokenEnd();
        int realLength = tokenEnd - tokenStart;

        int delta = textProcessor.getContentRange().getStartOffset();
        int originalStart = textProcessor.getOffsetInHost(tokenStart - delta);
        int originalEnd = textProcessor.getOffsetInHost(tokenEnd - delta);

        if (textProcessor.containsRange(tokenStart, tokenEnd) && originalStart != -1 && originalEnd != -1) {
            realLength = originalEnd - originalStart;
            int masqueLength = tokenEnd - tokenStart;

            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + masqueLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += masqueLength;
        } else {
            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + realLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += realLength;
        }

        realPos += realLength;

        lexer.advance();
    }

    myShrunkCharSequence = charSequenceBuilder.toString();
    myShrunkSequenceSize = myShrunkSequence.size();
}