com.intellij.lexer.Lexer Java Examples

public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}
 

@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{if $var}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 

protected void checkCorrectRestart(String text) {
  Lexer mainLexer = createLexer();
  String allTokens = printTokens(text, 0, mainLexer);

  Lexer auxLexer = createLexer();
  auxLexer.start(text);
  while (true) {
    IElementType type = auxLexer.getTokenType();
    if (type == null) {
      break;
    }
    if (auxLexer.getState() == 0) {
      int tokenStart = auxLexer.getTokenStart();
      String subTokens = printTokens(text, tokenStart, mainLexer);
      if (!allTokens.endsWith(subTokens)) {
        assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens);
      }
    }
    auxLexer.advance();
  }
}
 

private static Expression parseVariable(Lexer lexer, String expression) {
  String variableName = getString(lexer, expression);
  advance(lexer);

  if (lexer.getTokenType() == null) {
    if (TemplateImpl.END.equals(variableName)) {
      return new EmptyNode();
    }

    return new VariableNode(variableName, null);
  }

  if (lexer.getTokenType() != MacroTokenType.EQ) {
    return new VariableNode(variableName, null);
  }

  advance(lexer);
  Expression node = parseMacro(lexer, expression);
  return new VariableNode(variableName, node);
}
 

@Override
public IElementType merge(IElementType type, Lexer lexer) {
    for (MergeTuple currentTuple : mergeTuples) {
        TokenSet tokensToMerge = currentTuple.getTokensToMerge();

        if (tokensToMerge.contains(type)) {
            IElementType current = lexer.getTokenType();
            //merge all upcoming tokens into the target token type
            while (tokensToMerge.contains(current)) {
                lexer.advance();

                current = lexer.getTokenType();
            }

            return currentTuple.getTargetType();
        }
    }

    return type;
}
 

public static String printTokens(CharSequence text, int start, Lexer lexer) {
  lexer.start(text, start, text.length());
  String result = "";
  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      break;
    }
    String tokenText = getTokenText(lexer);
    String tokenTypeName = tokenType.toString();
    String line = tokenTypeName + " ('" + tokenText + "')\n";
    result += line;
    lexer.advance();
  }
  return result;
}
 

@Nullable
public static Lexer getLexerBasedOnLexerHighlighter(CharSequence text, VirtualFile virtualFile, Project project) {
  EditorHighlighter highlighter = null;

  PsiFile psiFile = virtualFile != null ? PsiManager.getInstance(project).findFile(virtualFile) : null;
  final Document document = psiFile != null ? PsiDocumentManager.getInstance(project).getDocument(psiFile) : null;
  final EditorHighlighter cachedEditorHighlighter;
  boolean alreadyInitializedHighlighter = false;

  if (document != null &&
      (cachedEditorHighlighter = getEditorHighlighterForCachesBuilding(document)) != null &&
      PlatformIdTableBuilding.checkCanUseCachedEditorHighlighter(text, cachedEditorHighlighter)) {
    highlighter = cachedEditorHighlighter;
    alreadyInitializedHighlighter = true;
  }
  else if (virtualFile != null) {
    highlighter = EditorHighlighterFactory.getInstance().createEditorHighlighter(project, virtualFile);
  }

  if (highlighter != null) {
    return new LexerEditorHighlighterLexer(highlighter, alreadyInitializedHighlighter);
  }
  return null;
}
 

@Override
@Nonnull
public SyntaxHighlighter getSyntaxHighlighter(final Project project, final VirtualFile virtualFile) {
  return new SyntaxHighlighterBase() {
    @Nonnull
    @Override
    public Lexer getHighlightingLexer() {
      return createPlainTextLexer();
    }

    @Nonnull
    @Override
    public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
      return EMPTY;
    }
  };
}
 

@Test
@SuppressWarnings("unchecked")
public void testArrayItemsWithDoubleArrow() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{block test, te => $item, test . 1 => 123}{/block}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "te"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS, "=>"), //todo: weird behavior, it must be T_PHP_DOUBLE_ARROW (it is because is workaround for "=>" in LatteParser.bnf)
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$item"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_CONCATENATION, "."),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "1"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DOUBLE_ARROW, "=>"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 

@Nonnull
@Override
public PsiBuilder createBuilder(@Nonnull final ParserDefinition parserDefinition,
                                @Nonnull final Lexer lexer,
                                @Nonnull LanguageVersion languageVersion,
                                @Nonnull final CharSequence seq) {
  return new PsiBuilderImpl(null, null, parserDefinition, lexer, languageVersion, null, seq, null, null);
}
 

@Override
protected void lookAhead(Lexer baseLexer) {
	IElementType currentToken = baseLexer.getTokenType();

	if (currentToken != LatteTypes.T_TEXT && LatteHtmlUtil.HTML_TOKENS.contains(currentToken)) {
		advanceLexer(baseLexer);
		replaceCachedType(0, LatteTypes.T_TEXT);

	} else {
		super.lookAhead(baseLexer);
	}
}
 

@Nonnull
public abstract PsiBuilder createBuilder(@Nonnull Project project,
                                         @Nonnull LighterLazyParseableNode chameleon,
                                         @Nullable Lexer lexer,
                                         @Nonnull Language lang,
                                         @Nonnull LanguageVersion languageVersion,
                                         @Nonnull CharSequence seq);
 

/**
 * Checks that given lexer returns the correct tokens.
 *
 * @param lexer
 * @param expectedTokens
 */
public static void assertTokens(Lexer lexer, Pair<IElementType, String>[] expectedTokens) {
	int i;
	for (i = 0; lexer.getTokenType() != null; i++) {
		if (i == expectedTokens.length) fail("Too many tokens from lexer; unexpected " + lexer.getTokenType());
		assertEquals("Wrong token type at index " + i, expectedTokens[i].first, lexer.getTokenType());
		assertEquals("Wrong token text at index " + i, expectedTokens[i].second, lexer.getTokenText());
		lexer.advance();
	}
	if (i < expectedTokens.length) fail("Not enough tokens from lexer; expected " + expectedTokens[i].first + " but got nothing.");
}
 

@Nonnull
public static Expression parse(@Nullable String expression) {
  if (StringUtil.isEmpty(expression)) {
    return new ConstantNode("");
  }
  Lexer lexer = new MacroLexer();
  lexer.start(expression);
  skipWhitespaces(lexer);
  return parseMacro(lexer, expression);
}
 

@Test
@SuppressWarnings("unchecked")
public void testMethod() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{$object->getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->getFoo()->getBar($var)}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 

private static void parseParameters(MacroCallNode macroCallNode, Lexer lexer, String expression) {
  if (lexer.getTokenType() != MacroTokenType.RPAREN) {
    while (lexer.getTokenType() != null) {
      Expression node = parseMacro(lexer, expression);
      macroCallNode.addParameter(node);

      if (lexer.getTokenType() == MacroTokenType.COMMA) {
        advance(lexer);
      }
      else {
        break;
      }
    }
  }
}
 

private PsiBuilderImpl(@Nullable Project project,
                       @Nullable PsiFile containingFile,
                       @Nonnull LanguageVersion languageVersion,
                       @Nonnull ParserDefinition parserDefinition,
                       @Nonnull Lexer lexer,
                       @Nullable CharTable charTable,
                       @Nonnull CharSequence text,
                       @Nullable ASTNode originalTree,
                       @Nullable CharSequence lastCommittedText,
                       @Nullable MyTreeStructure parentLightTree,
                       @Nullable Object parentCachingNode) {
  myProject = project;
  myFile = containingFile;
  myLanguageVersion = languageVersion;
  myText = text;
  myTextArray = CharArrayUtil.fromSequenceWithoutCopying(text);
  myLexer = lexer;

  myWhitespaces = parserDefinition.getWhitespaceTokens(languageVersion);
  myComments = parserDefinition.getCommentTokens(languageVersion);
  myCharTable = charTable;
  myOriginalTree = originalTree;
  myLastCommittedText = lastCommittedText;
  if ((originalTree == null) != (lastCommittedText == null)) {
    throw new IllegalArgumentException("originalTree and lastCommittedText must be null/notnull together but got: originalTree=" +
                                       originalTree +
                                       "; lastCommittedText=" +
                                       (lastCommittedText == null ? null : "'" + StringUtil.first(lastCommittedText, 80, true) + "'"));
  }
  myParentLightTree = parentLightTree;
  myOffset = parentCachingNode instanceof LazyParseableToken ? ((LazyParseableToken)parentCachingNode).getStartOffset() : 0;

  cacheLexemes(parentCachingNode);
}
 

@Nullable
public static String isReservedKeyword(@NotNull String string) {
  Lexer lexer = JAVA_LEXER;
  lexer.start(string);
  if (lexer.getTokenType() != JavaTokenType.IDENTIFIER) {
    if (lexer.getTokenType() instanceof IKeywordElementType) {
      return "Package names cannot contain Java keywords like '" + string + "'";
    }
    if (string.isEmpty()) {
      return "Package segments must be of non-zero length";
    }
    return string + " is not a valid identifier";
  }
  return null;
}
 

public UnescapingPsiBuilder(@NotNull final Project project,
                            @NotNull final ParserDefinition parserDefinition,
                            @NotNull final Lexer lexer,
                            @NotNull final ASTNode chameleon,
                            @NotNull final CharSequence originalText,
                            @NotNull final CharSequence processedText,
                            @NotNull final TextPreprocessor textProcessor) {
    this(new PsiBuilderImpl(project, parserDefinition, lexer, chameleon, originalText), textProcessor, processedText);
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new SimpleLexerAdapter();
}
 

@Nonnull
@VisibleForTesting
static Lexer createDefaultLexer() {
  return new MergingLexerAdapter(new FileTemplateTextLexer(), TokenSet.create(FileTemplateTokenType.TEXT));
}
 

@Nonnull
public TextChunk[] createTextChunks(@Nonnull UsageInfo2UsageAdapter usageInfo2UsageAdapter,
                                    @Nonnull CharSequence chars,
                                    int start,
                                    int end,
                                    boolean selectUsageWithBold,
                                    @Nonnull List<TextChunk> result) {
  final Lexer lexer = myHighlighter.getHighlightingLexer();
  final SyntaxHighlighterOverEditorHighlighter highlighter = myHighlighter;

  LOG.assertTrue(start <= end);

  int i = StringUtil.indexOf(chars, '\n', start, end);
  if (i != -1) end = i;

  if (myDocumentStamp != myDocument.getModificationStamp()) {
    highlighter.restart(chars);
    myDocumentStamp = myDocument.getModificationStamp();
  }
  else if (lexer.getTokenType() == null || lexer.getTokenStart() > start) {
    highlighter.resetPosition(0);  // todo restart from nearest position with initial state
  }

  boolean isBeginning = true;

  for (; lexer.getTokenType() != null; lexer.advance()) {
    int hiStart = lexer.getTokenStart();
    int hiEnd = lexer.getTokenEnd();

    if (hiStart >= end) break;

    hiStart = Math.max(hiStart, start);
    hiEnd = Math.min(hiEnd, end);
    if (hiStart >= hiEnd) {
      continue;
    }

    if (isBeginning) {
      String text = chars.subSequence(hiStart, hiEnd).toString();
      if (text.trim().isEmpty()) continue;
    }
    isBeginning = false;
    IElementType tokenType = lexer.getTokenType();
    TextAttributesKey[] tokenHighlights = highlighter.getTokenHighlights(tokenType);

    processIntersectingRange(usageInfo2UsageAdapter, chars, hiStart, hiEnd, tokenHighlights, selectUsageWithBold, result);
  }

  return result.toArray(new TextChunk[result.size()]);
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
	return new HeaderLexerAdapter();
}
 

@Test
@SuppressWarnings("unchecked")
public void testMacroLexer() {
	Lexer lexer = new LatteHighlightingLexer(new LatteLookAheadLexer(new LatteLexer()));

	lexer.start("{include #blockName}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "include"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_BLOCK_NAME, "#"),
		Pair.create(T_BLOCK_NAME, "blockName"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= dump ()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_METHOD, "dump"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Foo\\Bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Bar}");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
			Pair.create(T_MACRO_SHORTNAME, "="),
			Pair.create(T_WHITESPACE, " "),
			Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
			Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
			Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 

static Lexer createLexerStatic(Project project) {
  return new FlexAdapter(new _CppLexer(false, false, true, true, true)); // TODO: different lexers needed!
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new FlexAdapter(new DustLexer((Reader) null));
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new JenkinsLexerAdapter();
}
 

public String fun(final CharSequence charSequence) {
  Lexer lexer = LanguageParserDefinitions.INSTANCE.forLanguage(HaxeLanguage.INSTANCE).createLexer(null);
  lexer.start(charSequence);
  return readPackageName(charSequence, lexer);
}
 

@NotNull
public Lexer createLexer(Project project) {
    return createBashLexer(project);
}
 

public Parser(Lexer lexer) {
  myLexer = lexer;
}