Java Code Examples for com.intellij.lexer.Lexer
The following examples show how to use
com.intellij.lexer.Lexer. These examples are extracted from open source projects.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: consulo Source File: PlainTextSyntaxHighlighterFactory.java License: Apache License 2.0 | 6 votes |
@Override @Nonnull public SyntaxHighlighter getSyntaxHighlighter(final Project project, final VirtualFile virtualFile) { return new SyntaxHighlighterBase() { @Nonnull @Override public Lexer getHighlightingLexer() { return createPlainTextLexer(); } @Nonnull @Override public TextAttributesKey[] getTokenHighlights(IElementType tokenType) { return EMPTY; } }; }
Example 2
Source Project: BashSupport Source File: MergingLexer.java License: Apache License 2.0 | 6 votes |
@Override public IElementType merge(IElementType type, Lexer lexer) { for (MergeTuple currentTuple : mergeTuples) { TokenSet tokensToMerge = currentTuple.getTokensToMerge(); if (tokensToMerge.contains(type)) { IElementType current = lexer.getTokenType(); //merge all upcoming tokens into the target token type while (tokensToMerge.contains(current)) { lexer.advance(); current = lexer.getTokenType(); } return currentTuple.getTargetType(); } } return type; }
Example 3
Source Project: intellij-latte Source File: LattePhpLexerTest.java License: MIT License | 6 votes |
@Test @SuppressWarnings("unchecked") public void testBasic() throws Exception { Lexer lexer = new LatteLexer(); lexer.start("{if $var}B{/if}"); assertTokens(lexer, new Pair[] { Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"), Pair.create(T_MACRO_NAME, "if"), Pair.create(T_WHITESPACE, " "), Pair.create(T_MACRO_ARGS_VAR, "$var"), Pair.create(T_MACRO_TAG_CLOSE, "}"), Pair.create(T_TEXT, "B"), Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"), Pair.create(T_MACRO_NAME, "if"), Pair.create(T_MACRO_TAG_CLOSE, "}"), }); }
Example 4
Source Project: consulo Source File: SelectWordUtil.java License: Apache License 2.0 | 6 votes |
public static void addWordHonoringEscapeSequences(CharSequence editorText, TextRange literalTextRange, int cursorOffset, Lexer lexer, List<TextRange> result) { lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset()); while (lexer.getTokenType() != null) { if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) { if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) { result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd())); } else { TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION); if (word != null) { result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()), Math.min(word.getEndOffset(), lexer.getTokenEnd()))); } } break; } lexer.advance(); } }
Example 5
Source Project: consulo Source File: LexerTestCase.java License: Apache License 2.0 | 6 votes |
protected void checkCorrectRestart(String text) { Lexer mainLexer = createLexer(); String allTokens = printTokens(text, 0, mainLexer); Lexer auxLexer = createLexer(); auxLexer.start(text); while (true) { IElementType type = auxLexer.getTokenType(); if (type == null) { break; } if (auxLexer.getState() == 0) { int tokenStart = auxLexer.getTokenStart(); String subTokens = printTokens(text, tokenStart, mainLexer); if (!allTokens.endsWith(subTokens)) { assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens); } } auxLexer.advance(); } }
Example 6
Source Project: consulo Source File: MacroParser.java License: Apache License 2.0 | 6 votes |
private static Expression parseVariable(Lexer lexer, String expression) { String variableName = getString(lexer, expression); advance(lexer); if (lexer.getTokenType() == null) { if (TemplateImpl.END.equals(variableName)) { return new EmptyNode(); } return new VariableNode(variableName, null); } if (lexer.getTokenType() != MacroTokenType.EQ) { return new VariableNode(variableName, null); } advance(lexer); Expression node = parseMacro(lexer, expression); return new VariableNode(variableName, node); }
Example 7
Source Project: consulo Source File: LexerTestCase.java License: Apache License 2.0 | 6 votes |
public static String printTokens(CharSequence text, int start, Lexer lexer) { lexer.start(text, start, text.length()); String result = ""; while (true) { IElementType tokenType = lexer.getTokenType(); if (tokenType == null) { break; } String tokenText = getTokenText(lexer); String tokenTypeName = tokenType.toString(); String line = tokenTypeName + " ('" + tokenText + "')\n"; result += line; lexer.advance(); } return result; }
Example 8
Source Project: consulo Source File: EditorHighlighterCache.java License: Apache License 2.0 | 6 votes |
@Nullable public static Lexer getLexerBasedOnLexerHighlighter(CharSequence text, VirtualFile virtualFile, Project project) { EditorHighlighter highlighter = null; PsiFile psiFile = virtualFile != null ? PsiManager.getInstance(project).findFile(virtualFile) : null; final Document document = psiFile != null ? PsiDocumentManager.getInstance(project).getDocument(psiFile) : null; final EditorHighlighter cachedEditorHighlighter; boolean alreadyInitializedHighlighter = false; if (document != null && (cachedEditorHighlighter = getEditorHighlighterForCachesBuilding(document)) != null && PlatformIdTableBuilding.checkCanUseCachedEditorHighlighter(text, cachedEditorHighlighter)) { highlighter = cachedEditorHighlighter; alreadyInitializedHighlighter = true; } else if (virtualFile != null) { highlighter = EditorHighlighterFactory.getInstance().createEditorHighlighter(project, virtualFile); } if (highlighter != null) { return new LexerEditorHighlighterLexer(highlighter, alreadyInitializedHighlighter); } return null; }
Example 9
Source Project: consulo Source File: PsiBuilderFactory.java License: Apache License 2.0 | 5 votes |
@Nonnull public abstract PsiBuilder createBuilder(@Nonnull Project project, @Nonnull LighterLazyParseableNode chameleon, @Nullable Lexer lexer, @Nonnull Language lang, @Nonnull LanguageVersion languageVersion, @Nonnull CharSequence seq);
Example 10
Source Project: consulo Source File: PsiBuilderFactoryImpl.java License: Apache License 2.0 | 5 votes |
@Nonnull @Override public PsiBuilder createBuilder(@Nonnull final ParserDefinition parserDefinition, @Nonnull final Lexer lexer, @Nonnull LanguageVersion languageVersion, @Nonnull final CharSequence seq) { return new PsiBuilderImpl(null, null, parserDefinition, lexer, languageVersion, null, seq, null, null); }
Example 11
Source Project: intellij-latte Source File: LatteHtmlHighlightingLexer.java License: MIT License | 5 votes |
@Override protected void lookAhead(Lexer baseLexer) { IElementType currentToken = baseLexer.getTokenType(); if (currentToken != LatteTypes.T_TEXT && LatteHtmlUtil.HTML_TOKENS.contains(currentToken)) { advanceLexer(baseLexer); replaceCachedType(0, LatteTypes.T_TEXT); } else { super.lookAhead(baseLexer); } }
Example 12
Source Project: intellij-latte Source File: LattePhpLexerTest.java License: MIT License | 5 votes |
@Test @SuppressWarnings("unchecked") public void testArrayItemsWithDoubleArrow() throws Exception { Lexer lexer = new LatteHighlightingLexer(new LatteLexer()); lexer.start("{block test, te => $item, test . 1 => 123}{/block}"); assertTokens(lexer, new Pair[] { Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"), Pair.create(T_MACRO_NAME, "block"), Pair.create(T_WHITESPACE, " "), Pair.create(T_PHP_IDENTIFIER, "test"), Pair.create(T_MACRO_ARGS, ","), Pair.create(T_WHITESPACE, " "), Pair.create(T_PHP_IDENTIFIER, "te"), Pair.create(T_WHITESPACE, " "), Pair.create(T_MACRO_ARGS, "=>"), //todo: weird behavior, it must be T_PHP_DOUBLE_ARROW (it is because is workaround for "=>" in LatteParser.bnf) Pair.create(T_WHITESPACE, " "), Pair.create(T_MACRO_ARGS_VAR, "$item"), Pair.create(T_MACRO_ARGS, ","), Pair.create(T_WHITESPACE, " "), Pair.create(T_PHP_IDENTIFIER, "test"), Pair.create(T_WHITESPACE, " "), Pair.create(T_PHP_CONCATENATION, "."), Pair.create(T_WHITESPACE, " "), Pair.create(T_MACRO_ARGS_NUMBER, "1"), Pair.create(T_WHITESPACE, " "), Pair.create(T_PHP_DOUBLE_ARROW, "=>"), Pair.create(T_WHITESPACE, " "), Pair.create(T_MACRO_ARGS_NUMBER, "123"), Pair.create(T_MACRO_TAG_CLOSE, "}"), Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"), Pair.create(T_MACRO_NAME, "block"), Pair.create(T_MACRO_TAG_CLOSE, "}"), }); }
Example 13
Source Project: intellij-latte Source File: Assert.java License: MIT License | 5 votes |
/** * Checks that given lexer returns the correct tokens. * * @param lexer * @param expectedTokens */ public static void assertTokens(Lexer lexer, Pair<IElementType, String>[] expectedTokens) { int i; for (i = 0; lexer.getTokenType() != null; i++) { if (i == expectedTokens.length) fail("Too many tokens from lexer; unexpected " + lexer.getTokenType()); assertEquals("Wrong token type at index " + i, expectedTokens[i].first, lexer.getTokenType()); assertEquals("Wrong token text at index " + i, expectedTokens[i].second, lexer.getTokenText()); lexer.advance(); } if (i < expectedTokens.length) fail("Not enough tokens from lexer; expected " + expectedTokens[i].first + " but got nothing."); }
Example 14
Source Project: consulo Source File: MacroParser.java License: Apache License 2.0 | 5 votes |
private static void parseParameters(MacroCallNode macroCallNode, Lexer lexer, String expression) { if (lexer.getTokenType() != MacroTokenType.RPAREN) { while (lexer.getTokenType() != null) { Expression node = parseMacro(lexer, expression); macroCallNode.addParameter(node); if (lexer.getTokenType() == MacroTokenType.COMMA) { advance(lexer); } else { break; } } } }
Example 15
Source Project: intellij-latte Source File: LattePhpLexerTest.java License: MIT License | 5 votes |
@Test @SuppressWarnings("unchecked") public void testMethod() throws Exception { Lexer lexer = new LatteHighlightingLexer(new LatteLexer()); lexer.start("{$object->getFoo()}"); assertTokens(lexer, new Pair[] { Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"), Pair.create(T_MACRO_ARGS_VAR, "$object"), Pair.create(T_PHP_OBJECT_OPERATOR, "->"), Pair.create(T_PHP_METHOD, "getFoo"), Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("), Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"), Pair.create(T_MACRO_TAG_CLOSE, "}"), }); lexer.start("{$object->getFoo()->getBar($var)}"); assertTokens(lexer, new Pair[] { Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"), Pair.create(T_MACRO_ARGS_VAR, "$object"), Pair.create(T_PHP_OBJECT_OPERATOR, "->"), Pair.create(T_PHP_METHOD, "getFoo"), Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("), Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"), Pair.create(T_PHP_OBJECT_OPERATOR, "->"), Pair.create(T_PHP_METHOD, "getBar"), Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("), Pair.create(T_MACRO_ARGS_VAR, "$var"), Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"), Pair.create(T_MACRO_TAG_CLOSE, "}"), }); }
Example 16
Source Project: consulo Source File: PsiBuilderImpl.java License: Apache License 2.0 | 5 votes |
private PsiBuilderImpl(@Nullable Project project, @Nullable PsiFile containingFile, @Nonnull LanguageVersion languageVersion, @Nonnull ParserDefinition parserDefinition, @Nonnull Lexer lexer, @Nullable CharTable charTable, @Nonnull CharSequence text, @Nullable ASTNode originalTree, @Nullable CharSequence lastCommittedText, @Nullable MyTreeStructure parentLightTree, @Nullable Object parentCachingNode) { myProject = project; myFile = containingFile; myLanguageVersion = languageVersion; myText = text; myTextArray = CharArrayUtil.fromSequenceWithoutCopying(text); myLexer = lexer; myWhitespaces = parserDefinition.getWhitespaceTokens(languageVersion); myComments = parserDefinition.getCommentTokens(languageVersion); myCharTable = charTable; myOriginalTree = originalTree; myLastCommittedText = lastCommittedText; if ((originalTree == null) != (lastCommittedText == null)) { throw new IllegalArgumentException("originalTree and lastCommittedText must be null/notnull together but got: originalTree=" + originalTree + "; lastCommittedText=" + (lastCommittedText == null ? null : "'" + StringUtil.first(lastCommittedText, 80, true) + "'")); } myParentLightTree = parentLightTree; myOffset = parentCachingNode instanceof LazyParseableToken ? ((LazyParseableToken)parentCachingNode).getStartOffset() : 0; cacheLexemes(parentCachingNode); }
Example 17
Source Project: flutter-intellij Source File: AndroidUtils.java License: BSD 3-Clause "New" or "Revised" License | 5 votes |
@Nullable public static String isReservedKeyword(@NotNull String string) { Lexer lexer = JAVA_LEXER; lexer.start(string); if (lexer.getTokenType() != JavaTokenType.IDENTIFIER) { if (lexer.getTokenType() instanceof IKeywordElementType) { return "Package names cannot contain Java keywords like '" + string + "'"; } if (string.isEmpty()) { return "Package segments must be of non-zero length"; } return string + " is not a valid identifier"; } return null; }
Example 18
Source Project: consulo Source File: MacroParser.java License: Apache License 2.0 | 5 votes |
@Nonnull public static Expression parse(@Nullable String expression) { if (StringUtil.isEmpty(expression)) { return new ConstantNode(""); } Lexer lexer = new MacroLexer(); lexer.start(expression); skipWhitespaces(lexer); return parseMacro(lexer, expression); }
Example 19
Source Project: BashSupport Source File: UnescapingPsiBuilder.java License: Apache License 2.0 | 5 votes |
public UnescapingPsiBuilder(@NotNull final Project project, @NotNull final ParserDefinition parserDefinition, @NotNull final Lexer lexer, @NotNull final ASTNode chameleon, @NotNull final CharSequence originalText, @NotNull final CharSequence processedText, @NotNull final TextPreprocessor textProcessor) { this(new PsiBuilderImpl(project, parserDefinition, lexer, chameleon, originalText), textProcessor, processedText); }
Example 20
Source Project: intellij-jsonnet Source File: JsonnetSyntaxHighlighter.java License: Apache License 2.0 | 4 votes |
@NotNull @Override public Lexer getHighlightingLexer() { return new JsonnetLexerAdapter(); }
Example 21
Source Project: intellij-jsonnet Source File: JsonnetParserDefinition.java License: Apache License 2.0 | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { return new JsonnetLexerAdapter(); }
Example 22
Source Project: JetBrains-NASM-Language Source File: NASMParserDefinition.java License: MIT License | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { return new NASMLexer(); }
Example 23
Source Project: JetBrains-NASM-Language Source File: NASMSyntaxHighlighter.java License: MIT License | 4 votes |
@NotNull @Override public Lexer getHighlightingLexer() { return new NASMLexer(); }
Example 24
Source Project: intellij-plugin-v4 Source File: ANTLRv4ParserDefinition.java License: BSD 3-Clause "New" or "Revised" License | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { ANTLRv4Lexer lexer = new ANTLRv4Lexer(null); return new ANTLRv4LexerAdaptor(lexer); }
Example 25
Source Project: intellij-thrift Source File: ThriftSyntaxHighlighter.java License: Apache License 2.0 | 4 votes |
@NotNull public Lexer getHighlightingLexer() { return new ThriftLexer(); }
Example 26
Source Project: intellij-haskforce Source File: HaskellSyntaxHighlighter.java License: Apache License 2.0 | 4 votes |
@NotNull @Override public Lexer getHighlightingLexer() { return new HaskellSyntaxHighlightingLexer(); }
Example 27
Source Project: intellij-xquery Source File: XQueryParserDefinition.java License: Apache License 2.0 | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { return new XQueryLexer(); }
Example 28
Source Project: reasonml-idea-plugin Source File: OclParserDefinition.java License: MIT License | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { return new OclLexer(); }
Example 29
Source Project: reasonml-idea-plugin Source File: DuneParserDefinition.java License: MIT License | 4 votes |
@NotNull @Override public Lexer createLexer(Project project) { return new FlexAdapter(new DuneLexer(DuneTypes.INSTANCE)); }
Example 30
Source Project: consulo-csharp Source File: CSharpPreprocessorSyntaxHighlighter.java License: Apache License 2.0 | 4 votes |
@Nonnull @Override public Lexer getHighlightingLexer() { return new CSharpPreprocessorHightlightLexer(); }