Java Code Examples for com.intellij.lexer.Lexer

The following examples show how to use com.intellij.lexer.Lexer. These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: consulo   Source File: PlainTextSyntaxHighlighterFactory.java    License: Apache License 2.0 6 votes vote down vote up
@Override
@Nonnull
public SyntaxHighlighter getSyntaxHighlighter(final Project project, final VirtualFile virtualFile) {
  return new SyntaxHighlighterBase() {
    @Nonnull
    @Override
    public Lexer getHighlightingLexer() {
      return createPlainTextLexer();
    }

    @Nonnull
    @Override
    public TextAttributesKey[] getTokenHighlights(IElementType tokenType) {
      return EMPTY;
    }
  };
}
 
Example 2
Source Project: BashSupport   Source File: MergingLexer.java    License: Apache License 2.0 6 votes vote down vote up
@Override
public IElementType merge(IElementType type, Lexer lexer) {
    for (MergeTuple currentTuple : mergeTuples) {
        TokenSet tokensToMerge = currentTuple.getTokensToMerge();

        if (tokensToMerge.contains(type)) {
            IElementType current = lexer.getTokenType();
            //merge all upcoming tokens into the target token type
            while (tokensToMerge.contains(current)) {
                lexer.advance();

                current = lexer.getTokenType();
            }

            return currentTuple.getTargetType();
        }
    }

    return type;
}
 
Example 3
Source Project: intellij-latte   Source File: LattePhpLexerTest.java    License: MIT License 6 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{if $var}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 
Example 4
Source Project: consulo   Source File: SelectWordUtil.java    License: Apache License 2.0 6 votes vote down vote up
public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}
 
Example 5
Source Project: consulo   Source File: LexerTestCase.java    License: Apache License 2.0 6 votes vote down vote up
protected void checkCorrectRestart(String text) {
  Lexer mainLexer = createLexer();
  String allTokens = printTokens(text, 0, mainLexer);

  Lexer auxLexer = createLexer();
  auxLexer.start(text);
  while (true) {
    IElementType type = auxLexer.getTokenType();
    if (type == null) {
      break;
    }
    if (auxLexer.getState() == 0) {
      int tokenStart = auxLexer.getTokenStart();
      String subTokens = printTokens(text, tokenStart, mainLexer);
      if (!allTokens.endsWith(subTokens)) {
        assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens);
      }
    }
    auxLexer.advance();
  }
}
 
Example 6
Source Project: consulo   Source File: MacroParser.java    License: Apache License 2.0 6 votes vote down vote up
private static Expression parseVariable(Lexer lexer, String expression) {
  String variableName = getString(lexer, expression);
  advance(lexer);

  if (lexer.getTokenType() == null) {
    if (TemplateImpl.END.equals(variableName)) {
      return new EmptyNode();
    }

    return new VariableNode(variableName, null);
  }

  if (lexer.getTokenType() != MacroTokenType.EQ) {
    return new VariableNode(variableName, null);
  }

  advance(lexer);
  Expression node = parseMacro(lexer, expression);
  return new VariableNode(variableName, node);
}
 
Example 7
Source Project: consulo   Source File: LexerTestCase.java    License: Apache License 2.0 6 votes vote down vote up
public static String printTokens(CharSequence text, int start, Lexer lexer) {
  lexer.start(text, start, text.length());
  String result = "";
  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      break;
    }
    String tokenText = getTokenText(lexer);
    String tokenTypeName = tokenType.toString();
    String line = tokenTypeName + " ('" + tokenText + "')\n";
    result += line;
    lexer.advance();
  }
  return result;
}
 
Example 8
Source Project: consulo   Source File: EditorHighlighterCache.java    License: Apache License 2.0 6 votes vote down vote up
@Nullable
public static Lexer getLexerBasedOnLexerHighlighter(CharSequence text, VirtualFile virtualFile, Project project) {
  EditorHighlighter highlighter = null;

  PsiFile psiFile = virtualFile != null ? PsiManager.getInstance(project).findFile(virtualFile) : null;
  final Document document = psiFile != null ? PsiDocumentManager.getInstance(project).getDocument(psiFile) : null;
  final EditorHighlighter cachedEditorHighlighter;
  boolean alreadyInitializedHighlighter = false;

  if (document != null &&
      (cachedEditorHighlighter = getEditorHighlighterForCachesBuilding(document)) != null &&
      PlatformIdTableBuilding.checkCanUseCachedEditorHighlighter(text, cachedEditorHighlighter)) {
    highlighter = cachedEditorHighlighter;
    alreadyInitializedHighlighter = true;
  }
  else if (virtualFile != null) {
    highlighter = EditorHighlighterFactory.getInstance().createEditorHighlighter(project, virtualFile);
  }

  if (highlighter != null) {
    return new LexerEditorHighlighterLexer(highlighter, alreadyInitializedHighlighter);
  }
  return null;
}
 
Example 9
Source Project: consulo   Source File: PsiBuilderFactory.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
public abstract PsiBuilder createBuilder(@Nonnull Project project,
                                         @Nonnull LighterLazyParseableNode chameleon,
                                         @Nullable Lexer lexer,
                                         @Nonnull Language lang,
                                         @Nonnull LanguageVersion languageVersion,
                                         @Nonnull CharSequence seq);
 
Example 10
Source Project: consulo   Source File: PsiBuilderFactoryImpl.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public PsiBuilder createBuilder(@Nonnull final ParserDefinition parserDefinition,
                                @Nonnull final Lexer lexer,
                                @Nonnull LanguageVersion languageVersion,
                                @Nonnull final CharSequence seq) {
  return new PsiBuilderImpl(null, null, parserDefinition, lexer, languageVersion, null, seq, null, null);
}
 
Example 11
Source Project: intellij-latte   Source File: LatteHtmlHighlightingLexer.java    License: MIT License 5 votes vote down vote up
@Override
protected void lookAhead(Lexer baseLexer) {
	IElementType currentToken = baseLexer.getTokenType();

	if (currentToken != LatteTypes.T_TEXT && LatteHtmlUtil.HTML_TOKENS.contains(currentToken)) {
		advanceLexer(baseLexer);
		replaceCachedType(0, LatteTypes.T_TEXT);

	} else {
		super.lookAhead(baseLexer);
	}
}
 
Example 12
Source Project: intellij-latte   Source File: LattePhpLexerTest.java    License: MIT License 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testArrayItemsWithDoubleArrow() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{block test, te => $item, test . 1 => 123}{/block}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "te"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS, "=>"), //todo: weird behavior, it must be T_PHP_DOUBLE_ARROW (it is because is workaround for "=>" in LatteParser.bnf)
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$item"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_CONCATENATION, "."),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "1"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DOUBLE_ARROW, "=>"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 
Example 13
Source Project: intellij-latte   Source File: Assert.java    License: MIT License 5 votes vote down vote up
/**
 * Checks that given lexer returns the correct tokens.
 *
 * @param lexer
 * @param expectedTokens
 */
public static void assertTokens(Lexer lexer, Pair<IElementType, String>[] expectedTokens) {
	int i;
	for (i = 0; lexer.getTokenType() != null; i++) {
		if (i == expectedTokens.length) fail("Too many tokens from lexer; unexpected " + lexer.getTokenType());
		assertEquals("Wrong token type at index " + i, expectedTokens[i].first, lexer.getTokenType());
		assertEquals("Wrong token text at index " + i, expectedTokens[i].second, lexer.getTokenText());
		lexer.advance();
	}
	if (i < expectedTokens.length) fail("Not enough tokens from lexer; expected " + expectedTokens[i].first + " but got nothing.");
}
 
Example 14
Source Project: consulo   Source File: MacroParser.java    License: Apache License 2.0 5 votes vote down vote up
private static void parseParameters(MacroCallNode macroCallNode, Lexer lexer, String expression) {
  if (lexer.getTokenType() != MacroTokenType.RPAREN) {
    while (lexer.getTokenType() != null) {
      Expression node = parseMacro(lexer, expression);
      macroCallNode.addParameter(node);

      if (lexer.getTokenType() == MacroTokenType.COMMA) {
        advance(lexer);
      }
      else {
        break;
      }
    }
  }
}
 
Example 15
Source Project: intellij-latte   Source File: LattePhpLexerTest.java    License: MIT License 5 votes vote down vote up
@Test
@SuppressWarnings("unchecked")
public void testMethod() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{$object->getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->getFoo()->getBar($var)}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}
 
Example 16
Source Project: consulo   Source File: PsiBuilderImpl.java    License: Apache License 2.0 5 votes vote down vote up
private PsiBuilderImpl(@Nullable Project project,
                       @Nullable PsiFile containingFile,
                       @Nonnull LanguageVersion languageVersion,
                       @Nonnull ParserDefinition parserDefinition,
                       @Nonnull Lexer lexer,
                       @Nullable CharTable charTable,
                       @Nonnull CharSequence text,
                       @Nullable ASTNode originalTree,
                       @Nullable CharSequence lastCommittedText,
                       @Nullable MyTreeStructure parentLightTree,
                       @Nullable Object parentCachingNode) {
  myProject = project;
  myFile = containingFile;
  myLanguageVersion = languageVersion;
  myText = text;
  myTextArray = CharArrayUtil.fromSequenceWithoutCopying(text);
  myLexer = lexer;

  myWhitespaces = parserDefinition.getWhitespaceTokens(languageVersion);
  myComments = parserDefinition.getCommentTokens(languageVersion);
  myCharTable = charTable;
  myOriginalTree = originalTree;
  myLastCommittedText = lastCommittedText;
  if ((originalTree == null) != (lastCommittedText == null)) {
    throw new IllegalArgumentException("originalTree and lastCommittedText must be null/notnull together but got: originalTree=" +
                                       originalTree +
                                       "; lastCommittedText=" +
                                       (lastCommittedText == null ? null : "'" + StringUtil.first(lastCommittedText, 80, true) + "'"));
  }
  myParentLightTree = parentLightTree;
  myOffset = parentCachingNode instanceof LazyParseableToken ? ((LazyParseableToken)parentCachingNode).getStartOffset() : 0;

  cacheLexemes(parentCachingNode);
}
 
Example 17
@Nullable
public static String isReservedKeyword(@NotNull String string) {
  Lexer lexer = JAVA_LEXER;
  lexer.start(string);
  if (lexer.getTokenType() != JavaTokenType.IDENTIFIER) {
    if (lexer.getTokenType() instanceof IKeywordElementType) {
      return "Package names cannot contain Java keywords like '" + string + "'";
    }
    if (string.isEmpty()) {
      return "Package segments must be of non-zero length";
    }
    return string + " is not a valid identifier";
  }
  return null;
}
 
Example 18
Source Project: consulo   Source File: MacroParser.java    License: Apache License 2.0 5 votes vote down vote up
@Nonnull
public static Expression parse(@Nullable String expression) {
  if (StringUtil.isEmpty(expression)) {
    return new ConstantNode("");
  }
  Lexer lexer = new MacroLexer();
  lexer.start(expression);
  skipWhitespaces(lexer);
  return parseMacro(lexer, expression);
}
 
Example 19
Source Project: BashSupport   Source File: UnescapingPsiBuilder.java    License: Apache License 2.0 5 votes vote down vote up
public UnescapingPsiBuilder(@NotNull final Project project,
                            @NotNull final ParserDefinition parserDefinition,
                            @NotNull final Lexer lexer,
                            @NotNull final ASTNode chameleon,
                            @NotNull final CharSequence originalText,
                            @NotNull final CharSequence processedText,
                            @NotNull final TextPreprocessor textProcessor) {
    this(new PsiBuilderImpl(project, parserDefinition, lexer, chameleon, originalText), textProcessor, processedText);
}
 
Example 20
Source Project: intellij-jsonnet   Source File: JsonnetSyntaxHighlighter.java    License: Apache License 2.0 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new JsonnetLexerAdapter();
}
 
Example 21
Source Project: intellij-jsonnet   Source File: JsonnetParserDefinition.java    License: Apache License 2.0 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new JsonnetLexerAdapter();
}
 
Example 22
Source Project: JetBrains-NASM-Language   Source File: NASMParserDefinition.java    License: MIT License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new NASMLexer();
}
 
Example 23
Source Project: JetBrains-NASM-Language   Source File: NASMSyntaxHighlighter.java    License: MIT License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new NASMLexer();
}
 
Example 24
@NotNull
@Override
public Lexer createLexer(Project project) {
	ANTLRv4Lexer lexer = new ANTLRv4Lexer(null);
	return new ANTLRv4LexerAdaptor(lexer);
}
 
Example 25
Source Project: intellij-thrift   Source File: ThriftSyntaxHighlighter.java    License: Apache License 2.0 4 votes vote down vote up
@NotNull
public Lexer getHighlightingLexer() {
  return new ThriftLexer();
}
 
Example 26
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new HaskellSyntaxHighlightingLexer();
}
 
Example 27
Source Project: intellij-xquery   Source File: XQueryParserDefinition.java    License: Apache License 2.0 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new XQueryLexer();
}
 
Example 28
Source Project: reasonml-idea-plugin   Source File: OclParserDefinition.java    License: MIT License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new OclLexer();
}
 
Example 29
Source Project: reasonml-idea-plugin   Source File: DuneParserDefinition.java    License: MIT License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new FlexAdapter(new DuneLexer(DuneTypes.INSTANCE));
}
 
Example 30
@Nonnull
@Override
public Lexer getHighlightingLexer()
{
	return new CSharpPreprocessorHightlightLexer();
}