Java Code Examples for com.intellij.lexer.Lexer#start()

The following examples show how to use com.intellij.lexer.Lexer#start() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.

Example 1

Source File: SelectWordUtil.java From consulo with Apache License 2.0

6 votes

public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}

Example 2

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testProperty() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object->foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->foo->bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "foo"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_IDENTIFIER, "bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 3

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{if $var}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 4

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

6 votes

@Test
@SuppressWarnings("unchecked")
public void testVarDefinition() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{var $foo = 123}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "var"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DEFINITION_OPERATOR, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 5

Source File: LexerTestCase.java From consulo with Apache License 2.0

6 votes

public static String printTokens(CharSequence text, int start, Lexer lexer) {
  lexer.start(text, start, text.length());
  String result = "";
  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      break;
    }
    String tokenText = getTokenText(lexer);
    String tokenTypeName = tokenType.toString();
    String line = tokenTypeName + " ('" + tokenText + "')\n";
    result += line;
    lexer.advance();
  }
  return result;
}

Example 6

Source File: AndroidUtils.java From flutter-intellij with BSD 3-Clause "New" or "Revised" License

5 votes

@Nullable
public static String isReservedKeyword(@NotNull String string) {
  Lexer lexer = JAVA_LEXER;
  lexer.start(string);
  if (lexer.getTokenType() != JavaTokenType.IDENTIFIER) {
    if (lexer.getTokenType() instanceof IKeywordElementType) {
      return "Package names cannot contain Java keywords like '" + string + "'";
    }
    if (string.isEmpty()) {
      return "Package segments must be of non-zero length";
    }
    return string + " is not a valid identifier";
  }
  return null;
}

Example 7

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testArrayItemsWithDoubleArrow() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{block test, te => $item, test . 1 => 123}{/block}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "te"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS, "=>"), //todo: weird behavior, it must be T_PHP_DOUBLE_ARROW (it is because is workaround for "=>" in LatteParser.bnf)
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$item"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "test"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_CONCATENATION, "."),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "1"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_DOUBLE_ARROW, "=>"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "block"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 8

Source File: LatteTopLexerAdapterTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testHtml() {
	Lexer lexer = new LatteTopLexerAdapter();

	lexer.start("");
	assertTokens(lexer, new Pair[] {});

	lexer.start("abc<strong>def</strong>ghi");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "strong"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
			Pair.create(T_TEXT, "def"),
			Pair.create(T_HTML_CLOSE_TAG_OPEN, "</"),
			Pair.create(T_TEXT, "strong"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
			Pair.create(T_TEXT, "ghi"),
	});

	lexer.start("abc<img src=\"\" />ghi");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "img src=\"\" "),
			Pair.create(T_HTML_OPEN_TAG_CLOSE, "/>"),
			Pair.create(T_TEXT, "ghi"),
	});

	lexer.start("abc<style></style>");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_TEXT, "abc"),
			Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
			Pair.create(T_TEXT, "style></style"),
			Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}

Example 9

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testStaticProperty() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object::$foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{\\Foo\\Bar::$foo::$bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{Foo\\Bar::$foo}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$foo"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 10

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testMethod() throws Exception {
	Lexer lexer = new LatteHighlightingLexer(new LatteLexer());
	lexer.start("{$object->getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{$object->getFoo()->getBar($var)}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_OBJECT_OPERATOR, "->"),
		Pair.create(T_PHP_METHOD, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 11

Source File: LatteLexerTest.java From intellij-latte with MIT License

5 votes

@Test
@SuppressWarnings("unchecked")
public void testBasic() throws Exception {
	Lexer lexer = new LatteLexer();

	lexer.start("<e>{if XYZ}B{/if}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "e"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "XYZ"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{if XYZ}B{/if}<e>");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_IDENTIFIER, "XYZ"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_TEXT, "B"),
		Pair.create(T_MACRO_CLOSE_TAG_OPEN, "{/"),
		Pair.create(T_MACRO_NAME, "if"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "e"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}

Example 12

Source File: LanguageUtil.java From consulo with Apache License 2.0

5 votes

@Nonnull
@RequiredReadAction
public static ParserDefinition.SpaceRequirements canStickTokensTogetherByLexer(ASTNode left, ASTNode right, Lexer lexer) {
  String textStr = left.getText() + right.getText();

  lexer.start(textStr, 0, textStr.length());
  if(lexer.getTokenType() != left.getElementType()) return ParserDefinition.SpaceRequirements.MUST;
  if(lexer.getTokenEnd() != left.getTextLength()) return ParserDefinition.SpaceRequirements.MUST;
  lexer.advance();
  if(lexer.getTokenEnd() != textStr.length()) return ParserDefinition.SpaceRequirements.MUST;
  if(lexer.getTokenType() != right.getElementType()) return ParserDefinition.SpaceRequirements.MUST;
  return ParserDefinition.SpaceRequirements.MAY;
}

Example 13

Source File: MacroParser.java From consulo with Apache License 2.0

5 votes

@Nonnull
public static Expression parse(@Nullable String expression) {
  if (StringUtil.isEmpty(expression)) {
    return new ConstantNode("");
  }
  Lexer lexer = new MacroLexer();
  lexer.start(expression);
  skipWhitespaces(lexer);
  return parseMacro(lexer, expression);
}

Example 14

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testVarType() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{varType \\Foo\\Bar|null $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NULL, "null"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Permissions\\Permission[]|string|null $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Permissions"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Permission"),
		Pair.create(T_PHP_LEFT_BRACKET, "["),
		Pair.create(T_PHP_RIGHT_BRACKET, "]"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NULL, "null"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Foo\\Bar\\_TestClass1|Foo\\Bar|string|int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "_TestClass1"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType Foo\\Bar\\_TestClass1|\\Foo\\Bar|string|int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "_TestClass1"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_PHP_TYPE, "int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{varType string|$int $_var1}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "varType"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_TYPE, "string"),
		Pair.create(T_PHP_OR_INCLUSIVE, "|"),
		Pair.create(T_MACRO_ARGS_VAR, "$int"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_VAR, "$_var1"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 15

Source File: HaxeProjectStructureDetector.java From intellij-haxe with Apache License 2.0

4 votes

public String fun(final CharSequence charSequence) {
  Lexer lexer = LanguageParserDefinitions.INSTANCE.forLanguage(HaxeLanguage.INSTANCE).createLexer(null);
  lexer.start(charSequence);
  return readPackageName(charSequence, lexer);
}

Example 16

Source File: EnterHandler.java From consulo with Apache License 2.0

4 votes

public static boolean isCommentComplete(PsiComment comment, CodeDocumentationAwareCommenter commenter, Editor editor) {
  for (CommentCompleteHandler handler : CommentCompleteHandler.EP_NAME.getExtensionList()) {
    if (handler.isApplicable(comment, commenter)) {
      return handler.isCommentComplete(comment, commenter, editor);
    }
  }

  String commentText = comment.getText();
  final boolean docComment = isDocComment(comment, commenter);
  final String expectedCommentEnd = docComment ? commenter.getDocumentationCommentSuffix() : commenter.getBlockCommentSuffix();
  if (!commentText.endsWith(expectedCommentEnd)) return false;

  final PsiFile containingFile = comment.getContainingFile();
  final Language language = containingFile.getLanguage();
  ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
  if (parserDefinition == null) {
    return true;
  }
  Lexer lexer = parserDefinition.createLexer(containingFile.getLanguageVersion());
  final String commentPrefix = docComment ? commenter.getDocumentationCommentPrefix() : commenter.getBlockCommentPrefix();
  lexer.start(commentText, commentPrefix == null ? 0 : commentPrefix.length(), commentText.length());
  QuoteHandler fileTypeHandler = TypedHandler.getQuoteHandler(containingFile, editor);
  JavaLikeQuoteHandler javaLikeQuoteHandler = fileTypeHandler instanceof JavaLikeQuoteHandler ? (JavaLikeQuoteHandler)fileTypeHandler : null;

  while (true) {
    IElementType tokenType = lexer.getTokenType();
    if (tokenType == null) {
      return false;
    }

    if (javaLikeQuoteHandler != null && javaLikeQuoteHandler.getStringTokenTypes() != null && javaLikeQuoteHandler.getStringTokenTypes().contains(tokenType)) {
      String text = commentText.substring(lexer.getTokenStart(), lexer.getTokenEnd());
      int endOffset = comment.getTextRange().getEndOffset();

      if (text.endsWith(expectedCommentEnd) && endOffset < containingFile.getTextLength() && containingFile.getText().charAt(endOffset) == '\n') {
        return true;
      }
    }
    if (tokenType == commenter.getDocumentationCommentTokenType() || tokenType == commenter.getBlockCommentTokenType()) {
      return false;
    }
    if (tokenType == commenter.getLineCommentTokenType() && lexer.getTokenText().contains(commentPrefix)) {
      return false;
    }
    if (lexer.getTokenEnd() == commentText.length()) {
      if (tokenType == commenter.getLineCommentTokenType()) {
        String prefix = commenter.getLineCommentPrefix();
        lexer.start(commentText, lexer.getTokenStart() + (prefix == null ? 0 : prefix.length()), commentText.length());
        lexer.advance();
        continue;
      }
      else if (isInvalidPsi(comment)) {
        return false;
      }
      return true;
    }
    lexer.advance();
  }
}

Example 17

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testStaticMethod() throws Exception {
	Lexer lexer = new LatteLexer();
	lexer.start("{$object::getFoo()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_ARGS_VAR, "$object"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{\\Foo\\Bar::getFoo($var, 'test', 123)::getBar()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_MACRO_ARGS_VAR, "$var"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_SINGLE_QUOTE_LEFT, "'"),
		Pair.create(T_MACRO_ARGS_STRING, "test"),
		Pair.create(T_PHP_SINGLE_QUOTE_RIGHT, "'"),
		Pair.create(T_MACRO_ARGS, ","),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_MACRO_ARGS_NUMBER, "123"),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getBar"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{Foo\\Bar::getFoo()::$bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_IDENTIFIER, "Bar"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_PHP_IDENTIFIER, "getFoo"),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_PHP_DOUBLE_COLON, "::"),
		Pair.create(T_MACRO_ARGS_VAR, "$bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 18

Source File: LatteHighlightingLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testMacroLexer() {
	Lexer lexer = new LatteHighlightingLexer(new LatteLookAheadLexer(new LatteLexer()));

	lexer.start("{include #blockName}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "include"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_BLOCK_NAME, "#"),
		Pair.create(T_BLOCK_NAME, "blockName"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= dump ()}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_METHOD, "dump"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_LEFT_NORMAL_BRACE, "("),
		Pair.create(T_PHP_RIGHT_NORMAL_BRACE, ")"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Foo\\Bar}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_SHORTNAME, "="),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Foo"),
		Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
		Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{= \\Bar}");
	assertTokens(lexer, new Pair[] {
			Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
			Pair.create(T_MACRO_SHORTNAME, "="),
			Pair.create(T_WHITESPACE, " "),
			Pair.create(T_PHP_NAMESPACE_RESOLUTION, "\\"),
			Pair.create(T_PHP_NAMESPACE_REFERENCE, "Bar"),
			Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});
}

Example 19

Source File: UnescapingPsiBuilder.java From BashSupport with Apache License 2.0

4 votes

private void initTokenListAndCharSequence(Lexer lexer) {
    lexer.start(processedText);

    myShrunkSequence = new ArrayList<MyShiftedToken>(512); //assume a larger token size by default
    StringBuilder charSequenceBuilder = new StringBuilder();

    int realPos = 0;
    int shrunkPos = 0;
    while (lexer.getTokenType() != null) {
        final IElementType tokenType = lexer.getTokenType();
        final String tokenText = lexer.getTokenText();

        int tokenStart = lexer.getTokenStart();
        int tokenEnd = lexer.getTokenEnd();
        int realLength = tokenEnd - tokenStart;

        int delta = textProcessor.getContentRange().getStartOffset();
        int originalStart = textProcessor.getOffsetInHost(tokenStart - delta);
        int originalEnd = textProcessor.getOffsetInHost(tokenEnd - delta);

        if (textProcessor.containsRange(tokenStart, tokenEnd) && originalStart != -1 && originalEnd != -1) {
            realLength = originalEnd - originalStart;
            int masqueLength = tokenEnd - tokenStart;

            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + masqueLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += masqueLength;
        } else {
            myShrunkSequence.add(new MyShiftedToken(tokenType,
                    realPos, realPos + realLength,
                    shrunkPos, shrunkPos + realLength, tokenText));
            charSequenceBuilder.append(tokenText);

            shrunkPos += realLength;
        }

        realPos += realLength;

        lexer.advance();
    }

    myShrunkCharSequence = charSequenceBuilder.toString();
    myShrunkSequenceSize = myShrunkSequence.size();
}

Example 20

Source File: LattePhpLexerTest.java From intellij-latte with MIT License

4 votes

@Test
@SuppressWarnings("unchecked")
public void testLinkDestination() throws Exception {
	Lexer lexer = new LatteLookAheadLexer(new LatteLexer());
	lexer.start("{link default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{link Presenter:default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, "Presenter"),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("{link :Presenter:default}");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_MACRO_OPEN_TAG_OPEN, "{"),
		Pair.create(T_MACRO_NAME, "link"),
		Pair.create(T_WHITESPACE, " "),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "Presenter"),
		Pair.create(T_LINK_DESTINATION, ":"),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_MACRO_TAG_CLOSE, "}"),
	});

	lexer.start("<a n:href=\"default\"></a>");
	assertTokens(lexer, new Pair[] {
		Pair.create(T_HTML_OPEN_TAG_OPEN, "<"),
		Pair.create(T_TEXT, "a "),
		Pair.create(T_HTML_TAG_NATTR_NAME, "n:href"),
		Pair.create(T_HTML_TAG_ATTR_EQUAL_SIGN, "="),
		Pair.create(T_HTML_TAG_ATTR_DQ, "\""),
		Pair.create(T_LINK_DESTINATION, "default"),
		Pair.create(T_HTML_TAG_ATTR_DQ, "\""),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
		Pair.create(T_HTML_CLOSE_TAG_OPEN, "</"),
		Pair.create(T_TEXT, "a"),
		Pair.create(T_HTML_TAG_CLOSE, ">"),
	});
}