Java Code Examples for com.intellij.lexer.Lexer

The following are top voted examples for showing how to use com.intellij.lexer.Lexer. These examples are extracted from open source projects. You can vote up the examples you like and your votes will be used in our system to generate more good examples.
Example 1
Project: bamboo-soy   File: SoyLexer.java   Source Code and License 6 votes vote down vote up
@Override
public MergeFunction getMergeFunction() {
  return ((final IElementType type, final Lexer originalLexer) -> {
    if (type == SoyTypes.OTHER || type == TokenType.WHITE_SPACE) {
      IElementType returnType =  type;
      while (originalLexer.getTokenType() == SoyTypes.OTHER
          || originalLexer.getTokenType() == TokenType.WHITE_SPACE) {
        if (originalLexer.getTokenType() == SoyTypes.OTHER) {
          returnType = SoyTypes.OTHER;
        }
        originalLexer.advance();
      }
      return returnType;
    }

    return type;
  });
}
 
Example 2
Project: intellij-ce-playground   File: SelectWordUtil.java   Source Code and License 6 votes vote down vote up
public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}
 
Example 3
Project: intellij-ce-playground   File: LexerTestCase.java   Source Code and License 6 votes vote down vote up
protected void checkCorrectRestart(String text) {
  Lexer mainLexer = createLexer();
  String allTokens = printTokens(text, 0, mainLexer);

  Lexer auxLexer = createLexer();
  auxLexer.start(text);
  while (true) {
    IElementType type = auxLexer.getTokenType();
    if (type == null) {
      break;
    }
    if (auxLexer.getState() == 0) {
      int tokenStart = auxLexer.getTokenStart();
      String subTokens = printTokens(text, tokenStart, mainLexer);
      if (!allTokens.endsWith(subTokens)) {
        assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens);
      }
    }
    auxLexer.advance();
  }
}
 
Example 4
Project: lua-for-idea   File: LuaParsingLexerMergingAdapter.java   Source Code and License 6 votes vote down vote up
@Override
public MergeFunction getMergeFunction() {
    return new MergeFunction() {

        @Override
        public IElementType merge(IElementType type, Lexer originalLexer) {
            if (!allMergables.contains(type)) {
                return type;
            }

            TokenSet merging = tokensToMerge.contains(type) ? tokensToMerge : tokensToMerge2;

            while (true) {
                final IElementType tokenType = originalLexer.getTokenType();
                if (!merging.contains(tokenType)) break;
                originalLexer.advance();
            }

            return merging == tokensToMerge ? LONGSTRING : LONGCOMMENT;
        }
    };
}
 
Example 5
Project: intellij   File: BuildParserTest.java   Source Code and License 6 votes vote down vote up
private ASTNode createAST(String text) {
  ParserDefinition definition = new BuildParserDefinition();
  PsiParser parser = definition.createParser(getProject());
  Lexer lexer = definition.createLexer(getProject());
  PsiBuilderImpl psiBuilder =
      new PsiBuilderImpl(
          getProject(), null, definition, lexer, new CharTableImpl(), text, null, null);
  PsiBuilderAdapter adapter =
      new PsiBuilderAdapter(psiBuilder) {
        @Override
        public void error(String messageText) {
          super.error(messageText);
          errors.add(messageText);
        }
      };
  return parser.parse(definition.getFileNodeType(), adapter);
}
 
Example 6
Project: intellij-ce-playground   File: FunctionDeclarationParsing.java   Source Code and License 6 votes vote down vote up
@Nullable
public static String parseQName(Lexer lexer) {
  String name;
  if (lexer.getTokenType() == XPathTokenTypes.NCNAME) {
    name = lexer.getTokenText();
    lexer.advance();
    if (lexer.getTokenType() == XPathTokenTypes.COL) {
      lexer.advance();
      assert lexer.getTokenType() == XPathTokenTypes.NCNAME;
      name += (":" + lexer.getTokenText());
      lexer.advance();
    }
  } else {
    name = null;
  }
  return name;
}
 
Example 7
Project: RtplPlugin   File: BaseHtmlLexer.java   Source Code and License 6 votes vote down vote up
protected void registerHandler(IElementType elementType, TokenHandler value) {
  final TokenHandler tokenHandler = tokenHandlers.get(elementType);

  if (tokenHandler != null) {
    final TokenHandler newHandler = value;
    value = new TokenHandler() {
      @Override
      public void handleElement(final Lexer lexer) {
        tokenHandler.handleElement(lexer);
        newHandler.handleElement(lexer);
      }
    };
  }

  tokenHandlers.put(elementType,value);
}
 
Example 8
Project: intellij-ce-playground   File: PropertiesLexerTest.java   Source Code and License 6 votes vote down vote up
private static void doTest(String text, String[] expectedTokens,Lexer lexer) {
  lexer.start(text);
  int idx = 0;
  while (lexer.getTokenType() != null) {
    if (idx >= expectedTokens.length) fail("Too many tokens");
    String tokenName = lexer.getTokenType().toString();
    String expectedTokenType = expectedTokens[idx++];
    String expectedTokenText = expectedTokens[idx++];
    assertEquals(expectedTokenType, tokenName);
    String tokenText = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
    assertEquals(expectedTokenText, tokenText);
    lexer.advance();
  }

  if (idx < expectedTokens.length) fail("Not enough tokens");
}
 
Example 9
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testError2() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "ab");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    // Simplified lexer ignores this error
    //assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 
Example 10
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 6 votes vote down vote up
@Test
public void testError3() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "@bad!");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 
Example 11
Project: intellij-ce-playground   File: JavaElementType.java   Source Code and License 6 votes vote down vote up
@Override
public int getErrorsCount(final CharSequence seq, Language fileLanguage, final Project project) {
  Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);

  lexer.start(seq);
  if (lexer.getTokenType() != JavaTokenType.LBRACE) return IErrorCounterReparseableElementType.FATAL_ERROR;
  lexer.advance();
  int balance = 1;
  while (true) {
    IElementType type = lexer.getTokenType();
    if (type == null) break;
    if (balance == 0) return IErrorCounterReparseableElementType.FATAL_ERROR;
    if (type == JavaTokenType.LBRACE) {
      balance++;
    }
    else if (type == JavaTokenType.RBRACE) {
      balance--;
    }
    lexer.advance();
  }
  return balance;
}
 
Example 12
Project: intellij-ce-playground   File: JavaSpacePropertyProcessor.java   Source Code and License 6 votes vote down vote up
private static boolean canStickJavaTokens(ASTNode token1, ASTNode token2) {
  IElementType type1 = token1.getElementType();
  IElementType type2 = token2.getElementType();

  Pair<IElementType, IElementType> pair = Pair.create(type1, type2);
  Boolean res = myCanStickJavaTokensMatrix.get(pair);
  if (res == null) {
    if (!checkToken(token1) || !checkToken(token2)) return true;
    String text = token1.getText() + token2.getText();
    Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    lexer.start(text);
    boolean canMerge = lexer.getTokenType() == type1;
    lexer.advance();
    canMerge &= lexer.getTokenType() == type2;
    res = canMerge;
    myCanStickJavaTokensMatrix.put(pair, res);
  }
  return res.booleanValue();
}
 
Example 13
Project: intellij-csv-validator   File: CsvStructureViewElement.java   Source Code and License 5 votes vote down vote up
protected PsiElement createEmptyPsiField() {
    final String text = "<undefined>";
    final IElementType type = CsvTypes.FIELD;
    final Project project = this.element.getProject();
    final PsiManager psiManager = PsiManager.getInstance(project);
    final DummyHolder dummyHolder = DummyHolderFactory.createHolder(psiManager, null);
    final FileElement fileElement = dummyHolder.getTreeElement();
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(CsvLanguage.INSTANCE);
    final Lexer lexer = parserDefinition.createLexer(project);
    final PsiBuilder psiBuilder = PsiBuilderFactory.getInstance().createBuilder(project, fileElement, lexer, CsvLanguage.INSTANCE, text);
    final ASTNode node = parserDefinition.createParser(project).parse(type, psiBuilder);
    fileElement.rawAddChildren((com.intellij.psi.impl.source.tree.TreeElement)node);
    return node.getPsi();
}
 
Example 14
Project: intellij-ce-playground   File: XmlFileHighlighter.java   Source Code and License 5 votes vote down vote up
@Override
@NotNull
public Lexer getHighlightingLexer() {
  if (myIsDtd) {
    return new DtdLexer(true);
  } else if (myIsXHtml) {
    return new XHtmlHighlightingLexer();
  } else {
    return new XmlHighlightingLexer();
  }
}
 
Example 15
Project: intellij-ce-playground   File: XPathLexer.java   Source Code and License 5 votes vote down vote up
private boolean replaceTokenNotFollowedBy(Lexer baseLexer, final IElementType replacement, IElementType nextToken) {
  if (!lookingAt(baseLexer, nextToken)) {
    treatAs(baseLexer, replacement);
    return true;
  }
  return false;
}
 
Example 16
Project: intellij-ce-playground   File: GroovyIndexPatternBuilder.java   Source Code and License 5 votes vote down vote up
@Override
public Lexer getIndexingLexer(@NotNull PsiFile file) {
    if (file instanceof GroovyFile) {
        return new GroovyLexer();
    }
    return null;
}
 
Example 17
Project: intellij-ce-playground   File: XPathLexer.java   Source Code and License 5 votes vote down vote up
private static boolean lookingAt(Lexer baseLexer, IElementType... tokens) {
  final LexerPosition position = baseLexer.getCurrentPosition();
  try {
    for (IElementType token : tokens) {
      baseLexer.advance();
      skipWhitespaceAnComments(baseLexer);
      if (baseLexer.getTokenType() != token) {
        return false;
      }
    }
    return true;
  } finally {
    baseLexer.restore(position);
  }
}
 
Example 18
Project: RtplPlugin   File: BaseHtmlLexer.java   Source Code and License 5 votes vote down vote up
@Override
public void handleElement(Lexer lexer) {
  if (seenAttribute) {
    seenStyle = false;
    seenScript = false;
    seenAttribute = false;
  }
  seenContentType = false;
  seenStylesheetType = false;
}
 
Example 19
Project: RtplPlugin   File: BaseHtmlLexer.java   Source Code and License 5 votes vote down vote up
@Override
public void handleElement(Lexer lexer) {
  if (seenContentType && seenScript && !seenAttribute) {
    @NonNls String mimeType = TreeUtil.getTokenText(lexer);
    scriptType = caseInsensitive ? mimeType.toLowerCase(Locale.US) : mimeType;
  }
  if (seenStylesheetType && seenStyle && !seenAttribute) {
    @NonNls String rel = TreeUtil.getTokenText(lexer).trim();
    styleType = caseInsensitive ? rel.toLowerCase(Locale.US) : rel;
  }
}
 
Example 20
Project: RtplPlugin   File: BaseHtmlLexer.java   Source Code and License 5 votes vote down vote up
@Override
public void handleElement(Lexer lexer) {
  seenStyle=false;
  seenScript=false;
  seenAttribute=false;
  seenContentType=false;
  seenStylesheetType=false;
  scriptType = null;
  styleType = null;
}
 
Example 21
Project: intellij-ce-playground   File: PyLexerTestCase.java   Source Code and License 5 votes vote down vote up
public static void doLexerTest(String text,
                               Lexer lexer,
                               boolean checkTokenText,
                               String... expectedTokens) {
  lexer.start(text);
  int idx = 0;
  int tokenPos = 0;
  while (lexer.getTokenType() != null) {
    if (idx >= expectedTokens.length) {
      StringBuilder remainingTokens = new StringBuilder("\"" + lexer.getTokenType().toString() + "\"");
      lexer.advance();
      while (lexer.getTokenType() != null) {
        remainingTokens.append(",");
        remainingTokens.append(" \"").append(checkTokenText ? lexer.getTokenText() : lexer.getTokenType().toString()).append("\"");
        lexer.advance();
      }
      fail("Too many tokens. Following tokens: " + remainingTokens.toString());
    }
    assertEquals("Token offset mismatch at position " + idx, tokenPos, lexer.getTokenStart());
    String tokenName = checkTokenText ? lexer.getTokenText() : lexer.getTokenType().toString();
    assertEquals("Token mismatch at position " + idx, expectedTokens[idx], tokenName);
    idx++;
    tokenPos = lexer.getTokenEnd();
    lexer.advance();
  }

  if (idx < expectedTokens.length) fail("Not enough tokens");
}
 
Example 22
Project: intellij-ce-playground   File: GroovySourceRootDetector.java   Source Code and License 5 votes vote down vote up
@Nullable
public static String getPackageName(CharSequence text) {
  Lexer lexer = new GroovyLexer();
  lexer.start(text);
  skipWhitespacesAndComments(lexer);
  final IElementType firstToken = lexer.getTokenType();
  if (firstToken != GroovyTokenTypes.kPACKAGE) {
    return "";
  }
  lexer.advance();
  skipWhitespacesAndComments(lexer);

  final StringBuilder buffer = StringBuilderSpinAllocator.alloc();
  try {
    while(true){
      if (lexer.getTokenType() != GroovyTokenTypes.mIDENT) break;
      buffer.append(text, lexer.getTokenStart(), lexer.getTokenEnd());
      lexer.advance();
      skipWhitespacesAndComments(lexer);
      if (lexer.getTokenType() != GroovyTokenTypes.mDOT) break;
      buffer.append('.');
      lexer.advance();
      skipWhitespacesAndComments(lexer);
    }
    String packageName = buffer.toString();
    if (packageName.isEmpty() || StringUtil.endsWithChar(packageName, '.')) return null;
    return packageName;
  }
  finally {
    StringBuilderSpinAllocator.dispose(buffer);
  }
}
 
Example 23
Project: intellij-ce-playground   File: PyTokenSeparatorGenerator.java   Source Code and License 5 votes vote down vote up
public ASTNode generateWhitespaceBetweenTokens(ASTNode left, ASTNode right) {
  PsiManager manager = right.getTreeParent().getPsi().getManager();
  if (left.getElementType() == PyTokenTypes.END_OF_LINE_COMMENT) {
    return createLineBreak(manager);
  }

  if (left.getPsi().isValid() && right.getPsi().isValid()) {
    final PsiElement commonParent = PsiTreeUtil.findCommonParent(left.getPsi(), right.getPsi());
    if (commonParent == null) return null;
    final PsiElement leftPrevAncestor = PsiTreeUtil.findPrevParent(commonParent, left.getPsi());
    final PsiElement rightPrevAncestor = PsiTreeUtil.findPrevParent(commonParent, right.getPsi());

    if (isStatementOrFunction(leftPrevAncestor) && isStatementOrFunction(rightPrevAncestor)) {
      int leftIndent = PyPsiUtils.getElementIndentation(leftPrevAncestor);
      int rightIndent = PyPsiUtils.getElementIndentation(rightPrevAncestor);
      int maxIndent = Math.max(leftIndent, rightIndent);
      return createWhitespace(manager, "\n" + StringUtil.repeatSymbol(' ', maxIndent));
    }
  }

  if (right.getElementType() == PyTokenTypes.DEF_KEYWORD || right.getElementType() == PyTokenTypes.CLASS_KEYWORD) {
    return createLineBreak(manager);
  }
  if (left.getElementType() == TokenType.WHITE_SPACE || right.getElementType() == TokenType.WHITE_SPACE) {
    return null;
  }
  final PyStatement leftStatement = PsiTreeUtil.getParentOfType(left.getPsi(), PyStatement.class);
  if (leftStatement != null && !PsiTreeUtil.isAncestor(leftStatement, right.getPsi(), false)) {
    return createLineBreak(manager);
  }
  final Lexer lexer = new PythonIndentingLexer();
  if (LanguageUtil.canStickTokensTogetherByLexer(left, right, lexer) == ParserDefinition.SpaceRequirements.MUST) {
    return createSpace(manager);
  }
  return null;
}
 
Example 24
Project: intellij-ce-playground   File: InplaceVariableIntroducer.java   Source Code and License 5 votes vote down vote up
public InplaceVariableIntroducer(PsiNamedElement elementToRename,
                                 Editor editor,
                                 final Project project,
                                 String title, E[] occurrences,
                                 @Nullable E expr) {
  super(editor, elementToRename, project);
  myTitle = title;
  myOccurrences = occurrences;
  if (expr != null) {
    final ASTNode node = expr.getNode();
    ASTNode prev = node.getTreePrev();
    final ASTNode astNode = LanguageTokenSeparatorGenerators.INSTANCE.forLanguage(expr.getLanguage())
      .generateWhitespaceBetweenTokens(prev, node);
    if (astNode != null) {
      final Lexer lexer = LanguageParserDefinitions.INSTANCE.forLanguage(expr.getLanguage()).createLexer(project);
      if (LanguageUtil.canStickTokensTogetherByLexer(prev, prev, lexer) == ParserDefinition.SpaceRequirements.MUST) {
        PostprocessReformattingAspect.getInstance(project).disablePostprocessFormattingInside(new Runnable() {
          @Override
          public void run() {
            new WriteCommandAction<Object>(project, "Normalize declaration") {
              @Override
              protected void run(@NotNull Result<Object> result) throws Throwable {
                node.getTreeParent().addChild(astNode, node);
              }
            }.execute(); 
          }
        });
      }
    }
    myExpr = expr;
  }
  myExprMarker = myExpr != null && myExpr.isPhysical() ? createMarker(myExpr) : null;
  initOccurrencesMarkers();
}
 
Example 25
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 5 votes vote down vote up
@Test
public void testKeys() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "key: val");
    assertAndAdvance(l, YamlTokenTypes.YAML_Key, 0, 3, "key");
    assertAndAdvance(l, YamlTokenTypes.YAML_Value, 3, 4, ":");
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 5, 8, "val");
    assertEquals(null, l.getTokenType());
}
 
Example 26
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 5 votes vote down vote up
@Test
public void testKeywords() {
    Lexer l = createLexer();

    l.start("[true,off,TruE,\"true\",12,12.3,null]");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowSequenceStart, 0, 1, "[");
    assertAndAdvance(l, YamlTokenTypes.YAML_Tag_BOOL, 1, 5, "true");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 5, 6, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Tag_BOOL, 6, 9, "off");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 9, 10, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 10, 14, "TruE");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 14, 15, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 15, 21, "\"true\"");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_INT, l.getTokenType());
    assertEquals("12", l.getTokenText());
    l.advance();

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_FLOAT, l.getTokenType());
    assertEquals("12.3", l.getTokenText());
    l.advance();

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_NULL, l.getTokenType());
    assertEquals("null", l.getTokenText());
    l.advance();

    assertEquals(YamlTokenTypes.YAML_FlowSequenceEnd, l.getTokenType());
    assertEquals("]", l.getTokenText());
    l.advance();

    //assertEquals(null, l.getTokenType());
}
 
Example 27
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 5 votes vote down vote up
public void test73() throws Exception {
    Lexer l = createLexer();

    l.start(" c ");

    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace, 0, 1, " ");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 1, 3, "c ");
}
 
Example 28
Project: intellij-snakeyaml   File: YamlHighlightingLexerTest.java   Source Code and License 5 votes vote down vote up
private void assertAndAdvance(Lexer l, IElementType expectedType, int start, int end, String text) {
    assertEquals(expectedType, l.getTokenType());
    assertEquals(start, l.getTokenStart());
    assertEquals(end, l.getTokenEnd());
    assertEquals(text, l.getTokenText());
    l.advance();
}
 
Example 29
Project: intellij-ce-playground   File: PyFileElementType.java   Source Code and License 5 votes vote down vote up
@Nullable
@Override
public ASTNode parseContents(ASTNode node) {
  final LanguageLevel languageLevel = getLanguageLevel(node.getPsi());
  if (PydevConsoleRunner.isPythonConsole(node)) {
    return parseConsoleCode(node, PydevConsoleRunner.getPythonConsoleData(node));
  }
  else {
    final PsiElement psi = node.getPsi();
    if (psi != null) {
      final Project project = psi.getProject();
      final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
      final Language language = getLanguage();
      final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
      if (parserDefinition == null) {
        return null;
      }
      final Lexer lexer = parserDefinition.createLexer(project);
      final PsiParser parser = parserDefinition.createParser(project);
      final PsiBuilder builder = factory.createBuilder(project, node, lexer, language, node.getChars());
      if (parser instanceof PyParser) {
        final PyParser pythonParser = (PyParser)parser;
        pythonParser.setLanguageLevel(languageLevel);
        if (languageLevel == LanguageLevel.PYTHON26 && psi.getContainingFile().getName().equals("__builtin__.py")) {
          pythonParser.setFutureFlag(StatementParsing.FUTURE.PRINT_FUNCTION);
        }
      }
      return parser.parse(this, builder).getFirstChildNode();
    }
    return null;
  }
}
 
Example 30
Project: intellij-ce-playground   File: MacroParser.java   Source Code and License 5 votes vote down vote up
private static void parseParameters(MacroCallNode macroCallNode, Lexer lexer, String expression) {
  if (lexer.getTokenType() != MacroTokenType.RPAREN) {
    while (lexer.getTokenType() != null) {
      Expression node = parseMacro(lexer, expression);
      macroCallNode.addParameter(node);

      if (lexer.getTokenType() == MacroTokenType.COMMA) {
        advance(lexer);
      }
      else {
        break;
      }
    }
  }
}
 
Example 31
Project: intellij-ce-playground   File: JavaSpacePropertyProcessor.java   Source Code and License 5 votes vote down vote up
private static boolean checkToken(final ASTNode token1) {
  Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
  final String text = token1.getText();
  lexer.start(text);
  if (lexer.getTokenType() != token1.getElementType()) return false;
  lexer.advance();
  return lexer.getTokenType() == null;
}
 
Example 32
Project: intellij-ce-playground   File: PlainTextSyntaxHighlighterFactory.java   Source Code and License 5 votes vote down vote up
public static Lexer createPlainTextLexer() {
  ArrayList<TokenParser> tokenParsers = new ArrayList<TokenParser>();
  tokenParsers.add(new WhitespaceParser());

  tokenParsers.addAll(BraceTokenParser.getBraces());
  tokenParsers.addAll(BraceTokenParser.getParens());
  tokenParsers.addAll(BraceTokenParser.getBrackets());
  tokenParsers.addAll(BraceTokenParser.getAngleBrackets());

  return new MergingLexerAdapter(new AbstractCustomLexer(tokenParsers), TokenSet.create(CustomHighlighterTokenType.CHARACTER));
}
 
Example 33
Project: intellij-ce-playground   File: JspIndexPatternBuilder.java   Source Code and License 5 votes vote down vote up
@Override
public Lexer getIndexingLexer(@NotNull final PsiFile file) {
  if (JspPsiUtil.isInJspFile(file)) {
    return EditorHighlighterCache.getLexerBasedOnLexerHighlighter(file.getText(), file.getVirtualFile(), file.getProject());
  }

  return null;
}
 
Example 34
Project: intellij-ce-playground   File: TemplateDataElementType.java   Source Code and License 5 votes vote down vote up
protected OuterLanguageElementImpl createOuterLanguageElement(final Lexer lexer, final CharTable table,
                                                              @NotNull IElementType outerElementType) {
  final CharSequence buffer = lexer.getBufferSequence();
  final int tokenStart = lexer.getTokenStart();
  if (tokenStart < 0 || tokenStart > buffer.length()) {
    LOG.error("Invalid start: " + tokenStart + "; " + lexer);
  }
  final int tokenEnd = lexer.getTokenEnd();
  if (tokenEnd < 0 || tokenEnd > buffer.length()) {
    LOG.error("Invalid end: " + tokenEnd + "; " + lexer);
  }

  return new OuterLanguageElementImpl(outerElementType, table.intern(buffer, tokenStart, tokenEnd));
}
 
Example 35
Project: intellij-ce-playground   File: Html5CustomAttributesIndex.java   Source Code and License 5 votes vote down vote up
@Override
@NotNull
public Map<String, Void> map(@NotNull FileContent inputData) {
  CharSequence input = inputData.getContentAsText();
  Language language = ((LanguageFileType)inputData.getFileType()).getLanguage();
  if (language == HTMLLanguage.INSTANCE || language == XHTMLLanguage.INSTANCE) {
    final Lexer lexer = (language == HTMLLanguage.INSTANCE ? new HtmlHighlightingLexer(FileTypeManager.getInstance().getStdFileType("CSS")) : new XHtmlHighlightingLexer());
    lexer.start(input);
    Map<String, Void> result = new HashMap<String, Void>();
    IElementType tokenType = lexer.getTokenType();
    while (tokenType != null) {
      if (tokenType == XmlTokenType.XML_NAME) {
        String xmlName = input.subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
        if (HtmlUtil.isCustomHtml5Attribute(xmlName)) {
          result.put(xmlName, null);
        }
      }
      else if (tokenType == XmlTokenType.XML_DOCTYPE_PUBLIC || tokenType == XmlTokenType.XML_DOCTYPE_SYSTEM) {
        // this is not an HTML5 context
        break;
      }
      lexer.advance();
      tokenType = lexer.getTokenType();
    }
    return result;
  }
  return Collections.emptyMap();
}
 
Example 36
Project: TS-IJ   File: TSSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new TSLexerAdapter();
}
 
Example 37
Project: TS-IJ   File: TSParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new TSLexerAdapter();
}
 
Example 38
Project: AppleScript-IDEA   File: AppleScriptParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
  return new FlexAdapter(new _AppleScriptLexer(null));
}
 
Example 39
Project: AppleScript-IDEA   File: AppleScriptSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new FlexAdapter(new _AppleScriptLexer(null));
}
 
Example 40
Project: reasonml-idea-plugin   File: RmlParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(RmlTypes.INSTANCE);
}
 
Example 41
Project: reasonml-idea-plugin   File: OclP4ParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(OclTypes.INSTANCE);
}
 
Example 42
Project: reasonml-idea-plugin   File: OclParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(OclTypes.INSTANCE);
}
 
Example 43
Project: reasonml-idea-plugin   File: MlSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new LexerAdapter(m_types);
}
 
Example 44
Project: greycat-idea-plugin   File: GCMParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new FlexAdapter(new _GCMLexer());
}
 
Example 45
Project: greycat-idea-plugin   File: GCMSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new FlexAdapter(new _GCMLexer());
}
 
Example 46
Project: bamboo-soy   File: SoyParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
  return new SoyLexer();
}
 
Example 47
Project: bamboo-soy   File: SoySyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new SoyLexer();
}
 
Example 48
Project: idea-php-dotenv-plugin   File: DotEnvParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new DotEnvLexerAdapter();
}
 
Example 49
Project: EmojiPrefix   File: EmojiResourceParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
	return new EmojiResourceLexerAdapter();
}
 
Example 50
Project: EmojiPrefix   File: EmojiResourceSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
	return new EmojiResourceLexerAdapter();
}
 
Example 51
Project: intellij-postfix-templates   File: CptSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
	return new CptLexerAdapter();
}
 
Example 52
Project: intellij-postfix-templates   File: CptParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
	return new CptLexerAdapter();
}
 
Example 53
Project: intellij-postfix-templates   File: CptIndexer.java   Source Code and License 4 votes vote down vote up
public static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
	return new CptFilterLexer(new CptLexerAdapter(), consumer);
}
 
Example 54
Project: intellij-postfix-templates   File: CptIndexer.java   Source Code and License 4 votes vote down vote up
@Override
public Lexer createLexer(final OccurrenceConsumer consumer) {
	return createIndexingLexer(consumer);
}
 
Example 55
Project: intellij-postfix-templates   File: CptTodoIndexer.java   Source Code and License 4 votes vote down vote up
@Override
public Lexer createLexer(OccurrenceConsumer consumer) {
	return CptIndexer.createIndexingLexer(consumer);
}
 
Example 56
Project: intellij-postfix-templates   File: CptFilterLexer.java   Source Code and License 4 votes vote down vote up
public CptFilterLexer(final Lexer originalLexer, final OccurrenceConsumer table) {
	super(originalLexer, table);
}
 
Example 57
Project: hybris-integration-intellij-idea-plugin   File: FlexibleSearchParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(Project project) {
    return new FlexibleSearchLexerAdapter();
}
 
Example 58
Project: hybris-integration-intellij-idea-plugin   File: FlexibleSearchSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new FlexibleSearchLexerAdapter();
}
 
Example 59
Project: hybris-integration-intellij-idea-plugin   File: ImpexParserDefinition.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer createLexer(final Project project) {
    return new FlexAdapter(new ImpexLexer((Reader) null));
}
 
Example 60
Project: hybris-integration-intellij-idea-plugin   File: DefaultImpexSyntaxHighlighter.java   Source Code and License 4 votes vote down vote up
@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new ImpexLexerAdapter();
}