Java Code Examples for com.intellij.lexer.Lexer

@Override
public MergeFunction getMergeFunction() {
  return ((final IElementType type, final Lexer originalLexer) -> {
    if (type == SoyTypes.OTHER || type == TokenType.WHITE_SPACE) {
      IElementType returnType =  type;
      while (originalLexer.getTokenType() == SoyTypes.OTHER
          || originalLexer.getTokenType() == TokenType.WHITE_SPACE) {
        if (originalLexer.getTokenType() == SoyTypes.OTHER) {
          returnType = SoyTypes.OTHER;
        }
        originalLexer.advance();
      }
      return returnType;
    }

    return type;
  });
}
 

public static void addWordHonoringEscapeSequences(CharSequence editorText,
                                                  TextRange literalTextRange,
                                                  int cursorOffset,
                                                  Lexer lexer,
                                                  List<TextRange> result) {
  lexer.start(editorText, literalTextRange.getStartOffset(), literalTextRange.getEndOffset());

  while (lexer.getTokenType() != null) {
    if (lexer.getTokenStart() <= cursorOffset && cursorOffset < lexer.getTokenEnd()) {
      if (StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(lexer.getTokenType())) {
        result.add(new TextRange(lexer.getTokenStart(), lexer.getTokenEnd()));
      }
      else {
        TextRange word = getWordSelectionRange(editorText, cursorOffset, JAVA_IDENTIFIER_PART_CONDITION);
        if (word != null) {
          result.add(new TextRange(Math.max(word.getStartOffset(), lexer.getTokenStart()),
                                   Math.min(word.getEndOffset(), lexer.getTokenEnd())));
        }
      }
      break;
    }
    lexer.advance();
  }
}
 

protected void checkCorrectRestart(String text) {
  Lexer mainLexer = createLexer();
  String allTokens = printTokens(text, 0, mainLexer);

  Lexer auxLexer = createLexer();
  auxLexer.start(text);
  while (true) {
    IElementType type = auxLexer.getTokenType();
    if (type == null) {
      break;
    }
    if (auxLexer.getState() == 0) {
      int tokenStart = auxLexer.getTokenStart();
      String subTokens = printTokens(text, tokenStart, mainLexer);
      if (!allTokens.endsWith(subTokens)) {
        assertEquals("Restarting impossible from offset " + tokenStart + "; lexer state should not return 0 at this point", allTokens, subTokens);
      }
    }
    auxLexer.advance();
  }
}
 

@Override
public MergeFunction getMergeFunction() {
    return new MergeFunction() {

        @Override
        public IElementType merge(IElementType type, Lexer originalLexer) {
            if (!allMergables.contains(type)) {
                return type;
            }

            TokenSet merging = tokensToMerge.contains(type) ? tokensToMerge : tokensToMerge2;

            while (true) {
                final IElementType tokenType = originalLexer.getTokenType();
                if (!merging.contains(tokenType)) break;
                originalLexer.advance();
            }

            return merging == tokensToMerge ? LONGSTRING : LONGCOMMENT;
        }
    };
}
 

private ASTNode createAST(String text) {
  ParserDefinition definition = new BuildParserDefinition();
  PsiParser parser = definition.createParser(getProject());
  Lexer lexer = definition.createLexer(getProject());
  PsiBuilderImpl psiBuilder =
      new PsiBuilderImpl(
          getProject(), null, definition, lexer, new CharTableImpl(), text, null, null);
  PsiBuilderAdapter adapter =
      new PsiBuilderAdapter(psiBuilder) {
        @Override
        public void error(String messageText) {
          super.error(messageText);
          errors.add(messageText);
        }
      };
  return parser.parse(definition.getFileNodeType(), adapter);
}
 

@Nullable
public static String parseQName(Lexer lexer) {
  String name;
  if (lexer.getTokenType() == XPathTokenTypes.NCNAME) {
    name = lexer.getTokenText();
    lexer.advance();
    if (lexer.getTokenType() == XPathTokenTypes.COL) {
      lexer.advance();
      assert lexer.getTokenType() == XPathTokenTypes.NCNAME;
      name += (":" + lexer.getTokenText());
      lexer.advance();
    }
  } else {
    name = null;
  }
  return name;
}
 

protected void registerHandler(IElementType elementType, TokenHandler value) {
  final TokenHandler tokenHandler = tokenHandlers.get(elementType);

  if (tokenHandler != null) {
    final TokenHandler newHandler = value;
    value = new TokenHandler() {
      @Override
      public void handleElement(final Lexer lexer) {
        tokenHandler.handleElement(lexer);
        newHandler.handleElement(lexer);
      }
    };
  }

  tokenHandlers.put(elementType,value);
}
 

private static void doTest(String text, String[] expectedTokens,Lexer lexer) {
  lexer.start(text);
  int idx = 0;
  while (lexer.getTokenType() != null) {
    if (idx >= expectedTokens.length) fail("Too many tokens");
    String tokenName = lexer.getTokenType().toString();
    String expectedTokenType = expectedTokens[idx++];
    String expectedTokenText = expectedTokens[idx++];
    assertEquals(expectedTokenType, tokenName);
    String tokenText = lexer.getBufferSequence().subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
    assertEquals(expectedTokenText, tokenText);
    lexer.advance();
  }

  if (idx < expectedTokens.length) fail("Not enough tokens");
}
 

@Test
public void testError2() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "ab");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    // Simplified lexer ignores this error
    //assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 

@Test
public void testError3() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "a: x\n" +
            "b: y\n" +
            "@bad!");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 3, 5, "x\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Key);
    assertAndAdvance(l, YamlTokenTypes.YAML_Value);
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 8, 10, "y\n");

    assertAndAdvance(l, YamlTokenTypes.YAML_Error);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar);

    assertEquals(null, l.getTokenType());
}
 

@Override
public int getErrorsCount(final CharSequence seq, Language fileLanguage, final Project project) {
  Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);

  lexer.start(seq);
  if (lexer.getTokenType() != JavaTokenType.LBRACE) return IErrorCounterReparseableElementType.FATAL_ERROR;
  lexer.advance();
  int balance = 1;
  while (true) {
    IElementType type = lexer.getTokenType();
    if (type == null) break;
    if (balance == 0) return IErrorCounterReparseableElementType.FATAL_ERROR;
    if (type == JavaTokenType.LBRACE) {
      balance++;
    }
    else if (type == JavaTokenType.RBRACE) {
      balance--;
    }
    lexer.advance();
  }
  return balance;
}
 

private static boolean canStickJavaTokens(ASTNode token1, ASTNode token2) {
  IElementType type1 = token1.getElementType();
  IElementType type2 = token2.getElementType();

  Pair<IElementType, IElementType> pair = Pair.create(type1, type2);
  Boolean res = myCanStickJavaTokensMatrix.get(pair);
  if (res == null) {
    if (!checkToken(token1) || !checkToken(token2)) return true;
    String text = token1.getText() + token2.getText();
    Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
    lexer.start(text);
    boolean canMerge = lexer.getTokenType() == type1;
    lexer.advance();
    canMerge &= lexer.getTokenType() == type2;
    res = canMerge;
    myCanStickJavaTokensMatrix.put(pair, res);
  }
  return res.booleanValue();
}
 

protected PsiElement createEmptyPsiField() {
    final String text = "<undefined>";
    final IElementType type = CsvTypes.FIELD;
    final Project project = this.element.getProject();
    final PsiManager psiManager = PsiManager.getInstance(project);
    final DummyHolder dummyHolder = DummyHolderFactory.createHolder(psiManager, null);
    final FileElement fileElement = dummyHolder.getTreeElement();
    final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(CsvLanguage.INSTANCE);
    final Lexer lexer = parserDefinition.createLexer(project);
    final PsiBuilder psiBuilder = PsiBuilderFactory.getInstance().createBuilder(project, fileElement, lexer, CsvLanguage.INSTANCE, text);
    final ASTNode node = parserDefinition.createParser(project).parse(type, psiBuilder);
    fileElement.rawAddChildren((com.intellij.psi.impl.source.tree.TreeElement)node);
    return node.getPsi();
}
 

@Override
@NotNull
public Lexer getHighlightingLexer() {
  if (myIsDtd) {
    return new DtdLexer(true);
  } else if (myIsXHtml) {
    return new XHtmlHighlightingLexer();
  } else {
    return new XmlHighlightingLexer();
  }
}
 

private boolean replaceTokenNotFollowedBy(Lexer baseLexer, final IElementType replacement, IElementType nextToken) {
  if (!lookingAt(baseLexer, nextToken)) {
    treatAs(baseLexer, replacement);
    return true;
  }
  return false;
}
 

@Override
public Lexer getIndexingLexer(@NotNull PsiFile file) {
    if (file instanceof GroovyFile) {
        return new GroovyLexer();
    }
    return null;
}
 

private static boolean lookingAt(Lexer baseLexer, IElementType... tokens) {
  final LexerPosition position = baseLexer.getCurrentPosition();
  try {
    for (IElementType token : tokens) {
      baseLexer.advance();
      skipWhitespaceAnComments(baseLexer);
      if (baseLexer.getTokenType() != token) {
        return false;
      }
    }
    return true;
  } finally {
    baseLexer.restore(position);
  }
}
 

@Override
public void handleElement(Lexer lexer) {
  if (seenAttribute) {
    seenStyle = false;
    seenScript = false;
    seenAttribute = false;
  }
  seenContentType = false;
  seenStylesheetType = false;
}
 

@Override
public void handleElement(Lexer lexer) {
  if (seenContentType && seenScript && !seenAttribute) {
    @NonNls String mimeType = TreeUtil.getTokenText(lexer);
    scriptType = caseInsensitive ? mimeType.toLowerCase(Locale.US) : mimeType;
  }
  if (seenStylesheetType && seenStyle && !seenAttribute) {
    @NonNls String rel = TreeUtil.getTokenText(lexer).trim();
    styleType = caseInsensitive ? rel.toLowerCase(Locale.US) : rel;
  }
}
 

@Override
public void handleElement(Lexer lexer) {
  seenStyle=false;
  seenScript=false;
  seenAttribute=false;
  seenContentType=false;
  seenStylesheetType=false;
  scriptType = null;
  styleType = null;
}
 

public static void doLexerTest(String text,
                               Lexer lexer,
                               boolean checkTokenText,
                               String... expectedTokens) {
  lexer.start(text);
  int idx = 0;
  int tokenPos = 0;
  while (lexer.getTokenType() != null) {
    if (idx >= expectedTokens.length) {
      StringBuilder remainingTokens = new StringBuilder("\"" + lexer.getTokenType().toString() + "\"");
      lexer.advance();
      while (lexer.getTokenType() != null) {
        remainingTokens.append(",");
        remainingTokens.append(" \"").append(checkTokenText ? lexer.getTokenText() : lexer.getTokenType().toString()).append("\"");
        lexer.advance();
      }
      fail("Too many tokens. Following tokens: " + remainingTokens.toString());
    }
    assertEquals("Token offset mismatch at position " + idx, tokenPos, lexer.getTokenStart());
    String tokenName = checkTokenText ? lexer.getTokenText() : lexer.getTokenType().toString();
    assertEquals("Token mismatch at position " + idx, expectedTokens[idx], tokenName);
    idx++;
    tokenPos = lexer.getTokenEnd();
    lexer.advance();
  }

  if (idx < expectedTokens.length) fail("Not enough tokens");
}
 

@Nullable
public static String getPackageName(CharSequence text) {
  Lexer lexer = new GroovyLexer();
  lexer.start(text);
  skipWhitespacesAndComments(lexer);
  final IElementType firstToken = lexer.getTokenType();
  if (firstToken != GroovyTokenTypes.kPACKAGE) {
    return "";
  }
  lexer.advance();
  skipWhitespacesAndComments(lexer);

  final StringBuilder buffer = StringBuilderSpinAllocator.alloc();
  try {
    while(true){
      if (lexer.getTokenType() != GroovyTokenTypes.mIDENT) break;
      buffer.append(text, lexer.getTokenStart(), lexer.getTokenEnd());
      lexer.advance();
      skipWhitespacesAndComments(lexer);
      if (lexer.getTokenType() != GroovyTokenTypes.mDOT) break;
      buffer.append('.');
      lexer.advance();
      skipWhitespacesAndComments(lexer);
    }
    String packageName = buffer.toString();
    if (packageName.isEmpty() || StringUtil.endsWithChar(packageName, '.')) return null;
    return packageName;
  }
  finally {
    StringBuilderSpinAllocator.dispose(buffer);
  }
}
 

public ASTNode generateWhitespaceBetweenTokens(ASTNode left, ASTNode right) {
  PsiManager manager = right.getTreeParent().getPsi().getManager();
  if (left.getElementType() == PyTokenTypes.END_OF_LINE_COMMENT) {
    return createLineBreak(manager);
  }

  if (left.getPsi().isValid() && right.getPsi().isValid()) {
    final PsiElement commonParent = PsiTreeUtil.findCommonParent(left.getPsi(), right.getPsi());
    if (commonParent == null) return null;
    final PsiElement leftPrevAncestor = PsiTreeUtil.findPrevParent(commonParent, left.getPsi());
    final PsiElement rightPrevAncestor = PsiTreeUtil.findPrevParent(commonParent, right.getPsi());

    if (isStatementOrFunction(leftPrevAncestor) && isStatementOrFunction(rightPrevAncestor)) {
      int leftIndent = PyPsiUtils.getElementIndentation(leftPrevAncestor);
      int rightIndent = PyPsiUtils.getElementIndentation(rightPrevAncestor);
      int maxIndent = Math.max(leftIndent, rightIndent);
      return createWhitespace(manager, "\n" + StringUtil.repeatSymbol(' ', maxIndent));
    }
  }

  if (right.getElementType() == PyTokenTypes.DEF_KEYWORD || right.getElementType() == PyTokenTypes.CLASS_KEYWORD) {
    return createLineBreak(manager);
  }
  if (left.getElementType() == TokenType.WHITE_SPACE || right.getElementType() == TokenType.WHITE_SPACE) {
    return null;
  }
  final PyStatement leftStatement = PsiTreeUtil.getParentOfType(left.getPsi(), PyStatement.class);
  if (leftStatement != null && !PsiTreeUtil.isAncestor(leftStatement, right.getPsi(), false)) {
    return createLineBreak(manager);
  }
  final Lexer lexer = new PythonIndentingLexer();
  if (LanguageUtil.canStickTokensTogetherByLexer(left, right, lexer) == ParserDefinition.SpaceRequirements.MUST) {
    return createSpace(manager);
  }
  return null;
}
 

public InplaceVariableIntroducer(PsiNamedElement elementToRename,
                                 Editor editor,
                                 final Project project,
                                 String title, E[] occurrences,
                                 @Nullable E expr) {
  super(editor, elementToRename, project);
  myTitle = title;
  myOccurrences = occurrences;
  if (expr != null) {
    final ASTNode node = expr.getNode();
    ASTNode prev = node.getTreePrev();
    final ASTNode astNode = LanguageTokenSeparatorGenerators.INSTANCE.forLanguage(expr.getLanguage())
      .generateWhitespaceBetweenTokens(prev, node);
    if (astNode != null) {
      final Lexer lexer = LanguageParserDefinitions.INSTANCE.forLanguage(expr.getLanguage()).createLexer(project);
      if (LanguageUtil.canStickTokensTogetherByLexer(prev, prev, lexer) == ParserDefinition.SpaceRequirements.MUST) {
        PostprocessReformattingAspect.getInstance(project).disablePostprocessFormattingInside(new Runnable() {
          @Override
          public void run() {
            new WriteCommandAction<Object>(project, "Normalize declaration") {
              @Override
              protected void run(@NotNull Result<Object> result) throws Throwable {
                node.getTreeParent().addChild(astNode, node);
              }
            }.execute(); 
          }
        });
      }
    }
    myExpr = expr;
  }
  myExprMarker = myExpr != null && myExpr.isPhysical() ? createMarker(myExpr) : null;
  initOccurrencesMarkers();
}
 

@Test
public void testKeys() {
    Lexer l = createLexer();

    startSkippingHeaders(l, "key: val");
    assertAndAdvance(l, YamlTokenTypes.YAML_Key, 0, 3, "key");
    assertAndAdvance(l, YamlTokenTypes.YAML_Value, 3, 4, ":");
    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace);
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 5, 8, "val");
    assertEquals(null, l.getTokenType());
}
 

@Test
public void testKeywords() {
    Lexer l = createLexer();

    l.start("[true,off,TruE,\"true\",12,12.3,null]");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowSequenceStart, 0, 1, "[");
    assertAndAdvance(l, YamlTokenTypes.YAML_Tag_BOOL, 1, 5, "true");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 5, 6, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Tag_BOOL, 6, 9, "off");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 9, 10, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 10, 14, "TruE");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry, 14, 15, ",");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 15, 21, "\"true\"");

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_INT, l.getTokenType());
    assertEquals("12", l.getTokenText());
    l.advance();

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_FLOAT, l.getTokenType());
    assertEquals("12.3", l.getTokenText());
    l.advance();

    assertAndAdvance(l, YamlTokenTypes.YAML_FlowEntry);
    assertEquals(YamlTokenTypes.YAML_Tag_NULL, l.getTokenType());
    assertEquals("null", l.getTokenText());
    l.advance();

    assertEquals(YamlTokenTypes.YAML_FlowSequenceEnd, l.getTokenType());
    assertEquals("]", l.getTokenText());
    l.advance();

    //assertEquals(null, l.getTokenType());
}
 

public void test73() throws Exception {
    Lexer l = createLexer();

    l.start(" c ");

    assertAndAdvance(l, YamlTokenTypes.YAML_Whitespace, 0, 1, " ");
    assertAndAdvance(l, YamlTokenTypes.YAML_Scalar, 1, 3, "c ");
}
 

private void assertAndAdvance(Lexer l, IElementType expectedType, int start, int end, String text) {
    assertEquals(expectedType, l.getTokenType());
    assertEquals(start, l.getTokenStart());
    assertEquals(end, l.getTokenEnd());
    assertEquals(text, l.getTokenText());
    l.advance();
}
 

@Nullable
@Override
public ASTNode parseContents(ASTNode node) {
  final LanguageLevel languageLevel = getLanguageLevel(node.getPsi());
  if (PydevConsoleRunner.isPythonConsole(node)) {
    return parseConsoleCode(node, PydevConsoleRunner.getPythonConsoleData(node));
  }
  else {
    final PsiElement psi = node.getPsi();
    if (psi != null) {
      final Project project = psi.getProject();
      final PsiBuilderFactory factory = PsiBuilderFactory.getInstance();
      final Language language = getLanguage();
      final ParserDefinition parserDefinition = LanguageParserDefinitions.INSTANCE.forLanguage(language);
      if (parserDefinition == null) {
        return null;
      }
      final Lexer lexer = parserDefinition.createLexer(project);
      final PsiParser parser = parserDefinition.createParser(project);
      final PsiBuilder builder = factory.createBuilder(project, node, lexer, language, node.getChars());
      if (parser instanceof PyParser) {
        final PyParser pythonParser = (PyParser)parser;
        pythonParser.setLanguageLevel(languageLevel);
        if (languageLevel == LanguageLevel.PYTHON26 && psi.getContainingFile().getName().equals("__builtin__.py")) {
          pythonParser.setFutureFlag(StatementParsing.FUTURE.PRINT_FUNCTION);
        }
      }
      return parser.parse(this, builder).getFirstChildNode();
    }
    return null;
  }
}
 

private static void parseParameters(MacroCallNode macroCallNode, Lexer lexer, String expression) {
  if (lexer.getTokenType() != MacroTokenType.RPAREN) {
    while (lexer.getTokenType() != null) {
      Expression node = parseMacro(lexer, expression);
      macroCallNode.addParameter(node);

      if (lexer.getTokenType() == MacroTokenType.COMMA) {
        advance(lexer);
      }
      else {
        break;
      }
    }
  }
}
 

private static boolean checkToken(final ASTNode token1) {
  Lexer lexer = JavaParserDefinition.createLexer(LanguageLevel.HIGHEST);
  final String text = token1.getText();
  lexer.start(text);
  if (lexer.getTokenType() != token1.getElementType()) return false;
  lexer.advance();
  return lexer.getTokenType() == null;
}
 

public static Lexer createPlainTextLexer() {
  ArrayList<TokenParser> tokenParsers = new ArrayList<TokenParser>();
  tokenParsers.add(new WhitespaceParser());

  tokenParsers.addAll(BraceTokenParser.getBraces());
  tokenParsers.addAll(BraceTokenParser.getParens());
  tokenParsers.addAll(BraceTokenParser.getBrackets());
  tokenParsers.addAll(BraceTokenParser.getAngleBrackets());

  return new MergingLexerAdapter(new AbstractCustomLexer(tokenParsers), TokenSet.create(CustomHighlighterTokenType.CHARACTER));
}
 

@Override
public Lexer getIndexingLexer(@NotNull final PsiFile file) {
  if (JspPsiUtil.isInJspFile(file)) {
    return EditorHighlighterCache.getLexerBasedOnLexerHighlighter(file.getText(), file.getVirtualFile(), file.getProject());
  }

  return null;
}
 

protected OuterLanguageElementImpl createOuterLanguageElement(final Lexer lexer, final CharTable table,
                                                              @NotNull IElementType outerElementType) {
  final CharSequence buffer = lexer.getBufferSequence();
  final int tokenStart = lexer.getTokenStart();
  if (tokenStart < 0 || tokenStart > buffer.length()) {
    LOG.error("Invalid start: " + tokenStart + "; " + lexer);
  }
  final int tokenEnd = lexer.getTokenEnd();
  if (tokenEnd < 0 || tokenEnd > buffer.length()) {
    LOG.error("Invalid end: " + tokenEnd + "; " + lexer);
  }

  return new OuterLanguageElementImpl(outerElementType, table.intern(buffer, tokenStart, tokenEnd));
}
 

@Override
@NotNull
public Map<String, Void> map(@NotNull FileContent inputData) {
  CharSequence input = inputData.getContentAsText();
  Language language = ((LanguageFileType)inputData.getFileType()).getLanguage();
  if (language == HTMLLanguage.INSTANCE || language == XHTMLLanguage.INSTANCE) {
    final Lexer lexer = (language == HTMLLanguage.INSTANCE ? new HtmlHighlightingLexer(FileTypeManager.getInstance().getStdFileType("CSS")) : new XHtmlHighlightingLexer());
    lexer.start(input);
    Map<String, Void> result = new HashMap<String, Void>();
    IElementType tokenType = lexer.getTokenType();
    while (tokenType != null) {
      if (tokenType == XmlTokenType.XML_NAME) {
        String xmlName = input.subSequence(lexer.getTokenStart(), lexer.getTokenEnd()).toString();
        if (HtmlUtil.isCustomHtml5Attribute(xmlName)) {
          result.put(xmlName, null);
        }
      }
      else if (tokenType == XmlTokenType.XML_DOCTYPE_PUBLIC || tokenType == XmlTokenType.XML_DOCTYPE_SYSTEM) {
        // this is not an HTML5 context
        break;
      }
      lexer.advance();
      tokenType = lexer.getTokenType();
    }
    return result;
  }
  return Collections.emptyMap();
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new TSLexerAdapter();
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new TSLexerAdapter();
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
  return new FlexAdapter(new _AppleScriptLexer(null));
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new FlexAdapter(new _AppleScriptLexer(null));
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(RmlTypes.INSTANCE);
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(OclTypes.INSTANCE);
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new LexerAdapter(OclTypes.INSTANCE);
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new LexerAdapter(m_types);
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new FlexAdapter(new _GCMLexer());
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new FlexAdapter(new _GCMLexer());
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
  return new SoyLexer();
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
  return new SoyLexer();
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new DotEnvLexerAdapter();
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
	return new EmojiResourceLexerAdapter();
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
	return new EmojiResourceLexerAdapter();
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
	return new CptLexerAdapter();
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
	return new CptLexerAdapter();
}
 

public static Lexer createIndexingLexer(OccurrenceConsumer consumer) {
	return new CptFilterLexer(new CptLexerAdapter(), consumer);
}
 

@Override
public Lexer createLexer(final OccurrenceConsumer consumer) {
	return createIndexingLexer(consumer);
}
 

@Override
public Lexer createLexer(OccurrenceConsumer consumer) {
	return CptIndexer.createIndexingLexer(consumer);
}
 

public CptFilterLexer(final Lexer originalLexer, final OccurrenceConsumer table) {
	super(originalLexer, table);
}
 

@NotNull
@Override
public Lexer createLexer(Project project) {
    return new FlexibleSearchLexerAdapter();
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new FlexibleSearchLexerAdapter();
}
 

@NotNull
@Override
public Lexer createLexer(final Project project) {
    return new FlexAdapter(new ImpexLexer((Reader) null));
}
 

@NotNull
@Override
public Lexer getHighlightingLexer() {
    return new ImpexLexerAdapter();
}