Java Code Examples for com.intellij.lang.PsiBuilder#eof()

The following examples show how to use com.intellij.lang.PsiBuilder#eof() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: MMPsiParser.java    From netbeans-mmd-plugin with Apache License 2.0 6 votes vote down vote up
private void parseHeader(@Nonnull final PsiBuilder builder) {
  boolean doLoop = true;
  while (doLoop && !builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();

    if (builder.getTokenType() == null) {
      marker.drop();
    } else {
      final IElementType token = builder.getTokenType();

      if (token == MMTokens.HEADER_DELIMITER) {
        marker.done(token);
        doLoop = false;
      } else if (token == MMTokens.HEADER_LINE
          || token == MMTokens.UNKNOWN
          || token == MMTokens.WHITE_SPACE
          || token == MMTokens.ATTRIBUTES) {
        marker.done(token);
      } else {
        throw Assertions.fail("Unexpected header token : " + token);
      }
    }
    builder.advanceLexer();
  }
}
 
Example 2
Source File: MMPsiParser.java    From netbeans-mmd-plugin with Apache License 2.0 6 votes vote down vote up
private void parseTopics(@Nonnull final PsiBuilder builder) {
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    final IElementType token = builder.getTokenType();

    if (token == null) {
      marker.drop();
    } else {
      if (token == MMTokens.TOPIC_LEVEL) {
        final PsiBuilder.Marker levelMarker = builder.mark();
        levelMarker.done(token);
        final int topicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
        if (topicLevel != 1) {
          marker.done(MMTokens.UNKNOWN);
        } else {
          builder.advanceLexer();
          recursiveParseTopic(builder, topicLevel);
          marker.done(MMTokens.TOPIC);
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
      }
    }
    builder.advanceLexer();
  }
}
 
Example 3
Source File: CSharpFileFactory.java    From consulo-csharp with Apache License 2.0 6 votes vote down vote up
@Nonnull
private static IElementType createElementType(String id, BiConsumer<PsiBuilder, LanguageVersion> consumer)
{
	return new IFileElementType(id, CSharpLanguage.INSTANCE)
	{
		@Override
		protected ASTNode doParseContents(@Nonnull ASTNode chameleon, @Nonnull PsiElement psi)
		{
			final Project project = psi.getProject();
			final Language languageForParser = getLanguageForParser(psi);
			final LanguageVersion tempLanguageVersion = chameleon.getUserData(LanguageVersion.KEY);
			final LanguageVersion languageVersion = tempLanguageVersion == null ? psi.getLanguageVersion() : tempLanguageVersion;
			final PsiBuilder builder = PsiBuilderFactory.getInstance().createBuilder(project, chameleon, null, languageForParser, languageVersion, chameleon.getChars());
			consumer.accept(builder, languageVersion);
			while(!builder.eof())
			{
				builder.advanceLexer();
			}
			return builder.getTreeBuilt();
		}
	};
}
 
Example 4
Source File: GLSLParser.java    From glsl4idea with GNU Lesser General Public License v3.0 6 votes vote down vote up
@Override
public void parseLight(IElementType root, PsiBuilder builder) {
    if (crashing) {
        builder.setDebugMode(true);
    }
    final PsiBuilder.Marker rootMarker = builder.mark();
    if (!builder.eof()) { //Empty file is not an error
        final GLSLParsing theRealParser = new GLSLParsing(builder);

        theRealParser.parseTranslationUnit();
        while (!builder.eof()) // exhaust the file if unable to parse everything
            builder.advanceLexer();
    }

    rootMarker.done(root);
}
 
Example 5
Source File: MakefileParserDefinition.java    From CppTools with Apache License 2.0 5 votes vote down vote up
private static void readIdentifier(PsiBuilder psiBuilder, IElementType identifierEnd, IElementType identifierType) {
  PsiBuilder.Marker identifier = psiBuilder.mark();
  psiBuilder.advanceLexer();
  IElementType currentType;

  while(!psiBuilder.eof() && (currentType = psiBuilder.getTokenType()) != identifierEnd) {
    PsiBuilder.Marker marker = shouldProduceComposite(currentType) ? psiBuilder.mark() : null;
    psiBuilder.advanceLexer();
    if (marker != null) marker.done(currentType);
  }
  identifier.done(identifierType);
}
 
Example 6
Source File: MakefileParserDefinition.java    From CppTools with Apache License 2.0 5 votes vote down vote up
@NotNull
public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) {
  final PsiBuilder.Marker marker = psiBuilder.mark();
  PsiBuilder.Marker statement = psiBuilder.mark();
  IElementType statementType = null;

  while(!psiBuilder.eof()) {
    final IElementType tokenType = psiBuilder.getTokenType();

    if (tokenType == MakefileTokenTypes.SEMANTIC_WHITESPACE) {
      psiBuilder.advanceLexer();
      statement.done(statementType != null ? statementType:MakefileTokenTypes.STATEMENT);
      statementType = null;
      statement = psiBuilder.mark();
      continue;
    } else if (shouldProduceComposite(tokenType)) {
      PsiBuilder.Marker identifier = psiBuilder.mark();
      psiBuilder.advanceLexer();
      identifier.done(tokenType);
      continue;
    } else if (tokenType == MakefileTokenTypes.VAR_DEFINITION) {
      statementType = tokenType;
    } else if (tokenType == MakefileTokenTypes.IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.IDENTIFIER_END, MakefileTokenTypes.IDENTIFIER);
    }
    else if (tokenType == MakefileTokenTypes.TARGET_IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.TARGET_IDENTIFIER_END, MakefileTokenTypes.TARGET_IDENTIFIER);
      if (!psiBuilder.eof()) psiBuilder.advanceLexer();

      statementType = MakefileTokenTypes.TARGET_DECLARATION;
      continue;
    }
    psiBuilder.advanceLexer();
  }

  statement.done(MakefileTokenTypes.STATEMENT);
  marker.done(iElementType);
  return psiBuilder.getTreeBuilt();
}
 
Example 7
Source File: CSharpFragmentFactory.java    From consulo-csharp with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	SharedParsingHelpers.parseType(new CSharpBuilderWrapper(builder, languageVersion), SharedParsingHelpers.VAR_SUPPORT);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 8
Source File: CSharpFragmentFactory.java    From consulo-csharp with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	ExpressionParsing.parse(new CSharpBuilderWrapper(builder, languageVersion), ModifierSet.EMPTY);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 9
Source File: CSharpPreprocessorParser.java    From consulo-csharp with Apache License 2.0 5 votes vote down vote up
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();

	while(!builder.eof())
	{
		PreprocessorParsing.parse(builder);
	}

	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 10
Source File: HaskellParserUtilBase.java    From intellij-haskforce with Apache License 2.0 5 votes vote down vote up
public static boolean indented(@NotNull PsiBuilder builder, int level, Parser geq) {
    if (!(builder instanceof Builder)) return false;
    PsiParser wrapper = ((Builder) builder).parser;
    if (!(wrapper instanceof HaskellParserWrapper)) return false;
    if (builder.eof()) return false;

    // IElementType currtok = builder.getTokenType();
    Pair<Integer, IElementType> prevtok = previousElem(builder);
    if (prevtok == null) return true;

    int offs = builder.getCurrentOffset();
    int line = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs);
    int prevline = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs + prevtok.getFirst());
    if (prevline == line) return true;

    int thisLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), line, 0);
    int prevLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), prevline, 0);
    // CharSequence lineStuff = builder.getOriginalText().subSequence(thisLineStart, offs);
    CharSequence prevLineStuff = builder.getOriginalText().subSequence(prevLineStart, thisLineStart - 1);
    int indentation = indentationLevel(prevLineStuff);
    int myindentation = offs - thisLineStart;
    // String tokName = builder.getTokenText();

    boolean geqVal = geq.parse(builder, level);
    if (geqVal && myindentation >= indentation ||
            !geqVal && myindentation > indentation) return true;
    return false;
}
 
Example 11
Source File: HaskellParserUtilBase.java    From intellij-haskforce with Apache License 2.0 5 votes vote down vote up
/**
 * External rule used to determine if the current lexer position is
 * in an indent. Semantically, this returns true unless the previous
 * character was a newline and the current one is not whitespace.
 * Via recoverWhile, this rule will be used to consume input until
 * it reaches an unindented token, presumably a top-level element
 * of some sort, and resume at that time. It's quite convenient to just
 * resume parsing once we encounter a new top-level element.
 */
public static boolean inIndentRecover(@NotNull PsiBuilder builder, int level) {
    if (builder.eof()) return false;
    final int offset = builder.getCurrentOffset();
    if (offset == 0) return false;
    final CharSequence text = builder.getOriginalText();
    return !(
        text.charAt(offset - 1) == '\n'
            && !Character.isWhitespace(text.charAt(offset)));
}
 
Example 12
Source File: HaskellParserUtilBase.java    From intellij-haskforce with Apache License 2.0 5 votes vote down vote up
public static boolean toplevel_recover_debug(PsiBuilder builder, int level) {
    IElementType typ = builder.getTokenType();
    final boolean res = !(TOPLEVEL_RECOVER_TYPES.contains(typ) || builder.eof());
    System.out.println(
        "toplevel_recover_debug: res: " + res
          + "; offset: " + builder.getCurrentOffset()
          + "; type: " + typ
          + "; text: '" + builder.getTokenText() + "'");
    return res;
}
 
Example 13
Source File: SimplePsiParser.java    From intellij-haskforce with Apache License 2.0 5 votes vote down vote up
@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
    PsiBuilder.Marker marker = builder.mark();
    while (!builder.eof()) {
        builder.advanceLexer();
    }
    marker.done(root);
    return builder.getTreeBuilt();
}
 
Example 14
Source File: UnescapingPsiBuilder.java    From BashSupport with Apache License 2.0 5 votes vote down vote up
/**
 * @param exact if true then positions should be equal;
 *              else delegate should be behind, not including exactly all foreign (skipped) or whitespace tokens
 */
private void synchronizePositions(boolean exact) {
    final PsiBuilder delegate = getDelegate();

    if (myLexPosition >= myShrunkSequenceSize || delegate.eof()) {
        myLexPosition = myShrunkSequenceSize;
        while (!delegate.eof()) {
            delegate.advanceLexer();
        }
        return;
    }

    if (delegate.getCurrentOffset() > myShrunkSequence.get(myLexPosition).realStart) {
        LOG.debug("delegate is ahead of my builder!");
        return;
    }

    final int keepUpPosition = getKeepUpPosition(exact);

    while (!delegate.eof()) {
        final int delegatePosition = delegate.getCurrentOffset();

        if (delegatePosition < keepUpPosition) {
            delegate.advanceLexer();
        } else {
            break;
        }
    }
}
 
Example 15
Source File: MMPsiParser.java    From netbeans-mmd-plugin with Apache License 2.0 5 votes vote down vote up
private boolean parseExtraBlock(@Nonnull final PsiBuilder builder) {
  // read type
  final PsiBuilder.Marker type = builder.mark();
  if (builder.getTokenType() != MMTokens.EXTRA_TYPE) {
    throw Assertions.fail("Unexpected token " + builder.getTokenType());
  }
  builder.advanceLexer();
  type.done(MMTokens.EXTRA_TYPE);

  boolean dataFound = false;

  // read body
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    if (builder.eof() || builder.getTokenType() == null) {
      marker.drop();
      break;
    } else {
      final IElementType token = builder.getTokenType();
      if (token == MMTokens.TOPIC_LEVEL || token == MMTokens.EXTRA_TYPE) {
        marker.rollbackTo();
        return true;
      } else if (token == MMTokens.EXTRA_BODY || token == MMTokens.WHITE_SPACE) {
        if (dataFound && token == MMTokens.EXTRA_BODY) {
          builder.advanceLexer();
          marker.done(MMTokens.UNKNOWN);
          break;
        } else {
          builder.advanceLexer();
          marker.done(token);
          dataFound = dataFound || token == MMTokens.EXTRA_BODY;
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
        break;
      }
    }
  }
  return false;
}
 
Example 16
Source File: HaskellParserUtilBase.java    From intellij-haskforce with Apache License 2.0 4 votes vote down vote up
public static boolean anyNonSemi(@NotNull PsiBuilder builder, int level) {
    return
        !builder.eof() && (
            builder.getTokenType() == HaskellTypes.SEMICOLON
              || builder.getTokenType() == HaskellTypes.WHITESPACESEMITOK);
}
 
Example 17
Source File: ShaderLabParser.java    From consulo-unity3d with Apache License 2.0 4 votes vote down vote up
public static void parseElementsInBraces(@Nonnull PsiBuilder builder, IElementType open, IElementType close, IElementType valid)
{
	if(builder.getTokenType() == open)
	{
		builder.advanceLexer();

		while(!builder.eof())
		{
			IElementType tokenType = builder.getTokenType();
			if(tokenType == close)
			{
				break;
			}

			if(valid != null)
			{
				if(tokenType == valid)
				{
					builder.advanceLexer();
				}
				else
				{
					doneError(builder, "Unexpected token");
				}
			}
			else
			{
				builder.advanceLexer();
			}

			if(builder.getTokenType() == ShaderLabTokens.COMMA)
			{
				builder.advanceLexer();
			}
			else if(builder.getTokenType() != close)
			{
				doneError(builder, "Unexpected token");
			}
		}
		expectWithError(builder, close, "Unexpected token");
	}
}
 
Example 18
Source File: MMPsiParser.java    From netbeans-mmd-plugin with Apache License 2.0 4 votes vote down vote up
private int recursiveParseTopic(@Nonnull final PsiBuilder builder, final int level) {

    while (!builder.eof()) {
      final PsiBuilder.Marker marker = builder.mark();

      final IElementType token = builder.getTokenType();

      if (token == null) {
        marker.drop();
      } else {
        if (token == MMTokens.TOPIC_LEVEL) {
          final PsiBuilder.Marker levelMarker = builder.mark();
          levelMarker.done(token);

          final int theTopicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
          if (theTopicLevel <= 1) {
            marker.done(MMTokens.UNKNOWN);
          } else {
            if (theTopicLevel <= level) {
              marker.rollbackTo();
              return theTopicLevel;
            } else {
              builder.advanceLexer();
              final int parsedTopicLevel = recursiveParseTopic(builder, theTopicLevel);
              marker.done(MMTokens.TOPIC);
              if (parsedTopicLevel < theTopicLevel) {
                return parsedTopicLevel;
              }
              if (parsedTopicLevel == theTopicLevel) {
                continue;
              }
            }
          }
        } else if (token == MMTokens.TOPIC_TITLE || token == MMTokens.CODE_SNIPPET_BODY || token == MMTokens.CODE_SNIPPET_END || token == MMTokens.CODE_SNIPPET_START || token == MMTokens.ATTRIBUTES) {
          marker.done(token);
        } else if (token == MMTokens.EXTRA_TYPE) {
          try {
            if (parseExtraBlock(builder)) {
              continue;
            }
          } finally {
            marker.done(MMTokens.EXTRA_DATA);
          }
        } else if (token == MMTokens.WHITE_SPACE) {
          marker.done(token);
        } else {
          marker.done(MMTokens.UNKNOWN);
        }
      }
      builder.advanceLexer();
    }


    return level;
  }
 
Example 19
Source File: ANTLRParserAdaptor.java    From antlr4-intellij-adaptor with BSD 2-Clause "Simplified" License 4 votes vote down vote up
@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
	ProgressIndicatorProvider.checkCanceled();

	TokenSource source = new PSITokenSource(builder);
	TokenStream tokens = new CommonTokenStream(source);
	parser.setTokenStream(tokens);
	parser.setErrorHandler(new ErrorStrategyAdaptor()); // tweaks missing tokens
	parser.removeErrorListeners();
	parser.addErrorListener(new SyntaxErrorListener()); // trap errors
	ParseTree parseTree = null;
	PsiBuilder.Marker rollbackMarker = builder.mark();
	try {
		parseTree = parse(parser, root);
	}
	finally {
		rollbackMarker.rollbackTo();
	}

	// Now convert ANTLR parser tree to PSI tree by mimicking subtree
	// enter/exit with mark/done calls. I *think* this creates their parse
	// tree (AST as they call it) when you call {@link PsiBuilder#getTreeBuilt}
	ANTLRParseTreeToPSIConverter listener = createListener(parser, root, builder);
	PsiBuilder.Marker rootMarker = builder.mark();
	ParseTreeWalker.DEFAULT.walk(listener, parseTree);
	while (!builder.eof()) {
		ProgressIndicatorProvider.checkCanceled();
		builder.advanceLexer();
	}
	// NOTE: parse tree returned from parse will be the
	// usual ANTLR tree ANTLRParseTreeToPSIConverter will
	// convert that to the analogous jetbrains AST nodes
	// When parsing an entire file, the root IElementType
	// will be a IFileElementType.
	//
	// When trying to rename IDs and so on, you get a
	// dummy root and a type arg identifier IElementType.
	// This results in a weird tree that has for example
	// (ID (expr (primary ID))) with the ID IElementType
	// as a subtree root as well as the appropriate leaf
	// all the way at the bottom.  The dummy ID root is a
	// CompositeElement and created by
	// ParserDefinition.createElement() despite having
	// being TokenIElementType.
	rootMarker.done(root);
	return builder.getTreeBuilt(); // calls the ASTFactory.createComposite() etc...
}