Java Code Examples for com.intellij.lang.PsiBuilder#eof()

The following examples show how to use com.intellij.lang.PsiBuilder#eof() . These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source Project: netbeans-mmd-plugin   File: MMPsiParser.java    License: Apache License 2.0 6 votes vote down vote up
private void parseHeader(@Nonnull final PsiBuilder builder) {
  boolean doLoop = true;
  while (doLoop && !builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();

    if (builder.getTokenType() == null) {
      marker.drop();
    } else {
      final IElementType token = builder.getTokenType();

      if (token == MMTokens.HEADER_DELIMITER) {
        marker.done(token);
        doLoop = false;
      } else if (token == MMTokens.HEADER_LINE
          || token == MMTokens.UNKNOWN
          || token == MMTokens.WHITE_SPACE
          || token == MMTokens.ATTRIBUTES) {
        marker.done(token);
      } else {
        throw Assertions.fail("Unexpected header token : " + token);
      }
    }
    builder.advanceLexer();
  }
}
 
Example 2
Source Project: netbeans-mmd-plugin   File: MMPsiParser.java    License: Apache License 2.0 6 votes vote down vote up
private void parseTopics(@Nonnull final PsiBuilder builder) {
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    final IElementType token = builder.getTokenType();

    if (token == null) {
      marker.drop();
    } else {
      if (token == MMTokens.TOPIC_LEVEL) {
        final PsiBuilder.Marker levelMarker = builder.mark();
        levelMarker.done(token);
        final int topicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
        if (topicLevel != 1) {
          marker.done(MMTokens.UNKNOWN);
        } else {
          builder.advanceLexer();
          recursiveParseTopic(builder, topicLevel);
          marker.done(MMTokens.TOPIC);
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
      }
    }
    builder.advanceLexer();
  }
}
 
Example 3
@Override
public void parseLight(IElementType root, PsiBuilder builder) {
    if (crashing) {
        builder.setDebugMode(true);
    }
    final PsiBuilder.Marker rootMarker = builder.mark();
    if (!builder.eof()) { //Empty file is not an error
        final GLSLParsing theRealParser = new GLSLParsing(builder);

        theRealParser.parseTranslationUnit();
        while (!builder.eof()) // exhaust the file if unable to parse everything
            builder.advanceLexer();
    }

    rootMarker.done(root);
}
 
Example 4
@Nonnull
private static IElementType createElementType(String id, BiConsumer<PsiBuilder, LanguageVersion> consumer)
{
	return new IFileElementType(id, CSharpLanguage.INSTANCE)
	{
		@Override
		protected ASTNode doParseContents(@Nonnull ASTNode chameleon, @Nonnull PsiElement psi)
		{
			final Project project = psi.getProject();
			final Language languageForParser = getLanguageForParser(psi);
			final LanguageVersion tempLanguageVersion = chameleon.getUserData(LanguageVersion.KEY);
			final LanguageVersion languageVersion = tempLanguageVersion == null ? psi.getLanguageVersion() : tempLanguageVersion;
			final PsiBuilder builder = PsiBuilderFactory.getInstance().createBuilder(project, chameleon, null, languageForParser, languageVersion, chameleon.getChars());
			consumer.accept(builder, languageVersion);
			while(!builder.eof())
			{
				builder.advanceLexer();
			}
			return builder.getTreeBuilt();
		}
	};
}
 
Example 5
Source Project: netbeans-mmd-plugin   File: MMPsiParser.java    License: Apache License 2.0 5 votes vote down vote up
private boolean parseExtraBlock(@Nonnull final PsiBuilder builder) {
  // read type
  final PsiBuilder.Marker type = builder.mark();
  if (builder.getTokenType() != MMTokens.EXTRA_TYPE) {
    throw Assertions.fail("Unexpected token " + builder.getTokenType());
  }
  builder.advanceLexer();
  type.done(MMTokens.EXTRA_TYPE);

  boolean dataFound = false;

  // read body
  while (!builder.eof()) {
    final PsiBuilder.Marker marker = builder.mark();
    if (builder.eof() || builder.getTokenType() == null) {
      marker.drop();
      break;
    } else {
      final IElementType token = builder.getTokenType();
      if (token == MMTokens.TOPIC_LEVEL || token == MMTokens.EXTRA_TYPE) {
        marker.rollbackTo();
        return true;
      } else if (token == MMTokens.EXTRA_BODY || token == MMTokens.WHITE_SPACE) {
        if (dataFound && token == MMTokens.EXTRA_BODY) {
          builder.advanceLexer();
          marker.done(MMTokens.UNKNOWN);
          break;
        } else {
          builder.advanceLexer();
          marker.done(token);
          dataFound = dataFound || token == MMTokens.EXTRA_BODY;
        }
      } else {
        marker.done(MMTokens.UNKNOWN);
        break;
      }
    }
  }
  return false;
}
 
Example 6
/**
 * @param exact if true then positions should be equal;
 *              else delegate should be behind, not including exactly all foreign (skipped) or whitespace tokens
 */
private void synchronizePositions(boolean exact) {
    final PsiBuilder delegate = getDelegate();

    if (myLexPosition >= myShrunkSequenceSize || delegate.eof()) {
        myLexPosition = myShrunkSequenceSize;
        while (!delegate.eof()) {
            delegate.advanceLexer();
        }
        return;
    }

    if (delegate.getCurrentOffset() > myShrunkSequence.get(myLexPosition).realStart) {
        LOG.debug("delegate is ahead of my builder!");
        return;
    }

    final int keepUpPosition = getKeepUpPosition(exact);

    while (!delegate.eof()) {
        final int delegatePosition = delegate.getCurrentOffset();

        if (delegatePosition < keepUpPosition) {
            delegate.advanceLexer();
        } else {
            break;
        }
    }
}
 
Example 7
@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
    PsiBuilder.Marker marker = builder.mark();
    while (!builder.eof()) {
        builder.advanceLexer();
    }
    marker.done(root);
    return builder.getTreeBuilt();
}
 
Example 8
public static boolean toplevel_recover_debug(PsiBuilder builder, int level) {
    IElementType typ = builder.getTokenType();
    final boolean res = !(TOPLEVEL_RECOVER_TYPES.contains(typ) || builder.eof());
    System.out.println(
        "toplevel_recover_debug: res: " + res
          + "; offset: " + builder.getCurrentOffset()
          + "; type: " + typ
          + "; text: '" + builder.getTokenText() + "'");
    return res;
}
 
Example 9
/**
 * External rule used to determine if the current lexer position is
 * in an indent. Semantically, this returns true unless the previous
 * character was a newline and the current one is not whitespace.
 * Via recoverWhile, this rule will be used to consume input until
 * it reaches an unindented token, presumably a top-level element
 * of some sort, and resume at that time. It's quite convenient to just
 * resume parsing once we encounter a new top-level element.
 */
public static boolean inIndentRecover(@NotNull PsiBuilder builder, int level) {
    if (builder.eof()) return false;
    final int offset = builder.getCurrentOffset();
    if (offset == 0) return false;
    final CharSequence text = builder.getOriginalText();
    return !(
        text.charAt(offset - 1) == '\n'
            && !Character.isWhitespace(text.charAt(offset)));
}
 
Example 10
public static boolean indented(@NotNull PsiBuilder builder, int level, Parser geq) {
    if (!(builder instanceof Builder)) return false;
    PsiParser wrapper = ((Builder) builder).parser;
    if (!(wrapper instanceof HaskellParserWrapper)) return false;
    if (builder.eof()) return false;

    // IElementType currtok = builder.getTokenType();
    Pair<Integer, IElementType> prevtok = previousElem(builder);
    if (prevtok == null) return true;

    int offs = builder.getCurrentOffset();
    int line = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs);
    int prevline = StringUtil.offsetToLineNumber(builder.getOriginalText(), offs + prevtok.getFirst());
    if (prevline == line) return true;

    int thisLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), line, 0);
    int prevLineStart = StringUtil.lineColToOffset(builder.getOriginalText(), prevline, 0);
    // CharSequence lineStuff = builder.getOriginalText().subSequence(thisLineStart, offs);
    CharSequence prevLineStuff = builder.getOriginalText().subSequence(prevLineStart, thisLineStart - 1);
    int indentation = indentationLevel(prevLineStuff);
    int myindentation = offs - thisLineStart;
    // String tokName = builder.getTokenText();

    boolean geqVal = geq.parse(builder, level);
    if (geqVal && myindentation >= indentation ||
            !geqVal && myindentation > indentation) return true;
    return false;
}
 
Example 11
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();

	while(!builder.eof())
	{
		PreprocessorParsing.parse(builder);
	}

	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 12
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	ExpressionParsing.parse(new CSharpBuilderWrapper(builder, languageVersion), ModifierSet.EMPTY);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 13
@Nonnull
@Override
public ASTNode parse(@Nonnull IElementType elementType, @Nonnull PsiBuilder builder, @Nonnull LanguageVersion languageVersion)
{
	PsiBuilder.Marker mark = builder.mark();
	SharedParsingHelpers.parseType(new CSharpBuilderWrapper(builder, languageVersion), SharedParsingHelpers.VAR_SUPPORT);
	while(!builder.eof())
	{
		builder.error("Unexpected token");
		builder.advanceLexer();
	}
	mark.done(elementType);
	return builder.getTreeBuilt();
}
 
Example 14
@NotNull
public ASTNode parse(IElementType iElementType, PsiBuilder psiBuilder) {
  final PsiBuilder.Marker marker = psiBuilder.mark();
  PsiBuilder.Marker statement = psiBuilder.mark();
  IElementType statementType = null;

  while(!psiBuilder.eof()) {
    final IElementType tokenType = psiBuilder.getTokenType();

    if (tokenType == MakefileTokenTypes.SEMANTIC_WHITESPACE) {
      psiBuilder.advanceLexer();
      statement.done(statementType != null ? statementType:MakefileTokenTypes.STATEMENT);
      statementType = null;
      statement = psiBuilder.mark();
      continue;
    } else if (shouldProduceComposite(tokenType)) {
      PsiBuilder.Marker identifier = psiBuilder.mark();
      psiBuilder.advanceLexer();
      identifier.done(tokenType);
      continue;
    } else if (tokenType == MakefileTokenTypes.VAR_DEFINITION) {
      statementType = tokenType;
    } else if (tokenType == MakefileTokenTypes.IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.IDENTIFIER_END, MakefileTokenTypes.IDENTIFIER);
    }
    else if (tokenType == MakefileTokenTypes.TARGET_IDENTIFIER_START) {
      readIdentifier(psiBuilder, MakefileTokenTypes.TARGET_IDENTIFIER_END, MakefileTokenTypes.TARGET_IDENTIFIER);
      if (!psiBuilder.eof()) psiBuilder.advanceLexer();

      statementType = MakefileTokenTypes.TARGET_DECLARATION;
      continue;
    }
    psiBuilder.advanceLexer();
  }

  statement.done(MakefileTokenTypes.STATEMENT);
  marker.done(iElementType);
  return psiBuilder.getTreeBuilt();
}
 
Example 15
private static void readIdentifier(PsiBuilder psiBuilder, IElementType identifierEnd, IElementType identifierType) {
  PsiBuilder.Marker identifier = psiBuilder.mark();
  psiBuilder.advanceLexer();
  IElementType currentType;

  while(!psiBuilder.eof() && (currentType = psiBuilder.getTokenType()) != identifierEnd) {
    PsiBuilder.Marker marker = shouldProduceComposite(currentType) ? psiBuilder.mark() : null;
    psiBuilder.advanceLexer();
    if (marker != null) marker.done(currentType);
  }
  identifier.done(identifierType);
}
 
Example 16
@NotNull
@Override
public ASTNode parse(IElementType root, PsiBuilder builder) {
	ProgressIndicatorProvider.checkCanceled();

	TokenSource source = new PSITokenSource(builder);
	TokenStream tokens = new CommonTokenStream(source);
	parser.setTokenStream(tokens);
	parser.setErrorHandler(new ErrorStrategyAdaptor()); // tweaks missing tokens
	parser.removeErrorListeners();
	parser.addErrorListener(new SyntaxErrorListener()); // trap errors
	ParseTree parseTree = null;
	PsiBuilder.Marker rollbackMarker = builder.mark();
	try {
		parseTree = parse(parser, root);
	}
	finally {
		rollbackMarker.rollbackTo();
	}

	// Now convert ANTLR parser tree to PSI tree by mimicking subtree
	// enter/exit with mark/done calls. I *think* this creates their parse
	// tree (AST as they call it) when you call {@link PsiBuilder#getTreeBuilt}
	ANTLRParseTreeToPSIConverter listener = createListener(parser, root, builder);
	PsiBuilder.Marker rootMarker = builder.mark();
	ParseTreeWalker.DEFAULT.walk(listener, parseTree);
	while (!builder.eof()) {
		ProgressIndicatorProvider.checkCanceled();
		builder.advanceLexer();
	}
	// NOTE: parse tree returned from parse will be the
	// usual ANTLR tree ANTLRParseTreeToPSIConverter will
	// convert that to the analogous jetbrains AST nodes
	// When parsing an entire file, the root IElementType
	// will be a IFileElementType.
	//
	// When trying to rename IDs and so on, you get a
	// dummy root and a type arg identifier IElementType.
	// This results in a weird tree that has for example
	// (ID (expr (primary ID))) with the ID IElementType
	// as a subtree root as well as the appropriate leaf
	// all the way at the bottom.  The dummy ID root is a
	// CompositeElement and created by
	// ParserDefinition.createElement() despite having
	// being TokenIElementType.
	rootMarker.done(root);
	return builder.getTreeBuilt(); // calls the ASTFactory.createComposite() etc...
}
 
Example 17
Source Project: netbeans-mmd-plugin   File: MMPsiParser.java    License: Apache License 2.0 4 votes vote down vote up
private int recursiveParseTopic(@Nonnull final PsiBuilder builder, final int level) {

    while (!builder.eof()) {
      final PsiBuilder.Marker marker = builder.mark();

      final IElementType token = builder.getTokenType();

      if (token == null) {
        marker.drop();
      } else {
        if (token == MMTokens.TOPIC_LEVEL) {
          final PsiBuilder.Marker levelMarker = builder.mark();
          levelMarker.done(token);

          final int theTopicLevel = ModelUtils.calcCharsOnStart('#', builder.getTokenText());
          if (theTopicLevel <= 1) {
            marker.done(MMTokens.UNKNOWN);
          } else {
            if (theTopicLevel <= level) {
              marker.rollbackTo();
              return theTopicLevel;
            } else {
              builder.advanceLexer();
              final int parsedTopicLevel = recursiveParseTopic(builder, theTopicLevel);
              marker.done(MMTokens.TOPIC);
              if (parsedTopicLevel < theTopicLevel) {
                return parsedTopicLevel;
              }
              if (parsedTopicLevel == theTopicLevel) {
                continue;
              }
            }
          }
        } else if (token == MMTokens.TOPIC_TITLE || token == MMTokens.CODE_SNIPPET_BODY || token == MMTokens.CODE_SNIPPET_END || token == MMTokens.CODE_SNIPPET_START || token == MMTokens.ATTRIBUTES) {
          marker.done(token);
        } else if (token == MMTokens.EXTRA_TYPE) {
          try {
            if (parseExtraBlock(builder)) {
              continue;
            }
          } finally {
            marker.done(MMTokens.EXTRA_DATA);
          }
        } else if (token == MMTokens.WHITE_SPACE) {
          marker.done(token);
        } else {
          marker.done(MMTokens.UNKNOWN);
        }
      }
      builder.advanceLexer();
    }


    return level;
  }
 
Example 18
Source Project: consulo-unity3d   File: ShaderLabParser.java    License: Apache License 2.0 4 votes vote down vote up
public static void parseElementsInBraces(@Nonnull PsiBuilder builder, IElementType open, IElementType close, IElementType valid)
{
	if(builder.getTokenType() == open)
	{
		builder.advanceLexer();

		while(!builder.eof())
		{
			IElementType tokenType = builder.getTokenType();
			if(tokenType == close)
			{
				break;
			}

			if(valid != null)
			{
				if(tokenType == valid)
				{
					builder.advanceLexer();
				}
				else
				{
					doneError(builder, "Unexpected token");
				}
			}
			else
			{
				builder.advanceLexer();
			}

			if(builder.getTokenType() == ShaderLabTokens.COMMA)
			{
				builder.advanceLexer();
			}
			else if(builder.getTokenType() != close)
			{
				doneError(builder, "Unexpected token");
			}
		}
		expectWithError(builder, close, "Unexpected token");
	}
}
 
Example 19
public static boolean anyNonSemi(@NotNull PsiBuilder builder, int level) {
    return
        !builder.eof() && (
            builder.getTokenType() == HaskellTypes.SEMICOLON
              || builder.getTokenType() == HaskellTypes.WHITESPACESEMITOK);
}