Java Code Examples for org.antlr.v4.runtime.TokenStream#get()

@NotNull
private String getText(TokenStream tokens, Interval interval)
{
    int start = interval.a;
    int stop = interval.b;
    if (start < 0 || stop < 0)
        return "";
    
    if (stop >= tokens.size())
        stop = tokens.size() - 1;
    
    StringBuilder buf = new StringBuilder();
    for (int i = start; i <= stop; i++)
    {
        Token t = tokens.get(i);
        if (t.getType() == Token.EOF)
            break;
        buf.append(t.getText());
        if (i != stop)
        {
            buf.append(" ");
        }
    }
    return buf.toString();
}
 

/**
 "If an operator has whitespace on the right side only, it is treated as a
 postfix unary operator. As an example, the ++ operator in a++ b is treated
 as a postfix unary operator."
 "If an operator has no whitespace on the left but is followed immediately
 by a dot (.), it is treated as a postfix unary operator. As an example,
 the ++ operator in a++.b is treated as a postfix unary operator (a++ .b
 rather than a ++ .b)."
 */
public static boolean isPostfixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result =
		!prevIsWS && nextIsWS ||
		!prevIsWS && nextToken.getType()==SwiftParser.DOT;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPostfixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

public Token addDecisionEventHighlighter(PreviewState previewState, MarkupModel markupModel,
                                         DecisionEventInfo info, Color errorStripeColor,
                                         EffectType effectType) {
	TokenStream tokens = previewState.parsingResult.parser.getInputStream();
	Token startToken = tokens.get(info.startIndex);
	Token stopToken = tokens.get(info.stopIndex);
	TextAttributes textAttributes =
		new TextAttributes(JBColor.BLACK, JBColor.WHITE, errorStripeColor,
		                   effectType, Font.PLAIN);
	textAttributes.setErrorStripeColor(errorStripeColor);
	final RangeHighlighter rangeHighlighter =
		markupModel.addRangeHighlighter(
			startToken.getStartIndex(), stopToken.getStopIndex()+1,
			HighlighterLayer.ADDITIONAL_SYNTAX, textAttributes,
			HighlighterTargetArea.EXACT_RANGE);
	rangeHighlighter.putUserData(DECISION_EVENT_INFO_KEY, info);
	rangeHighlighter.setErrorStripeMarkColor(errorStripeColor);
	return startToken;
}
 

public static boolean isOpNext(TokenStream tokens) {
	int start = tokens.index();
	Token lt = tokens.get(start);
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;
	System.out.printf("isOpNext: i=%d t='%s'", start, lt.getText());
	System.out.printf(", op='%s'\n", tokens.getText(Interval.of(start,stop)));
	return true;
}
 

/**
 "If an operator has whitespace around both sides or around neither side,
 it is treated as a binary operator. As an example, the + operator in a+b
 and a + b is treated as a binary operator."
 */
public static boolean isBinaryOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && nextIsWS || (!prevIsWS && !nextIsWS);
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isBinaryOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

/**
 "If an operator has whitespace on the left side only, it is treated as a
 prefix unary operator. As an example, the ++ operator in a ++b is treated
 as a prefix unary operator."
*/
public static boolean isPrefixOp(TokenStream tokens) {
	int stop = getLastOpTokenIndex(tokens);
	if ( stop==-1 ) return false;

	int start = tokens.index();
	Token prevToken = tokens.get(start-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(stop+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	boolean result = prevIsWS && !nextIsWS;
	String text = tokens.getText(Interval.of(start, stop));
	//System.out.println("isPrefixOp: '"+prevToken+"','"+text+"','"+nextToken+"' is "+result);
	return result;
}
 

/** Return two booleans packed into lowest 2 bits for left (high) and right (low)
 *  whitespace.
 */
public static int getLeftRightWS(TokenStream tokens, ParserRuleContext ctx) {
	int left = ctx.start.getTokenIndex();
	int right = ctx.stop.getTokenIndex();
	Token prevToken = tokens.get(left-1); // includes hidden-channel tokens
	Token nextToken = tokens.get(right+1);
	boolean prevIsWS = isLeftOperatorWS(prevToken);
	boolean nextIsWS = isRightOperatorWS(nextToken);
	int b = (prevIsWS ? 1 : 0) << 1 | (nextIsWS ? 1 : 0) ;
	return b;
}
 

public static Token getTokenForCharIndex(TokenStream tokens, int charIndex) {
	for (int i=0; i<tokens.size(); i++) {
		Token t = tokens.get(i);
		if ( charIndex>=t.getStartIndex() && charIndex<=t.getStopIndex() ) {
			return t;
		}
	}
	return null;
}
 

private void extractSelection(@NotNull PsiFile psiFile, Editor editor, SelectionModel selectionModel) {
	Document doc = editor.getDocument();
	String grammarText = psiFile.getText();
	ParsingResult results = ParsingUtils.parseANTLRGrammar(grammarText);
	final Parser parser = results.parser;
	final ParserRuleContext tree = (ParserRuleContext) results.tree;
	TokenStream tokens = parser.getTokenStream();

	int selStart = selectionModel.getSelectionStart();
	int selStop = selectionModel.getSelectionEnd() - 1; // I'm inclusive and they are exclusive for end offset

	// find appropriate tokens for bounds, don't include WS
	Token start = RefactorUtils.getTokenForCharIndex(tokens, selStart);
	Token stop = RefactorUtils.getTokenForCharIndex(tokens, selStop);
	if ( start==null || stop==null ) {
		return;
	}
	if ( start.getType()==ANTLRv4Lexer.WS ) {
		start = tokens.get(start.getTokenIndex() + 1);
	}
	if ( stop.getType()==ANTLRv4Lexer.WS ) {
		stop = tokens.get(stop.getTokenIndex() - 1);
	}

	selectionModel.setSelection(start.getStartIndex(), stop.getStopIndex() + 1);
	final Project project = psiFile.getProject();
	final ChooseExtractedRuleName nameChooser = new ChooseExtractedRuleName(project);
	nameChooser.show();
	if ( nameChooser.ruleName==null ) return;

	// make new rule string
	final String ruleText = selectionModel.getSelectedText();

	final int insertionPoint = RefactorUtils.getCharIndexOfNextRuleStart(tree, start.getTokenIndex());
	final String newRule = "\n" + nameChooser.ruleName + " : " + ruleText + " ;" + "\n";

	runWriteCommandAction(project, () -> {
		// do all as one operation.
		if ( insertionPoint >= doc.getTextLength() ) {
			doc.insertString(doc.getTextLength(), newRule);
		} else {
			doc.insertString(insertionPoint, newRule);
		}
		doc.replaceString(selectionModel.getSelectionStart(), selectionModel.getSelectionEnd(), nameChooser.ruleName);
	});

	// TODO: only allow selection of fully-formed syntactic entity.
	// E.g., "A (',' A" is invalid grammatically as a rule.
}
 

public void setProfilerData(PreviewState previewState, long parseTime_ns) {
	this.previewState = previewState;
	Parser parser = previewState.parsingResult.parser;
	ParseInfo parseInfo = parser.getParseInfo();
	updateTableModelPerExpertCheckBox(parseInfo);
	double parseTimeMS = parseTime_ns/(1000.0*1000.0);
	// microsecond decimal precision
	NumberFormat formatter = new DecimalFormat("#.###");
	parseTimeField.setText(formatter.format(parseTimeMS));
	double predTimeMS = parseInfo.getTotalTimeInPrediction()/(1000.0*1000.0);
	predictionTimeField.setText(
		String.format("%s = %3.2f%%", formatter.format(predTimeMS), 100*(predTimeMS)/parseTimeMS)
	                           );
	TokenStream tokens = parser.getInputStream();
	int numTokens = tokens.size();
	Token lastToken = tokens.get(numTokens-1);
	int numChar = lastToken.getStopIndex();
	int numLines = lastToken.getLine();
	if ( lastToken.getType()==Token.EOF ) {
		if ( numTokens<=1 ) {
			numLines = 0;
		}
		else {
			Token secondToLastToken = tokens.get(numTokens-2);
			numLines = secondToLastToken.getLine();
		}
	}
	inputSizeField.setText(String.format("%d char, %d lines",
	                                     numChar,
	                                     numLines));
	numTokensField.setText(String.valueOf(numTokens));
	double look =
		parseInfo.getTotalSLLLookaheadOps()+
			parseInfo.getTotalLLLookaheadOps();
	lookaheadBurdenField.setText(
		String.format("%d/%d = %3.2f", (long) look, numTokens, look/numTokens)
	                            );
	double atnLook = parseInfo.getTotalATNLookaheadOps();
	cacheMissRateField.setText(
		String.format("%d/%d = %3.2f%%", (long) atnLook, (long) look, atnLook*100.0/look)
	                          );
}