com.sun.tools.javac.parser.Tokens.Token Java Examples

private CT match(TK tk, TokenKind open) {
    Token tok = advance();
    db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
    if (stack.isEmpty()) {
        return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
    }
    Token p = stack.pop();
    if (p.kind != open) {
        return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
    }
    return new CT(tk, tok);
}
 

private Comment comment(String docString) {
    StringBuilder docComment = new StringBuilder()
                               .append("/**")
                               .append(docString)
                               .append("*/");
    Scanner scanner = scanners.newScanner(docComment, true);
    scanner.nextToken();
    Token token = scanner.token();
    return token.comment(CommentStyle.JAVADOC);
}
 

void assertKind(String input, JavaTokenizer tokenizer, TokenKind kind, String expectedText) {
    Token token = tokenizer.readToken();

    if (token.kind != kind) {
        throw new AssertionError("Unexpected token kind: " + token.kind);
    }

    String actualText = input.substring(token.pos, token.endPos);

    if (!Objects.equals(actualText, expectedText)) {
        throw new AssertionError("Unexpected token text: " + actualText);
    }
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

private CT(TK tk, Token tok, String msg) {
    this.kind = tk;
    this.endPos = tok.endPos;
    this.message = msg;
    //throw new InternalError(msg); /* for debugging */
}
 

private CT(TK tk, Token tok) {
    this.kind = tk;
    this.endPos = tok.endPos;
    this.message = null;
}
 

private Token advance() {
    Token prev = current;
    scanner.nextToken();
    current = scanner.token();
    return prev;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

public void nextToken() {
    super.nextToken();
    Token tk = token();
    System.err.format("Saw token %s %n", tk.kind);
    test.numTokens++;
}
 

@Override public Token readToken() {
	Token token = super.readToken();
	prevEndPosition = ((PositionUnicodeReader)reader).pos();
	return token;
}
 

/** Lex the input and return a list of {@link RawTok}s. */
public static ImmutableList<RawTok> getTokens(
    String source, Context context, Set<TokenKind> stopTokens) {
  if (source == null) {
    return ImmutableList.of();
  }
  ScannerFactory fac = ScannerFactory.instance(context);
  char[] buffer = (source + EOF_COMMENT).toCharArray();
  Scanner scanner =
      new AccessibleScanner(fac, new CommentSavingTokenizer(fac, buffer, buffer.length));
  ImmutableList.Builder<RawTok> tokens = ImmutableList.builder();
  int end = source.length();
  int last = 0;
  do {
    scanner.nextToken();
    Token t = scanner.token();
    if (t.comments != null) {
      for (Comment c : Lists.reverse(t.comments)) {
        if (last < c.getSourcePos(0)) {
          tokens.add(new RawTok(null, null, last, c.getSourcePos(0)));
        }
        tokens.add(
            new RawTok(null, null, c.getSourcePos(0), c.getSourcePos(0) + c.getText().length()));
        last = c.getSourcePos(0) + c.getText().length();
      }
    }
    if (stopTokens.contains(t.kind)) {
      if (t.kind != TokenKind.EOF) {
        end = t.pos;
      }
      break;
    }
    if (last < t.pos) {
      tokens.add(new RawTok(null, null, last, t.pos));
    }
    tokens.add(
        new RawTok(
            t.kind == TokenKind.STRINGLITERAL ? "\"" + t.stringVal() + "\"" : null,
            t.kind,
            t.pos,
            t.endPos));
    last = t.endPos;
  } while (scanner.token().kind != TokenKind.EOF);
  if (last < end) {
    tokens.add(new RawTok(null, null, last, end));
  }
  return tokens.build();
}