org.netbeans.api.lexer.Token Java Examples

private FileObject getTagFile(TokenSequence<?> tokenSequence, JspSyntaxSupport jspSup) {
    Token token = tokenSequence.token();
    if (token.id() == JspTokenId.TAG) {
        String image = token.text().toString().trim();
        if (image.startsWith("<")) {                                 // NOI18N
            image = image.substring(1).trim();
        }
        if (!image.startsWith("jsp:") && image.indexOf(':') != -1) {  // NOI18N
            List l = jspSup.getTags(image);
            if (l.size() == 1) {
                TagLibraryInfo libInfo = ((TagInfo) l.get(0)).getTagLibrary();
                if (libInfo != null) {
                    TagFileInfo fileInfo = libInfo.getTagFile(getTagName(image));
                    if (fileInfo != null) {
                        return JspUtils.getFileObject(jspSup.getDocument(),
                                fileInfo.getPath());
                    }
                }
            }
        }
    }
    return null;
}
 

private Token<JShellToken> createContinuationTokenForState(S s) {
    switch (s) {
        case INITIAL:
            return tokenFactory.createToken(JShellToken.WHITESPACE);
        case PROMPT_INPUT:
            return blockToken(JShellToken.JAVA);
        case PROMPT_MESSAGE:
            return blockToken(JShellToken.MESSAGE_TEXT);
        case MESSAGE:
            return blockToken(JShellToken.MESSAGE_TEXT);
        case JAVA:
            return blockToken(JShellToken.JAVA);
        case COMMAND:
            return blockToken(JShellToken.OUTPUT);
    }
    return null;
}
 

protected Token createToken(TokenId id, int index, int length) {
    String sampleText = null;
    if (Compatibility.charSequenceExists()) {
        SampleTextMatcher matcher = id.getSampleTextMatcher();
        if (matcher != null) {
            /* The recognizedText would not be a string
             * in the normal applications. It would be
             * a custom char sequence reused for every
             * recognized token. Here it's string
             * to simplify the code.
             */
            CharSequence recognizedText
                = (CharSequence)(Object)getDocumentText(index, length); // 1.3 compilability 
            sampleText = matcher.match(recognizedText);
        }
    }

    Token token;
    if (sampleText != null) {
        token = new DemoSampleToken(id, sampleText);
    } else {
        token = new DemoToken(this, id, index, length);
    }

    return token;
}
 

@Override
public <T extends Set<OffsetRange>> NodeVisitor<T> getInstantRenamerVisitor(EditorFeatureContext context, T result) {
    TokenSequence<CssTokenId> tokenSequence = context.getTokenSequence();
    int diff = tokenSequence.move(context.getCaretOffset());
    if (diff > 0 && tokenSequence.moveNext() || diff == 0 && tokenSequence.movePrevious()) {
        Token<CssTokenId> token = tokenSequence.token();
        final CharSequence elementName = token.text();
        return new NodeVisitor<T>(result) {
            @Override
            public boolean visit(Node node) {
                switch (node.type()) {
                    case cp_mixin_name:
                    case cp_variable:
                        if (LexerUtils.equals(elementName, node.image(), false, false)) {
                        OffsetRange range = new OffsetRange(node.from(), node.to());
                        getResult().add(range);
                        break;
                    }
                }
                return false;
            }
        };

    }
    return null;
}
 

private boolean testCreateAnnotation(TokenHierarchy hi, TokenSequence ts, ASTItem item, LanguagesAnnotation la) throws BadLocationException {
    if (ts.language () == null)
        throw new NullPointerException ("ts.language()==null");
    if (ts.language ().mimeType () == null)
        throw new NullPointerException ("TokenSequence.mimeType==null");
    if (ts.language().mimeType().equals(item.getMimeType())) {
            Token t = ts.token();
            if (t == null) throw new NullPointerException ();
            Position position = doc.createPosition(t.offset(hi));
            la.setPosition (position);
            doc.addAnnotation(position, t.length(), la);
            return true;
        } else {
            ts = ts.embedded();
            if(ts == null) {
                return false;
            } else {
                return ts.moveNext() ? testCreateAnnotation(hi, ts, item, la) : false;
            }
        }
}
 

private static CompletionContext findContext(TokenSequence<? extends TokenId> tokenSequence) {
    CompletionContext result = CompletionContext.ALL;
    do {
        Token<? extends TokenId> token = tokenSequence.token();
        if (token == null) {
            break;
        }
        TokenId tokenId = token.id();
        if (TwigBlockTokenId.T_TWIG_OTHER.equals(tokenId) || TwigVariableTokenId.T_TWIG_OTHER.equals(tokenId)) {
            result = CompletionContext.NONE;
            break;
        } else if (acceptTokenChains(tokenSequence, FILTER_TOKEN_CHAINS, true)) {
            result = CompletionContext.FILTER;
            break;
        } else if (tokenId instanceof TwigBlockTokenId) {
            result = CompletionContext.BLOCK;
            break;
        } else if (tokenId instanceof TwigVariableTokenId) {
            result = CompletionContext.VARIABLE;
            break;
        }
    } while (tokenSequence.movePrevious());
    return result;
}
 

/**
 * Search forwards in the token sequence until a matching closing token is
 * found so keeps track of nested pairs of up-down eg (()) is ignored if
 * we're searching for a )
 *
 * @param ts the TokenSequence set to the position after an up
 * @param up the opening token eg { or [
 * @param down the closing token eg } or ]
 * @return the Range of closing token in our case 1 char
 */
public static OffsetRange findBwd(TokenSequence<? extends SQLTokenId> ts, int up, int down) {
    int balance = 0;

    while (ts.movePrevious()) {
        Token<? extends SQLTokenId> token = ts.token();

        if (token.id().ordinal() == up) {
            if (balance == 0) {
                return new OffsetRange(ts.offset(), ts.offset() + token.length());
            }

            balance++;
        } else if (token.id().ordinal() == down) {
            balance--;
        }
    }

    return OffsetRange.NONE;
}
 

private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) {
    if (seq == null) {
        sb.append("Inactive TokenHierarchy"); //NOI18N
    } else {
        for(seq.moveStart(); seq.moveNext(); ) {
            TokenSequence<?> emSeq = seq.embedded();
            if (emSeq != null) {
                dumpSequence(emSeq, sb);
            } else {
                Token<?> token = seq.token();
                sb.append("<"); //NOI18N
                sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N
                sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N
                sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N
                sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N
                sb.append(tokenText(token));
                sb.append("'\n"); //NOI18N
            }
        }
    }
}
 

public LanguageEmbedding<?> findLanguageEmbedding(
Token<?> token, LanguagePath languagePath, InputAttributes inputAttributes) {
    if ("text/x-simple-plain".equals(languagePath.mimePath()) && token.id().name().equals("WORD")) {
        return LanguageEmbedding.create(TestCharTokenId.language(), 0, 0);
    } else {
        return null;
    }
}
 

/**
 * Finds the first previous token that is not in the ignore list
 * @param ts
 * @param ignores
 * @return 
 */
public static Token<? extends JsTokenId> findPrevious(TokenSequence<? extends JsTokenId> ts, List<JsTokenId> ignores) {
    if (ignores.contains(ts.token().id())) {
        while (ts.movePrevious() && ignores.contains(ts.token().id())) {}
    }
    return ts.token();
}
 

public static Token<?extends PHPTokenId> getToken(BaseDocument doc, int offset) {
    TokenSequence<?extends PHPTokenId> ts = getPositionedSequence(doc, offset);

    if (ts != null) {
        return ts.token();
    }

    return null;
}
 

void dumpTokenStream (TokenSequence ts){
        // System.out.println("#############################################");
        
        while (ts.moveNext()) {
              Token t = ts.token();
//              System.out.println("-------------------------------");  
//              System.out.println("Token-ID   :" + t.id());
//              if(t.toString() != null)
//                System.out.println("Token-Name   :" + t.toString());  
            }
    }
 

private void skipWhitespace() {
    Token<XMLTokenId> t;
    
    while ((t = nextToken()) != null && t.id() == XMLTokenId.WS) {
        consume();
    }
}
 

/** Compute the balance of begin/end tokens on the line. */
public static int getLineBalance(BaseDocument doc, int offset, TokenId up, TokenId down) {
    try {
        int begin = Utilities.getRowStart(doc, offset);
        int end = Utilities.getRowEnd(doc, offset);

        TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, begin);
        if (ts == null) {
            return 0;
        }

        ts.move(begin);

        if (!ts.moveNext()) {
            return 0;
        }

        int balance = 0;

        do {
            Token<GroovyTokenId> token = ts.token();
            TokenId id = token.id();

            if (id == up) {
                balance++;
            } else if (id == down) {
                balance--;
            }
        } while (ts.moveNext() && (ts.offset() <= end));

        return balance;
    } catch (BadLocationException ble) {
        Exceptions.printStackTrace(ble);

        return 0;
    }
}
 

public static Collection<String> getUsedFileInDefine(final Snapshot shanpshot, final int offset) {
    TokenSequence<? extends JsTokenId> ts = LexUtilities.getJsTokenSequence(shanpshot, offset);
    if (ts == null) {
        return Collections.emptyList();
    }
    ts.move(0);
    if (!ts.moveNext()) {
        return Collections.emptyList();
    }
    Token<? extends JsTokenId> token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.IDENTIFIER));
    while (token.id() == JsTokenId.IDENTIFIER && !DEFINE.equals(token.text().toString()) && ts.moveNext()) {
        token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.IDENTIFIER));
    }
    if (token.id() == JsTokenId.IDENTIFIER && DEFINE.equals(token.text().toString())) {
        // we are probably found the define method
        List<String> paths = new ArrayList<String>();
        token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.BRACKET_LEFT_BRACKET, JsTokenId.KEYWORD_FUNCTION, JsTokenId.BRACKET_LEFT_CURLY, JsTokenId.BRACKET_RIGHT_PAREN));
        if (token.id() == JsTokenId.BRACKET_LEFT_BRACKET) {
            do {
                token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.STRING, JsTokenId.OPERATOR_COMMA, JsTokenId.BRACKET_RIGHT_PAREN));
                if (token.id() == JsTokenId.STRING) {
                    paths.add(token.text().toString());
                }
            } while ((token.id() != JsTokenId.BRACKET_RIGHT_PAREN && token.id() != JsTokenId.OPERATOR_SEMICOLON && !token.id().isKeyword()) && ts.moveNext());
            return paths;
        }
    }
    return Collections.emptyList();
}
 

private static boolean skipWhitespaces(TokenSequence<PHPTokenId> tokenSequence) {
    Token<PHPTokenId> token = tokenSequence.token();
    while (token != null && isWhiteSpace(token)) {
        boolean retval = tokenSequence.movePrevious();
        token = tokenSequence.token();
        if (!retval) {
            return false;
        }
    }
    return true;
}
 

@Override
public List<Embedding> getEmbeddings(Snapshot snapshot) {
    TokenHierarchy<?> th = snapshot.getTokenHierarchy();
    TokenSequence<JspTokenId> sequence = th.tokenSequence(JspTokenId.language());
    List<Embedding> embeddings = new ArrayList<Embedding>();
    sequence.moveStart();
    boolean inAttributeValueWithEL = false;
    while (sequence.moveNext()) {
        Token t = sequence.token();
        if (t.id() == JspTokenId.ATTR_VALUE && t.length() == 1 && 
                (t.text().charAt(0) == '"' || t.text().charAt(0) == '\'')) {
            //a quote before/after attribute value with EL inside
            inAttributeValueWithEL = !inAttributeValueWithEL;
        }
        if (t.id() == JspTokenId.EL) {
            embeddings.add(snapshot.create(sequence.offset(), t.length(), "text/x-el")); //NOI18N
            //XXX hack - there's a need to distinguish between ELs inside or outside of attribute values
            if(inAttributeValueWithEL) {
                embeddings.add(snapshot.create(ATTRIBUTE_EL_MARKER, "text/x-el")); //NOI18N
            }
            
            // just to separate expressions for easier handling in EL parser
            embeddings.add(snapshot.create(Constants.LANGUAGE_SNIPPET_SEPARATOR, "text/x-el")); //NOI18N
         
        }
    }
    if (embeddings.isEmpty()) {
        return Collections.emptyList();
    } else {
        return Collections.singletonList(Embedding.create(embeddings));
    }
}
 

/**
 * Checks if {@code token} represents a comment.
 * 
 * @param token the {@link Token} to check
 * @return {@code true} if {@code token} represents a line comment, block
 *          comment; {@code false} otherwise or javadoc.
 */
private boolean isComment(final Token<JavaTokenId> token) {
    switch (token.id()) {
        case LINE_COMMENT:
        case BLOCK_COMMENT:
            return true;
        case JAVADOC_COMMENT:
        default:
            return false;
    }
}
 

private Token<JavaTokenId> keywordOrIdentifier(JavaTokenId keywordId, int c) {
    // Check whether the given char is non-ident and if so then return keyword
    if (c == EOF || !Character.isJavaIdentifierPart(c = translateSurrogates(c))) {
        // For surrogate 2 chars must be backed up
        backup((c >= Character.MIN_SUPPLEMENTARY_CODE_POINT) ? 2 : 1);
        return token(keywordId);
    } else // c is identifier part
        return finishIdentifier();
}
 

public static Token<?extends PHPTokenId> findPrevious(TokenSequence<?extends PHPTokenId> ts, List<PHPTokenId> ignores) {
    if (ignores.contains(ts.token().id())) {
        while (ts.movePrevious() && ignores.contains(ts.token().id())) {
        }
    }
    return ts.token();
}
 

public static <T extends TokenId> Token<T> findPrevious(JoinedTokenSequence<T> ts, List<T> ignores) {
    if (ignores.contains(ts.token().id())) {
        while (ts.movePrevious() && ignores.contains(ts.token().id())) {}
    }
    if (ts.token() == null || ignores.contains(ts.token().id())) {
        return null;
    }
    return ts.token();
}
 

public void testIsInaccessibleOutsideOuterClassForStaticMethodOfNestedClass() throws Exception {
    boolean[] wasResolved = new boolean[1];
    Collection<Token> changePoints = performTest("package test; public class Test { static class PPP { public static void method() {}}}", 73, wasResolved);

    assertNull(changePoints);
    assertTrue(wasResolved[0]);
}
 

public static boolean isWhiteSpaceFirst(Token<JavadocTokenId> token) {
    if (token == null || token.id() != JavadocTokenId.OTHER_TEXT || token.length() < 1) {
        return false;
    }

    CharSequence text = token.text();
    char c = text.charAt(0);
    return c == ' ' || c == '\t';
}
 

/** Compute the balance of begin/end tokens on the line.
 * @param doc the document
 * @param offset The offset somewhere on the line
 * @param upToOffset If true, only compute the line balance up to the given offset (inclusive),
 *   and if false compute the balance for the whole line
 */
public static int getBeginEndLineBalance(BaseDocument doc, int offset, boolean upToOffset) {
    try {
        int begin = Utilities.getRowStart(doc, offset);
        int end = upToOffset ? offset : Utilities.getRowEnd(doc, offset);

        TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, begin);
        if (ts == null) {
            return 0;
        }

        ts.move(begin);

        if (!ts.moveNext()) {
            return 0;
        }

        int balance = 0;

        do {
            Token<GroovyTokenId> token = ts.token();
            TokenId id = token.id();

            if (isBeginToken(id, doc, ts)) {
                balance++;
            } else if (id == GroovyTokenId.RBRACE) {
                balance--;
            }
        } while (ts.moveNext() && (ts.offset() <= end));

        return balance;
    } catch (BadLocationException ble) {
        Exceptions.printStackTrace(ble);

        return 0;
    }
}
 

/**
     * Fetch next token from underlying javacc tokenmanager.
     * <BR>The intId of the token that was found can be set
     * into the given tokenData parameter
     * by <CODE>TokenData.setTokenIntId()</CODE> in case there was
     * a valid token found.
     * <P>Token length of the fetched token can be set into tokenData
     * by <CODE>TokenData.setTokenLength()</CODE>.
     * If the token intId or length is not assigned in <CODE>fetchToken()</CODE>
     * it must be assigned later during either
     * {@link #ordinaryToken(OrdinaryTokenData)}
     * or {@link #extendedToken(ExtendedTokenData)} depending
     * which of these two gets called.
     * @param tokenData mutable info about the token being fetched.
     * @return true if a valid token was found or false
     *  if there are no more tokens on the input (in which case a call
     *  to <CODE>TokenData.setTokenIntId()</CODE> is not necessary).
     */
    protected boolean fetchToken(TokenData tokenData) {
        try {
            // Get javacc token from tokenmanager
            org.netbeans.spi.lexer.javacc.Token javaccToken = tokenManager.getNextToken();
            if (javaccToken != null) {
                int tokenKind = javaccToken.kind;
                tokenData.setTokenIntId(tokenKind);
                tokenData.setTokenLength(tokenData.getDefaultTokenLength());
                return (tokenKind != CalcConstants.EOF); // EOF presents no characters
                
            } else { // javaccToken is null
                return false;  // no more tokens from tokenManager
            }
                
        } catch (TokenMgrError e) {
            if (e.getErrorCode() == TokenMgrError.LEXICAL_ERROR) {
                if (tokenData.inExtendedToken()) {
                    switch (tokenData.getExtendedTokenIntId()) {
                        case CalcConstants.INCOMPLETE_ML_COMMENT:
                            // This should only happen at the end of input
                            tokenData.setTokenIntId(CalcConstants.EOF);
                            // Lookahead will be non-zero (for no chars the lexical
                            // error would not be thrown)
                            tokenData.setTokenLength(tokenData.getTextLookahead());
                            return true; // there are chars -> valid token exists
                            
                    }
                }
                
                // Fallback for other ERRORS
//                System.out.println("msg=" + e.getMessage());
                throw new IllegalStateException("Internal lexer error");
                
            } else { // non-lexical type of error
                throw e;
            }
        }
    }
 

public void testNonPrivateClassWithPrivateConstructor() throws Exception {
    boolean[] wasResolved = new boolean[1];
    Collection<Token> changePoints = performTest("package test; public class Test{public void run() {Test1 t = new Test1();} static class Test1{private Test1(){}}", 112 - 59, wasResolved);
    
    assertNull(changePoints);
    assertTrue(wasResolved[0]);
}
 

private Token<DTDTokenId> nextProcessingContent() {
    int ch;
    boolean white = false;
    int whiteStart = -1;
    
    while ((ch = input.read()) != LexerInput.EOF) {
        if (ch == '?') {
            if (whiteStart > 0) {
                return tokenFactory.createToken(DTDTokenId.PI_CONTENT, whiteStart);
            }
            if (input.readLength() > 1) {
                input.backup(1);
                return tokenFactory.createToken(DTDTokenId.PI_CONTENT);
            }
            ch = input.read();
            if (ch == '>') {
                return tokenFactory.createToken(DTDTokenId.SYMBOL);
            }
            // exit the processing content
            setState(ISI_INIT);
            return tokenFactory.createToken(DTDTokenId.ERROR);
        }
        if (Character.isWhitespace(ch)) {
            if (whiteStart == -1) {
                whiteStart = input.readLength();
            }
            if (input.readLength() == 1) {
                white = true;
            }
        } else if (white) {
            // report whitespace & continue lexing in the same state:
            return tokenFactory.createToken(DTDTokenId.WS);
        }
    }
    return tokenFactory.createToken(DTDTokenId.PI_CONTENT);
}
 

@Override
public Void visitUses(UsesTree tree, Void p) {
    tl.moveToOffset(sourcePositions.getStartPosition(info.getCompilationUnit(), tree));
    Token t = firstIdentifierToken("uses"); //NOI18N
    if (t != null) {
        contextKeywords.add(t);
    }
    return super.visitUses(tree, p);
}
 

@Override
public Token<GspTokenId> nextToken() {
    final GspTokenId tokenId = nextTokenId();
    if (tokenId == null) {
        return null; // EOF
    }
    return tokenFactory.createToken(tokenId);
}
 

private OffsetRange isInObjectValueOfProperty(String propertyName, TokenSequence<? extends JsTokenId> ts, int caretOffset) {
    // e.g. check if "demo" is inside the object which is value of "components" property
    // components: { propA: 'a', propB: 'demo' }
    ts.move(caretOffset);
    if (ts.moveNext()) {
        JsTokenId id = ts.token().id();
        if (id == JsTokenId.STRING) {
            OffsetRange result = new OffsetRange(ts.offset(), ts.offset() + ts.token().length());
            ts.movePrevious();
            Token<? extends JsTokenId> previous = LexUtilities.findPrevious(ts,
                    Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL,
                            JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT,
                            JsTokenId.STRING_BEGIN, JsTokenId.STRING, JsTokenId.STRING_END,
                            JsTokenId.OPERATOR_COMMA, JsTokenId.OPERATOR_COLON, JsTokenId.IDENTIFIER));
            if (previous != null && previous.id() == JsTokenId.BRACKET_LEFT_CURLY && ts.movePrevious()) {
                previous = LexUtilities.findPrevious(ts, Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL, JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT));
                if (previous != null && previous.id() == JsTokenId.OPERATOR_COLON && ts.movePrevious()) {
                    previous = LexUtilities.findPrevious(ts, Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL, JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT));
                    if (previous != null && previous.id() == JsTokenId.IDENTIFIER
                            && propertyName.equals(previous.text().toString())) {
                        return result;
                    }
                }
            }
        }
    }
    return null;
}