Java Code Examples for org.netbeans.api.lexer.TokenSequence#moveStart()

The following examples show how to use org.netbeans.api.lexer.TokenSequence#moveStart() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: HtmlLexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void test146930() {
    TokenHierarchy th = TokenHierarchy.create("<<body>", HTMLTokenId.language());
    TokenSequence ts = th.tokenSequence();
    ts.moveStart();

    assertTrue(ts.moveNext());
    assertEquals("<", ts.token().text().toString());
    assertEquals(HTMLTokenId.TEXT, ts.token().id());

    assertTrue(ts.moveNext());
    assertEquals("<", ts.token().text().toString());
    assertEquals(HTMLTokenId.TAG_OPEN_SYMBOL, ts.token().id());

    assertTrue(ts.moveNext());
    assertEquals("body", ts.token().text().toString());
    assertEquals(HTMLTokenId.TAG_OPEN, ts.token().id());

    assertTrue(ts.moveNext());
    assertEquals(">", ts.token().text().toString());
    assertEquals(HTMLTokenId.TAG_CLOSE_SYMBOL, ts.token().id());

    assertFalse(ts.moveNext());
}
Example 2
Source File: LibraryDeclarationChecker.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
private static boolean isFunctionLibraryPrefixUsedInEL(RuleContext context, Library lib, CharSequence sourceText) {
    String libraryPrefix = NamespaceUtils.getForNs(((HtmlParserResult)context.parserResult).getNamespaces(), lib.getNamespace());
    TokenHierarchy<CharSequence> th = TokenHierarchy.create(sourceText, Language.find("text/xhtml"));
    TokenSequence<?> ts = th.tokenSequence();
    ts.moveStart();
    while(ts.moveNext()) {
        TokenSequence<ELTokenId> elts = ts.embeddedJoined(ELTokenId.language());
        if(elts != null) {
            //check the EL expression for the function library prefix usages
            elts.moveStart();
            while(elts.moveNext()) {
                if(elts.token().id() == ELTokenId.TAG_LIB_PREFIX && CharSequenceUtilities.equals(libraryPrefix, elts.token().text())) {
                    return true;
                }
            }
        }
    }

    return false;
}
Example 3
Source File: NbCss3LexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void testErrorCase1() throws Exception {
    /*
    java.lang.ArrayIndexOutOfBoundsException: Array index out of range: 4
    at java.util.Vector.get(Vector.java:694)
    at org.netbeans.modules.css.lib.nblexer.NbLexerCharStream.rewind(NbLexerCharStream.java:131)
    at org.antlr.runtime.DFA.predict(DFA.java:149)
    at org.netbeans.modules.css.lib.Css3Lexer.mNUMBER(Css3Lexer.java:7440)
     */
    String source = "padding: .5em; ";

    TokenHierarchy th = TokenHierarchy.create(source, CssTokenId.language());
    TokenSequence ts = th.tokenSequence();
    ts.moveStart();

    assertToken("padding", CssTokenId.IDENT, ts);
    assertToken(":", CssTokenId.COLON, ts);
    assertToken(" ", CssTokenId.WS, ts);
    assertToken(".5em", CssTokenId.EMS, ts);

}
Example 4
Source File: LanguageManagerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void testCacheRefreshedE() {
    TokenHierarchy th = TokenHierarchy.create("abc", TestPlainTokenId.language());
    TokenSequence tokens = th.tokenSequence();
    tokens.moveStart();
    assertEquals(true, tokens.moveNext());
    
    TokenSequence embeddedA = tokens.embedded();
    assertNotNull("There should be an embedded language", embeddedA);
    
    SimpleLanguageProvider.fireTokenLanguageChange();
    
    TokenSequence embeddedB = tokens.embedded();
    assertNotNull("There should be an embedded language", embeddedB);
    
    assertNotSame("The token language cache has not been refreshed", embeddedA, embeddedB);
}
Example 5
Source File: CustomFoldManager.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
private List<FoldMarkInfo> getMarkList(TokenSequence seq) {
    List<FoldMarkInfo> markList = null;
    
    for(seq.moveStart(); seq.moveNext(); ) {
        Token token = seq.token();
        FoldMarkInfo info;
        try {
            info = scanToken(token);
        } catch (BadLocationException e) {
            LOG.log(Level.WARNING, null, e);
            info = null;
        }

        if (info != null) {
            if (markList == null) {
                markList = new ArrayList<FoldMarkInfo>();
            }
            markList.add(info);
        }
    }

    return markList;
}
Example 6
Source File: JspLexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void testScriptletExpressionInTagAttribute_Issue176211() {
    TokenHierarchy th = TokenHierarchy.create("<jsp:xxx attr=\"<%=expr%>\"> <% %>text", JspTokenId.language());
    TokenSequence ts = th.tokenSequence();
    ts.moveStart();

    assertToken(ts, "<", JspTokenId.SYMBOL);
    assertToken(ts, "jsp:xxx", JspTokenId.TAG);
    assertToken(ts, " ", JspTokenId.WHITESPACE);
    assertToken(ts, "attr", JspTokenId.ATTRIBUTE);
    assertToken(ts, "=", JspTokenId.SYMBOL);
    assertToken(ts, "\"", JspTokenId.ATTR_VALUE);
    assertToken(ts, "<%=", JspTokenId.SYMBOL2);
    assertToken(ts, "expr", JspTokenId.SCRIPTLET);
    assertToken(ts, "%>", JspTokenId.SYMBOL2);
    assertToken(ts, "\"", JspTokenId.ATTR_VALUE);
    assertToken(ts, ">", JspTokenId.SYMBOL);
    assertToken(ts, " ", JspTokenId.TEXT);
    assertToken(ts, "<%", JspTokenId.SYMBOL2);
    assertToken(ts, " ", JspTokenId.SCRIPTLET);
    assertToken(ts, "%>", JspTokenId.SYMBOL2);
    assertToken(ts, "text", JspTokenId.TEXT);

    assertFalse(ts.moveNext());

}
Example 7
Source File: SyntaxHighlighting.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) {
    if (seq == null) {
        sb.append("Inactive TokenHierarchy"); //NOI18N
    } else {
        for(seq.moveStart(); seq.moveNext(); ) {
            TokenSequence<?> emSeq = seq.embedded();
            if (emSeq != null) {
                dumpSequence(emSeq, sb);
            } else {
                Token<?> token = seq.token();
                sb.append("<"); //NOI18N
                sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N
                sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N
                sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N
                sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N
                sb.append(tokenText(token));
                sb.append("'\n"); //NOI18N
            }
        }
    }
}
Example 8
Source File: SQLLexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
private static CharSequence dumpTokens(TokenSequence<?> seq) {
    seq.moveStart();
    StringBuilder builder = new StringBuilder();
    Token<?> token = null;
    while (seq.moveNext()) {
        if (token != null) {
            builder.append('\n');
        }
        token = seq.token();
        builder.append(token.id());
        PartType part = token.partType();
        if (part != PartType.COMPLETE) {
            builder.append(' ');
            builder.append(token.partType());
        }
        builder.append(' ');
        builder.append('\'');
        builder.append(token.text());
        builder.append('\'');
    }
    return builder;
}
Example 9
Source File: HtmlLexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void testScriptType_value() {
    TokenHierarchy th = TokenHierarchy.create("<script type=\"text/plain\">plain</script>", HTMLTokenId.language());
    TokenSequence ts = th.tokenSequence();
    ts.moveStart();

    while(ts.moveNext()) {
        Token t = ts.token();
        if(t.id() == HTMLTokenId.SCRIPT) {
            String scriptType = (String)t.getProperty(HTMLTokenId.SCRIPT_TYPE_TOKEN_PROPERTY);
            assertNotNull(scriptType);
            assertEquals("text/plain", scriptType);
            return ;
        }
    }
    
    assertTrue("Couldn't find any SCRIPT token!", false);
}
Example 10
Source File: NbCss3LexerTest.java    From netbeans with Apache License 2.0 6 votes vote down vote up 
public void testIssue238864() throws Exception {
//        LexerTestUtilities.checkTokenDump(this, "testfiles/scss/large_empty.scss.txt",
//                CssTokenId.language());
        int LINES = 10 * 1000;
        
        StringBuilder source = new StringBuilder();
        for(int i = 0; i < LINES; i++) {
            source.append('\n');
        }
        
        TokenHierarchy th = TokenHierarchy.create(source, CssTokenId.language());
        TokenSequence ts = th.tokenSequence();
        ts.moveStart();
        
        assertTrue(ts.moveNext());
        Token token = ts.token();
        
        assertEquals(token.id(), CssTokenId.NL);
        assertEquals(ts.offset(), 0);
        assertEquals(token.length(), LINES);
        
        assertFalse(ts.moveNext());
       
    }
Example 11
Source File: JspLexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
public void generate_assertTokenCommands(String code) {
    TokenHierarchy th = TokenHierarchy.create(code, JspTokenId.language());
    TokenSequence<JspTokenId> ts = th.tokenSequence(JspTokenId.language());
    ts.moveStart();
    while (ts.moveNext()) {
        Token<JspTokenId> t = ts.token();
        System.out.println(String.format("assertToken(ts, \"%s\", JspTokenId.%s);", t.text(), t.id().name()));
    }


}
Example 12
Source File: UpdateStatementAnalyzer.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
public static UpdateStatement analyze(TokenSequence<SQLTokenId> seq, Quoter quoter) {
    seq.moveStart();
    if (!seq.moveNext()) {
        return null;
    }
    UpdateStatementAnalyzer sa = new UpdateStatementAnalyzer(seq, quoter);
    sa.parse();
    TablesClause tablesClause = sa.context.isAfter(Context.FROM) ? sa.createTablesClause(sa.tables) : null;
    return new UpdateStatement(sa.startOffset, seq.offset() + seq.token().length(), tablesClause, Collections.unmodifiableList(sa.subqueries), sa.offset2Context);
}
Example 13
Source File: PHPDocCommentLexerTest.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
public void testPropertyTagTags() throws Exception {
    TokenSequence<?> ts = PHPLexerUtils.seqForText("" +
            "comment 1\n" +
            " * @property int age how old she is\n" +
            " * @property-read string nick readonly property\n" +
            " * @property-write boolean death", PHPDocCommentTokenId.language());
    PHPLexerUtils.printTokenSequence(ts, "testSimpleComment"); ts.moveStart();
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_COMMENT, "comment 1\n * ");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_ANNOTATION, "@property");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_COMMENT, " int age how old she is\n * ");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_ANNOTATION, "@property-read");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_COMMENT, " string nick readonly property\n * ");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_ANNOTATION, "@property-write");
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_COMMENT, " boolean death");
}
Example 14
Source File: LatteHtmlEmbeddingProvider.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
@Override
public List<Embedding> getEmbeddings(Snapshot snapshot) {
    TokenHierarchy<CharSequence> th = TokenHierarchy.create(snapshot.getText(), LatteTopTokenId.language());
    TokenSequence<LatteTopTokenId> ts = th.tokenSequence(LatteTopTokenId.language());
    if (ts == null) {
        return Collections.<Embedding>emptyList();
    }
    ts.moveStart();
    List<Embedding> embeddings = new ArrayList<>();
    int from = -1;
    int length = 0;
    while (ts.moveNext()) {
        Token<LatteTopTokenId> token = ts.token();
        if (token != null && isPureHtmlToken(token)) {
            if (from < 0) {
                from = ts.offset();
            }
            length += token.length();
        } else {
            if (from >= 0) {
                embeddings.add(snapshot.create(from, length, TARGET_MIME_TYPE));
                embeddings.add(snapshot.create(GENERATED_CODE, TARGET_MIME_TYPE));
                from = -1;
                length = 0;
            }
        }
    }
    if (from >= 0) {
        embeddings.add(snapshot.create(from, length, TARGET_MIME_TYPE));
    }
    if (embeddings.isEmpty()) {
        return Collections.singletonList(snapshot.create("", TARGET_MIME_TYPE)); //NOI18N
    } else {
        return Collections.singletonList(Embedding.create(embeddings));
    }
}
Example 15
Source File: LexUtilities.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
private static <T1 extends TokenId> List<JoinedTokenSequence.CodeBlock<T1>> calculateCodeBlock(List<TokenSequence<T1>> tss,
        VirtualSource virtualSource
    ) throws BadLocationException {

    List<JoinedTokenSequence.CodeBlock<T1>> blocks = new ArrayList<JoinedTokenSequence.CodeBlock<T1>>();

    for (int i=0; i<tss.size(); i++) {
        TokenSequence<T1> ts =tss.get(i);

        List<TokenSequenceWrapper<T1>> tss2 = new ArrayList<TokenSequenceWrapper<T1>>();
        tss2.add(new TokenSequenceWrapper<T1>(ts, false));

        // try to find additional token sequences which comprise this language block:
        for (int j=i+1; j<tss.size(); j++) {
            TokenSequence<T1> prev = tss.get(j-1);
            prev.moveEnd();
            prev.movePrevious();
            TokenSequence<T1> next = tss.get(j);
            next.moveStart();
            next.moveNext();
            // check whether current token sequence is continuation of previous one:
            TokenSequence<T1> tsVirtual = LexUtilities.getVirtualTokens(virtualSource, prev.offset()+prev.token().length(), next.offset(), ts.language());
            if (tsVirtual != null) {
                tss2.add(new TokenSequenceWrapper<T1>(tsVirtual, true));
                tss2.add(new TokenSequenceWrapper<T1>(next, false));
                i++;
            } else {
                break;
            }
        }


        blocks.add(new JoinedTokenSequence.CodeBlock<T1>(tss2));
    }

    return blocks;
}
Example 16
Source File: OQLCompletionProvider.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
private Token<OQLTokenId> findCurrentToken(JTextComponent component, TokenSequence<OQLTokenId> ts) {
    Token<OQLTokenId> currentToken = null;
    ts.moveStart();
    int forPosition = component.getCaretPosition();
    int position = 0;
    while(ts.moveNext()) {
        position = ts.offset();
        if (position >= forPosition) {
            ts.movePrevious();
            break;
        }
        currentToken = ts.token();
    }
    return currentToken;
}
Example 17
Source File: HtmlEmbeddingProvider.java    From netbeans with Apache License 2.0 5 votes vote down vote up 
private TokenSequence<GspTokenId> getTokenSequence(Snapshot snapshot) {
    final Language<GspTokenId> gspLanguage = GspLexerLanguage.getLanguage();
    final TokenHierarchy<CharSequence> tokenHierarchy = TokenHierarchy.create(snapshot.getText(), gspLanguage);
    final TokenSequence<GspTokenId> sequence = tokenHierarchy.tokenSequence(gspLanguage);

    sequence.moveStart();
    return sequence;
}
Example 18
Source File: PHPDocCommentLexerTest.java    From netbeans with Apache License 2.0 4 votes vote down vote up 
public void testSimpleComment() throws Exception{
    TokenSequence<?> ts = PHPLexerUtils.seqForText("comment 1", PHPDocCommentTokenId.language());
    PHPLexerUtils.printTokenSequence(ts, "testSimpleComment"); ts.moveStart();
    PHPLexerUtils.next(ts, PHPDocCommentTokenId.PHPDOC_COMMENT, "comment 1");
}
Example 19
Source File: KODataBindDescriptor.java    From netbeans with Apache License 2.0 4 votes vote down vote up 
public static KODataBindDescriptor getDataBindDescriptor(Snapshot snapshot, TokenSequence<? extends JsTokenId> ts, boolean simpleForEach) {
    if (ts == null) {
        return null;
    }

    ts.moveStart();
    ts.moveNext();
    String name = null;
    String alias = null;
    String data = null;
    boolean forEach = false;
    Token<? extends JsTokenId> token = LexUtilities.findNextNonWsNonComment(ts);
    if (token.id() == JsTokenId.BRACKET_LEFT_CURLY) {
        while ((token = findNext(ts, JsTokenId.IDENTIFIER, false)) != null) {
            String text = token.text().toString();
            if ((NAME_PROPERTY.equals(text) || AS_PROPERTY.equals(text)) && ts.moveNext()) { // NOI18N
                token = LexUtilities.findNextNonWsNonComment(ts);
                if (token.id() == JsTokenId.OPERATOR_COLON && ts.moveNext()) {
                    token = LexUtilities.findNextNonWsNonComment(ts);
                    if (token.id() == JsTokenId.STRING_BEGIN && ts.moveNext()) {
                        token = LexUtilities.findNextNonWsNonComment(ts);
                        if (token.id() == JsTokenId.STRING) {
                            if (NAME_PROPERTY.equals(text)) { // NOI18N
                                name = token.text().toString();
                            } else {
                                alias = token.text().toString();
                            }
                        }
                    }
                }
            } else if ((DATA_PROPERTY.equals(text) || FOREACH_PROPERTY.equals(text)) && ts.moveNext()) { // NOI18N
                token = LexUtilities.findNextNonWsNonComment(ts);
                if (token.id() == JsTokenId.OPERATOR_COLON && ts.moveNext()) {
                    LexUtilities.findNextNonWsNonComment(ts);
                    int start = ts.offset();
                    token = findNext(ts, JsTokenId.OPERATOR_COMMA, true);
                    if (token != null) {
                        data = snapshot.getText().subSequence(start, ts.offset()).toString().trim();
                        forEach = FOREACH_PROPERTY.equals(text);
                    }
                }
            }
            if (token == null || token.id() != JsTokenId.OPERATOR_COMMA) {
                findNext(ts, JsTokenId.OPERATOR_COMMA, false);
            }
        }
        if ((name != null || simpleForEach) && data != null) {
            return new KODataBindDescriptor(name, data, forEach, (forEach || simpleForEach) ? alias : null);
        }
    }

    return null;
}
Example 20
Source File: JadeJsEmbeddingProvider.java    From netbeans with Apache License 2.0 4 votes vote down vote up 
@Override
public List<Embedding> getEmbeddings(Snapshot snapshot) {
    TokenHierarchy<?> th = snapshot.getTokenHierarchy();
    TokenSequence<JadeTokenId> ts = th.tokenSequence(JadeTokenId.jadeLanguage());
    
    if (ts == null) {
        LOGGER.log(
                Level.WARNING,
                "TokenHierarchy.tokenSequence(JadeTokenId.jadeLanguage()) == null " + "for static immutable Jade TokenHierarchy!\nFile = ''{0}'' ;snapshot mimepath=''{1}''",
                new Object[]{snapshot.getSource().getFileObject().getPath(), snapshot.getMimePath()});

        return Collections.emptyList();
    }
    
    ts.moveStart();
    
    List<Embedding> embeddings = new ArrayList<>();
    
    int from = -1;
    int len = 0;
    Token<JadeTokenId> lastTag = null;
    Token<JadeTokenId> lastMixing = null;
    
    while (ts.moveNext()) {
        Token<JadeTokenId> token = ts.token();
        if (token.id() == JadeTokenId.JAVASCRIPT) {
            if (from < 0) {
                from = ts.offset();
            }
            len += token.length();
        } else {
            if (from >= 0) {
                if (lastMixing != null) {
                    wrapAsFnParameter(snapshot, embeddings, lastMixing.text().toString(), from, len);
                } else {
                    embeddings.add(snapshot.create(from, len, JS_MIME_TYPE));
                    addNewLine(snapshot, embeddings, from + len - 1);
                }
            } 
            from = -1;
            len = 0;
        }
        if (token.id() == JadeTokenId.TAG) {
            lastTag = token;
            lastMixing = null;
        }
        if (token.id() == JadeTokenId.MIXIN_NAME) {
            lastTag = null;
            lastMixing = token;
        }
        if (token.id() == JadeTokenId.PLAIN_TEXT_DELIMITER) {
            // check whether there is not 
            if (lastTag != null && SCRIPT_TAG_NAME.equals(lastTag.text().toString().toLowerCase()) && ts.moveNext()) {
                token = ts.token();
                while (token.id() == JadeTokenId.EOL && ts.moveNext()) {
                    token = ts.token();
                }
                if (token.id() == JadeTokenId.PLAIN_TEXT || token.id() == JadeTokenId.JAVASCRIPT) {
                    embeddings.add(snapshot.create(ts.offset(), token.length(), JS_MIME_TYPE));
                }
            }
        }
    }
    if (from >= 0) {
        if (lastMixing != null) {
            wrapAsFnParameter(snapshot, embeddings, lastMixing.text().toString(), from, len);
        } else {
            embeddings.add(snapshot.create(from, len, JS_MIME_TYPE));
            addNewLine(snapshot, embeddings, from + len - 1);
        }
    }
    if (embeddings.isEmpty()) {
        return Collections.emptyList();
    }
    return Collections.singletonList(Embedding.create(embeddings));
}