Java Code Examples for org.netbeans.api.lexer.TokenSequence#embedded()

private static TokenSequence getDeepestTokenSequence (
    TokenHierarchy  tokenHierarchy,
    int             offset
) {
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence ();
    while(tokenSequence != null) {
        tokenSequence.move(offset - 1);
        if(!tokenSequence.moveNext()) {
            break;
        }
        TokenSequence ts = tokenSequence.embedded();
        if(ts == null) {
            return tokenSequence;
        } else {
            tokenSequence = ts;
        }
    }
    return tokenSequence;
}
 

private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) {
    if (seq == null) {
        sb.append("Inactive TokenHierarchy"); //NOI18N
    } else {
        for(seq.moveStart(); seq.moveNext(); ) {
            TokenSequence<?> emSeq = seq.embedded();
            if (emSeq != null) {
                dumpSequence(emSeq, sb);
            } else {
                Token<?> token = seq.token();
                sb.append("<"); //NOI18N
                sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N
                sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N
                sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N
                sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N
                sb.append(tokenText(token));
                sb.append("'\n"); //NOI18N
            }
        }
    }
}
 

public void testEmbedding() {
    TokenHierarchy th = TokenHierarchy.create("abc xyz 012 0xFF00 0-1-2-3-4-5-6-7-8-9", TestPlainTokenId.language());
    TokenSequence tokens = th.tokenSequence();
    
    for( ; tokens.moveNext(); ) {
        TokenId id = tokens.token().id();
        TokenSequence embedded = tokens.embedded();
        
        if (id == TestPlainTokenId.WHITESPACE) {
            assertNull("Whitespace should not have any embedded language", embedded);
        } else if (id == TestPlainTokenId.WORD) {
            assertNotNull("Word should have an embedded token sequence", embedded);
            assertNotNull("Word should have an embedded language", embedded.language());
            assertEquals("Wrong embedded language", TestCharTokenId.MIME_TYPE, embedded.language().mimeType());
        }
    }
}
 

/**
 * Collect language paths used within the given token sequence
 * 
 * @param ts non-null token sequence (or subsequence). <code>ts.moveNext()</code>
 * is called first on it.
 * @return collection of language paths present in the given token sequence.
 */
private Collection<LanguagePath> getActiveEmbeddedPaths(TokenSequence ts) {
    Collection<LanguagePath> lps = new HashSet<LanguagePath>();
    lps.add(ts.languagePath());
    List<TokenSequence<?>> tsStack = null;
    while (true) {
        while (ts.moveNext()) {
            TokenSequence<?> eTS = ts.embedded();
            if (eTS != null) {
                tsStack.add(ts);
                ts = eTS;
                lps.add(ts.languagePath());
            }
        }
        if (tsStack != null && tsStack.size() > 0) {
            ts = tsStack.get(tsStack.size() - 1);
            tsStack.remove(tsStack.size() - 1);
        } else {
            break;
        }
    }
    return lps;
}
 

private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) {
    if (seq == null) {
        sb.append("Inactive TokenHierarchy"); //NOI18N
    } else {
        for(seq.moveStart(); seq.moveNext(); ) {
            TokenSequence<?> emSeq = seq.embedded();
            if (emSeq != null) {
                dumpSequence(emSeq, sb);
            } else {
                Token<?> token = seq.token();
                sb.append("<"); //NOI18N
                sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N
                sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N
                sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N
                sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N
                sb.append(tokenText(token));
                sb.append("'\n"); //NOI18N
            }
        }
    }
}
 

/**Find span of the name in the DocTree's reference tree (see {@link #getReferenceName(com.sun.source.util.DocTreePath)}
 * identifier in the source. Returns starting and ending offset of the name in
 * the source code that was parsed (ie. {@link CompilationInfo.getText()}, which
 * may differ from the positions in the source document if it has been already
 * altered.
 * 
 * @param ref reference for which the identifier should be found
 * @return the span of the name, or null if cannot be found
 * @since 0.124
 */
public int[] findNameSpan(DocCommentTree docTree, ReferenceTree ref) {
    Name name = ((DCReference) ref).memberName;
    if (name == null || !SourceVersion.isIdentifier(name)) {
        //names like "<error>", etc.
        return null;
    }
    
    int pos = (int) info.getDocTrees().getSourcePositions().getStartPosition(info.getCompilationUnit(), docTree, ref);
    
    if (pos < 0)
        return null;
    
    TokenSequence<JavaTokenId> tokenSequence = info.getTokenHierarchy().tokenSequence(JavaTokenId.language());
    
    tokenSequence.move(pos);
    
    if (!tokenSequence.moveNext() || tokenSequence.token().id() != JavaTokenId.JAVADOC_COMMENT) return null;
    
    TokenSequence<JavadocTokenId> jdocTS = tokenSequence.embedded(JavadocTokenId.language());
    
    jdocTS.move(pos);
    
    boolean wasNext;
    
    while ((wasNext = jdocTS.moveNext()) && jdocTS.token().id() != JavadocTokenId.HASH)
        ;
    
    if (wasNext && jdocTS.moveNext()) {
        if (jdocTS.token().id() == JavadocTokenId.IDENT &&
            name.contentEquals(jdocTS.token().text())) {
            return new int[] {
                jdocTS.offset(),
                jdocTS.offset() + jdocTS.token().length()
            };
        }
    }
    
    return null;
}
 

public static TokenSequence getTokenSequence (Document document, int offset) {
    TokenHierarchy tokenHierarchy = TokenHierarchy.get (document);
    if (tokenHierarchy == null) return null;
    TokenSequence tokenSequence = tokenHierarchy.tokenSequence ();
    if (tokenSequence == null) return null;
    while (true) {
        tokenSequence.move (offset);
        if (!tokenSequence.moveNext ()) return tokenSequence;
        TokenSequence tokenSequence2 = tokenSequence.embedded ();
        if (tokenSequence2 == null) return tokenSequence;
        tokenSequence = tokenSequence2;
    }
}
 

public void testEmbeddingActivityChange() throws Exception {
    ModificationTextDocument doc = new ModificationTextDocument();
    // Assign a language to the document
    doc.putProperty(Language.class,TestTokenId.language());
    doc.insertString(0, "a/*abc def*/", null);
    LexerTestUtilities.initLastTokenHierarchyEventListening(doc);
    TokenHierarchy<?> hi = TokenHierarchy.get(doc);
    ((AbstractDocument)doc).readLock();
    try {
        TokenSequence<?> ts = hi.tokenSequence();
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "a", 0);
        assertTrue(ts.moveNext());
        LexerTestUtilities.assertTokenEquals(ts,TestTokenId.BLOCK_COMMENT, "/*abc def*/", 1);
        TokenSequence<?> ets = ts.embedded();
        assertNotNull(ets);
        assertTrue(ts.moveNext());
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "abc", 3);
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WHITESPACE, " ", 6);
        assertTrue(ets.moveNext());
        LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "def", 7);
        assertFalse(ets.moveNext());
    } finally {
        ((AbstractDocument)doc).readUnlock();
    }

    MutableTextInput input = (MutableTextInput) doc.getProperty(MutableTextInput.class);
    final TokenHierarchyControl control = input.tokenHierarchyControl();
    doc.runAtomic(new Runnable() {
        @Override
        public void run() {
            control.setActive(false);
            control.setActive(true);
        }
    });

}
 

private void processTopSequence(TokenSequence<TwigTopTokenId> tts) {
    tts.move(offset);
    if (tts.moveNext() || tts.movePrevious()) {
        TokenSequence<? extends TokenId> ts = tts.embedded(TwigBlockTokenId.language());
        if (ts == null) {
            ts = tts.embedded(TwigVariableTokenId.language());
        }
        processSequence(ts);
    }
}
 

@Override
public List<Embedding> translate(Snapshot snapshot) {
    TokenHierarchy<?> th = snapshot.getTokenHierarchy();
    if (th == null) {
        //the token hierarchy may be null if the language is not initialized yet
        //for example if ergonomics is used and j2ee cluster not activated
        return Collections.emptyList();
    }

    TokenSequence<? extends TokenId> tokenSequence = th.tokenSequence();
    List<Embedding> embeddings = new ArrayList<Embedding>();

    JsAnalyzerState state = new JsAnalyzerState();

    while (tokenSequence.moveNext()) {
        Token<? extends TokenId> token = tokenSequence.token();

        if (token.id().primaryCategory().equals("html")) { // NOI18N
            TokenSequence<? extends HTMLTokenId> ts = tokenSequence.embedded(HTMLTokenId.language());
            if (ts == null) {
                continue;
            }
            extractJavaScriptFromHtml(snapshot, ts, state, embeddings);

        } else if (token.id().primaryCategory().equals("expression-language") ) { // NOI18N
            if (state.in_inlined_javascript || state.in_javascript) {
                embeddings.add(snapshot.create(GENERATED_IDENTIFIER, JsTokenId.JAVASCRIPT_MIME_TYPE));
            }
        }
    }

    return embeddings;
}
 

private void moveNext(TokenSequence ts) {
    AttributeSet as = null;
    do {
        ts.move(startOffset1);
        if (!ts.moveNext()) {
            return;
        }
        Token t = ts.token();
        if (ts.language() == null)
            throw new NullPointerException ("ts.language()==null: TS " + ts + " : " + document.getProperty("mimeType"));
        as = highlighting.get (ts.offset(), ts.offset() + t.length());
        if (as != null) {
            attributeSet.addAttributes(as);
            endOffset1 = ts.offset() + t.length();
            return;
        }
        TokenSequence ts1 = ts.embedded();
        if (ts1 != null) {
            moveNext(ts1);
        }
        if (endOffset1 > startOffset1) {
            return;
        }
        if (ts.token() != null) {
            startOffset1 = ts.offset() + ts.token().length();
            endOffset1 = startOffset1;
        } else {
            return;
        }
    } while (startOffset1 < endOffset);
}
 

public static TokenSequence<JavadocTokenId> findJavadocTokenSequence(CompilationInfo javac, int offset) {
    TokenSequence<JavaTokenId> ts = SourceUtils.getJavaTokenSequence(javac.getTokenHierarchy(), offset);
    if (ts == null || !movedToJavadocToken(ts, offset)) {
        return null;
    }
    
    TokenSequence<JavadocTokenId> jdts = ts.embedded(JavadocTokenId.language());
    if (jdts == null) {
        return null;
    }
    
    jdts.move(offset);
    return jdts;
}
 

private OffsetRange getCoreHtmlReferenceSpan(Document doc, int caretOffset) {
    TokenHierarchy hi = TokenHierarchy.get(doc);
    final TokenSequence<HTMLTokenId> ts = Utils.getJoinedHtmlSequence(hi, caretOffset);
    if (ts == null) {
        return null;
    }

    //tag attribute value hyperlinking
    if (ts.token().id() == HTMLTokenId.VALUE) {
        return new AttributeValueAction<OffsetRange>(hi, ts) {

            @Override
            public OffsetRange resolve() {
                if (tagName != null && attrName != null) {
                    ValueCompletion<HtmlCompletionItem> support = AttrValuesCompletion.getSupport(tagName, attrName);
                    if (AttrValuesCompletion.FILE_NAME_SUPPORT == support) {
                        //some file to hyperlink to
                        return valueRange;
                    }
                }
                return null;
            }
        }.run();

    } else if (ts.token().id() == HTMLTokenId.VALUE_CSS) {
        //css class or id hyperlinking
        TokenSequence<CssTokenId> cssTs = ts.embedded(CssTokenId.language());
        if (cssTs != null) {
            cssTs.move(caretOffset);
            if (cssTs.moveNext() || cssTs.movePrevious()) {
                if (cssTs.token().id() == CssTokenId.IDENT) {
                    return new OffsetRange(cssTs.offset(), cssTs.offset() + cssTs.token().length());
                }
            }
        }
    }

    return null;
}
 

public void testUpdate1() throws Exception {
    Document d = new ModificationTextDocument();
    
    d.putProperty(Language.class,TestTokenId.language());
    
    d.insertString(0, "\"\\t\\b\\t test\"", null);
    
    TokenHierarchy<?> h = TokenHierarchy.get(d);
    
    ((AbstractDocument)d).readLock();
    try {
        h.tokenSequence().tokenCount();

        TokenSequence<?> s = h.tokenSequence();

        assertTrue(s.moveNext());

        s.embedded();
    } finally {
        ((AbstractDocument)d).readUnlock();
    }

    d.insertString(5, "t", null);
}
 

/**
 * Test whether the line is in JavaScript source.
 * @param line The line to test
 * @return <code>true</code> when the line is in JavaScript source, <code>false</code> otherwise.
 */
public static boolean isInJavaScript(Line line) {
    LOG.log(Level.FINER, "\nisInJavaScript({0}):", line);
    FileObject fo = line.getLookup().lookup(FileObject.class);
    if (isJavascriptSource(fo)) {
        LOG.fine("is JavaScript source file => true");
        return true;
    }
    EditorCookie editorCookie = line.getLookup().lookup(EditorCookie.class);
    StyledDocument document = editorCookie.getDocument();
    Boolean isJS = null;
    ((AbstractDocument) document).readLock();
    try {
        TokenHierarchy<Document> th = TokenHierarchy.get((Document) document);
        int ln = line.getLineNumber();
        int offset = NbDocument.findLineOffset(document, ln);
        int maxOffset = document.getLength() - 1;
        int maxLine = NbDocument.findLineNumber(document, maxOffset);
        int offset2;
        if (ln + 1 > maxLine) {
            offset2 = maxOffset;
        } else {
            offset2 = NbDocument.findLineOffset(document, ln+1) - 1;
        }
        // The line has offsets <offset, offset2>
        Set<LanguagePath> languagePaths = th.languagePaths();
        for (LanguagePath lp : languagePaths) {
            List<TokenSequence<?>> tsl = th.tokenSequenceList(lp, offset, offset2);
            for (TokenSequence ts : tsl) {
                if (ts.moveNext()) {
                    /*int to = ts.offset();
                    if (LOG.isLoggable(Level.FINER)) {
                        LOG.finer("Token offset = "+to+", offsets = <"+offset+", "+offset2+">, mimeType = "+ts.language().mimeType());
                    }
                    if (!(offset <= to && to < offset2)) {
                        continue;
                    }*/
                    TokenSequence ets;
                    ets = ts.embedded();
                    if (ets != null) {
                        ts = ets;
                    }
                    String mimeType = ts.language().mimeType();
                    LOG.log(Level.FINER, "Have language {0}", mimeType);
                    if (isJS == null && JAVASCRIPT_MIME_TYPE.equals(mimeType)) {
                        isJS = true;
                        if (!LOG.isLoggable(Level.FINER)) {
                            break;
                        }
                    }
                }
            }
        }
    } finally {
        ((AbstractDocument) document).readUnlock();
    }
    LOG.log(Level.FINER, "isJS = {0}", isJS);
    return isJS != null && isJS;
}
 

private static TokenSequence getEmbeddedExtDocTS(TokenSequence ts) {
    return ts.embedded(JsDocumentationTokenId.language());
}
 

private static TokenSequence getEmbeddedSDocTS(TokenSequence ts) {
    return ts.embedded(JsDocumentationTokenId.language());
}
 

public void testCreateEmbedding() {
    String text = "abc/*def ghi */// line comment";
    TokenHierarchy<?> hi = TokenHierarchy.create(text,TestTokenId.language());
    THListener listener = new THListener();
    hi.addTokenHierarchyListener(listener);
    TokenSequence<?> ts = hi.tokenSequence();
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.IDENTIFIER, "abc", 0);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.BLOCK_COMMENT, "/*def ghi */", 3);
    assertTrue(ts.moveNext());
    LexerTestUtilities.assertTokenEquals(ts,TestTokenId.LINE_COMMENT, "// line comment", 15);
    assertTrue(ts.createEmbedding(TestTokenId.language(), 3, 0));
    
    // Check the fired event
    TokenHierarchyEvent evt = listener.fetchLastEvent();
    assertNotNull(evt);
    TokenChange<?> tc = evt.tokenChange();
    assertNotNull(tc);
    assertEquals(2, tc.index());
    assertEquals(15, tc.offset());
    assertEquals(0, tc.addedTokenCount());
    assertEquals(0, tc.removedTokenCount());
    assertEquals(TestTokenId.language(), tc.language());
    assertEquals(1, tc.embeddedChangeCount());
    TokenChange<?> etc = tc.embeddedChange(0);
    assertEquals(0, etc.index());
    assertEquals(18, etc.offset());
    assertEquals(0, etc.addedTokenCount()); // 0 to allow for lazy lexing where this would be unknowns
    assertEquals(0, etc.removedTokenCount());
    assertEquals(TestTokenId.language(), etc.language());
    assertEquals(0, etc.embeddedChangeCount());
    
    // Test the contents of the embedded sequence
    TokenSequence<?> ets = ts.embedded(); // Over "// line comment"
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "line", 18);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.WHITESPACE, " ", 22);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "comment", 23);
    assertFalse(ets.moveNext());
    
    // Move main TS back and try extra embedding on comment
    assertTrue(ts.movePrevious());
    assertTrue(ts.createEmbedding(TestTokenId.language(), 2, 2));
    ets = ts.embedded(); // Should be the explicit one
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "def", 5);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.WHITESPACE, " ", 8);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.IDENTIFIER, "ghi", 9);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestTokenId.WHITESPACE, " ", 12);
    assertFalse(ets.moveNext());
    
    // Get the default embedding - should be available as well
    ets = ts.embedded(TestPlainTokenId.language()); // Should be the explicit one
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "def", 5);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WHITESPACE, " ", 8);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WORD, "ghi", 9);
    assertTrue(ets.moveNext());
    LexerTestUtilities.assertTokenEquals(ets,TestPlainTokenId.WHITESPACE, " ", 12);
    assertFalse(ets.moveNext());
    

    ets = ts.embedded(); // Get the custom embedded token sequence
    assertEquals(ets.language(), TestTokenId.language());
    assertTrue(ets.moveNext());
    assertTrue(ets.isValid());
    assertNotNull(ets);
    // Test removal of the embedding
    assertTrue(ts.removeEmbedding(TestTokenId.language()));
    // The embedded token sequence should no longer be valid
    assertFalse(ets.isValid());
    // Token sequence on which the removeEmbedding() was called should continue to be valid
    assertTrue(ts.isValid());
    // Repetitive removal should return false
    assertFalse(ts.removeEmbedding(TestTokenId.language()));

    
    // Check token sequence list
    // Create custom embedding again
    assertTrue(ts.createEmbedding(TestTokenId.language(), 2, 2));
    LanguagePath lpe = LanguagePath.get(TestTokenId.language()).embedded(TestTokenId.language());
    List<TokenSequence<?>> tsl = hi.tokenSequenceList(lpe, 0, Integer.MAX_VALUE);
    assertEquals(2, tsl.size());
    ts.removeEmbedding(TestTokenId.language());
    tsl = hi.tokenSequenceList(lpe, 0, Integer.MAX_VALUE);
    assertEquals(1, tsl.size());
}
 

public static List<OffsetRange> findForwardMatching(
        TokenSequence<? extends TwigTopTokenId> topTs,
        TwigTokenText start,
        TwigTokenText end,
        List<TwigTokenText> middle) {
    List<OffsetRange> result = new ArrayList<>();
    topTs.moveNext();
    int originalOffset = topTs.offset();
    int balance = 1;
    while (topTs.moveNext()) {
        Token<? extends TwigTopTokenId> token = topTs.token();
        if (token != null && (token.id() == TwigTopTokenId.T_TWIG_BLOCK || token.id() == TwigTopTokenId.T_TWIG_VAR)) {
            TokenSequence<TwigBlockTokenId> markupTs = topTs.embedded(TwigBlockTokenId.language());
            if (markupTs != null) {
                markupTs.moveNext();
                while (markupTs.moveNext()) {
                    Token<? extends TwigBlockTokenId> markupToken = markupTs.token();
                    if (start.matches(markupToken)) {
                        balance++;
                    } else if (end.matches(markupToken)) {
                        balance--;
                        if (balance == 0) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    } else if (matchesToken(middle, markupToken)) {
                        if (balance == 1) {
                            result.add(new OffsetRange(markupTs.offset(), markupTs.offset() + markupToken.length()));
                            break;
                        }
                    }
                }
                if (balance == 0) {
                    break;
                }
            }
        }
    }
    topTs.move(originalOffset);
    return result;
}
 

@Override
public List<Embedding> translate(Snapshot snapshot) {
    TokenHierarchy<?> th = snapshot.getTokenHierarchy();
    if (th == null) {
        //likely the latte language couldn't be found
        LOG.info("Cannot get TokenHierarchy from snapshot " + snapshot); //NOI18N
        return Collections.emptyList();
    }

    TokenSequence<? extends TokenId> tokenSequence = th.tokenSequence();
    List<Embedding> embeddings = new ArrayList<Embedding>();

    JsAnalyzerState state = new JsAnalyzerState();
    while (tokenSequence.moveNext()) {
        Token<? extends TokenId> token = tokenSequence.token();

        if (token.id().name().equals("T_HTML")) { //NOI18N
            TokenSequence<? extends HTMLTokenId> ts = tokenSequence.embedded(HTMLTokenId.language());
            if (ts == null) {
                continue;
            }
            extractJavaScriptFromHtml(snapshot, ts, state, embeddings);
        } else if (token.id().name().equals("T_LATTE")) { // NOI18N
            if (state.in_inlined_javascript || state.in_javascript) {
                //find end of the latte
                boolean hasNext = false;
                while (token.id().name().equals("T_LATTE")) { // NOI18N
                    hasNext = tokenSequence.moveNext();
                    if (!hasNext) {
                        break;
                    }
                    token = tokenSequence.token();
                }
                if (hasNext) {
                    tokenSequence.movePrevious();
                }
                embeddings.add(snapshot.create(GENERATED_IDENTIFIER, JsTokenId.JAVASCRIPT_MIME_TYPE));
            }
        }
    }

    return embeddings;
}