org.netbeans.api.lexer.Token Java Examples
The following examples show how to use
org.netbeans.api.lexer.Token.
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example #1
Source File: JspHyperlinkProvider.java From netbeans with Apache License 2.0 | 6 votes |
private FileObject getTagFile(TokenSequence<?> tokenSequence, JspSyntaxSupport jspSup) { Token token = tokenSequence.token(); if (token.id() == JspTokenId.TAG) { String image = token.text().toString().trim(); if (image.startsWith("<")) { // NOI18N image = image.substring(1).trim(); } if (!image.startsWith("jsp:") && image.indexOf(':') != -1) { // NOI18N List l = jspSup.getTags(image); if (l.size() == 1) { TagLibraryInfo libInfo = ((TagInfo) l.get(0)).getTagLibrary(); if (libInfo != null) { TagFileInfo fileInfo = libInfo.getTagFile(getTagName(image)); if (fileInfo != null) { return JspUtils.getFileObject(jspSup.getDocument(), fileInfo.getPath()); } } } } } return null; }
Example #2
Source File: JShellLexer.java From netbeans with Apache License 2.0 | 6 votes |
private Token<JShellToken> createContinuationTokenForState(S s) { switch (s) { case INITIAL: return tokenFactory.createToken(JShellToken.WHITESPACE); case PROMPT_INPUT: return blockToken(JShellToken.JAVA); case PROMPT_MESSAGE: return blockToken(JShellToken.MESSAGE_TEXT); case MESSAGE: return blockToken(JShellToken.MESSAGE_TEXT); case JAVA: return blockToken(JShellToken.JAVA); case COMMAND: return blockToken(JShellToken.OUTPUT); } return null; }
Example #3
Source File: DemoTokenUpdater.java From netbeans with Apache License 2.0 | 6 votes |
protected Token createToken(TokenId id, int index, int length) { String sampleText = null; if (Compatibility.charSequenceExists()) { SampleTextMatcher matcher = id.getSampleTextMatcher(); if (matcher != null) { /* The recognizedText would not be a string * in the normal applications. It would be * a custom char sequence reused for every * recognized token. Here it's string * to simplify the code. */ CharSequence recognizedText = (CharSequence)(Object)getDocumentText(index, length); // 1.3 compilability sampleText = matcher.match(recognizedText); } } Token token; if (sampleText != null) { token = new DemoSampleToken(id, sampleText); } else { token = new DemoToken(this, id, index, length); } return token; }
Example #4
Source File: CPCssEditorModule.java From netbeans with Apache License 2.0 | 6 votes |
@Override public <T extends Set<OffsetRange>> NodeVisitor<T> getInstantRenamerVisitor(EditorFeatureContext context, T result) { TokenSequence<CssTokenId> tokenSequence = context.getTokenSequence(); int diff = tokenSequence.move(context.getCaretOffset()); if (diff > 0 && tokenSequence.moveNext() || diff == 0 && tokenSequence.movePrevious()) { Token<CssTokenId> token = tokenSequence.token(); final CharSequence elementName = token.text(); return new NodeVisitor<T>(result) { @Override public boolean visit(Node node) { switch (node.type()) { case cp_mixin_name: case cp_variable: if (LexerUtils.equals(elementName, node.image(), false, false)) { OffsetRange range = new OffsetRange(node.from(), node.to()); getResult().add(range); break; } } return false; } }; } return null; }
Example #5
Source File: AnnotationManager.java From netbeans with Apache License 2.0 | 6 votes |
private boolean testCreateAnnotation(TokenHierarchy hi, TokenSequence ts, ASTItem item, LanguagesAnnotation la) throws BadLocationException { if (ts.language () == null) throw new NullPointerException ("ts.language()==null"); if (ts.language ().mimeType () == null) throw new NullPointerException ("TokenSequence.mimeType==null"); if (ts.language().mimeType().equals(item.getMimeType())) { Token t = ts.token(); if (t == null) throw new NullPointerException (); Position position = doc.createPosition(t.offset(hi)); la.setPosition (position); doc.addAnnotation(position, t.length(), la); return true; } else { ts = ts.embedded(); if(ts == null) { return false; } else { return ts.moveNext() ? testCreateAnnotation(hi, ts, item, la) : false; } } }
Example #6
Source File: TwigCompletionContextFinder.java From netbeans with Apache License 2.0 | 6 votes |
private static CompletionContext findContext(TokenSequence<? extends TokenId> tokenSequence) { CompletionContext result = CompletionContext.ALL; do { Token<? extends TokenId> token = tokenSequence.token(); if (token == null) { break; } TokenId tokenId = token.id(); if (TwigBlockTokenId.T_TWIG_OTHER.equals(tokenId) || TwigVariableTokenId.T_TWIG_OTHER.equals(tokenId)) { result = CompletionContext.NONE; break; } else if (acceptTokenChains(tokenSequence, FILTER_TOKEN_CHAINS, true)) { result = CompletionContext.FILTER; break; } else if (tokenId instanceof TwigBlockTokenId) { result = CompletionContext.BLOCK; break; } else if (tokenId instanceof TwigVariableTokenId) { result = CompletionContext.VARIABLE; break; } } while (tokenSequence.movePrevious()); return result; }
Example #7
Source File: LexerUtilities.java From netbeans with Apache License 2.0 | 6 votes |
/** * Search forwards in the token sequence until a matching closing token is * found so keeps track of nested pairs of up-down eg (()) is ignored if * we're searching for a ) * * @param ts the TokenSequence set to the position after an up * @param up the opening token eg { or [ * @param down the closing token eg } or ] * @return the Range of closing token in our case 1 char */ public static OffsetRange findBwd(TokenSequence<? extends SQLTokenId> ts, int up, int down) { int balance = 0; while (ts.movePrevious()) { Token<? extends SQLTokenId> token = ts.token(); if (token.id().ordinal() == up) { if (balance == 0) { return new OffsetRange(ts.offset(), ts.offset() + token.length()); } balance++; } else if (token.id().ordinal() == down) { balance--; } } return OffsetRange.NONE; }
Example #8
Source File: SyntaxHighlighting.java From netbeans with Apache License 2.0 | 6 votes |
private static void dumpSequence(TokenSequence<?> seq, StringBuilder sb) { if (seq == null) { sb.append("Inactive TokenHierarchy"); //NOI18N } else { for(seq.moveStart(); seq.moveNext(); ) { TokenSequence<?> emSeq = seq.embedded(); if (emSeq != null) { dumpSequence(emSeq, sb); } else { Token<?> token = seq.token(); sb.append("<"); //NOI18N sb.append(String.format("%3s", seq.offset())).append(", "); //NOI18N sb.append(String.format("%3s", seq.offset() + token.length())).append(", "); //NOI18N sb.append(String.format("%+3d", token.length())).append("> : "); //NOI18N sb.append(tokenId(token.id(), true)).append(" : '"); //NOI18N sb.append(tokenText(token)); sb.append("'\n"); //NOI18N } } } }
Example #9
Source File: SimpleLanguageProvider.java From netbeans with Apache License 2.0 | 5 votes |
public LanguageEmbedding<?> findLanguageEmbedding( Token<?> token, LanguagePath languagePath, InputAttributes inputAttributes) { if ("text/x-simple-plain".equals(languagePath.mimePath()) && token.id().name().equals("WORD")) { return LanguageEmbedding.create(TestCharTokenId.language(), 0, 0); } else { return null; } }
Example #10
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
/** * Finds the first previous token that is not in the ignore list * @param ts * @param ignores * @return */ public static Token<? extends JsTokenId> findPrevious(TokenSequence<? extends JsTokenId> ts, List<JsTokenId> ignores) { if (ignores.contains(ts.token().id())) { while (ts.movePrevious() && ignores.contains(ts.token().id())) {} } return ts.token(); }
Example #11
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
public static Token<?extends PHPTokenId> getToken(BaseDocument doc, int offset) { TokenSequence<?extends PHPTokenId> ts = getPositionedSequence(doc, offset); if (ts != null) { return ts.token(); } return null; }
Example #12
Source File: GroovyLexerStringConstantsTest.java From netbeans with Apache License 2.0 | 5 votes |
void dumpTokenStream (TokenSequence ts){ // System.out.println("#############################################"); while (ts.moveNext()) { Token t = ts.token(); // System.out.println("-------------------------------"); // System.out.println("Token-ID :" + t.id()); // if(t.toString() != null) // System.out.println("Token-Name :" + t.toString()); } }
Example #13
Source File: XmlLexerParser.java From netbeans with Apache License 2.0 | 5 votes |
private void skipWhitespace() { Token<XMLTokenId> t; while ((t = nextToken()) != null && t.id() == XMLTokenId.WS) { consume(); } }
Example #14
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
/** Compute the balance of begin/end tokens on the line. */ public static int getLineBalance(BaseDocument doc, int offset, TokenId up, TokenId down) { try { int begin = Utilities.getRowStart(doc, offset); int end = Utilities.getRowEnd(doc, offset); TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, begin); if (ts == null) { return 0; } ts.move(begin); if (!ts.moveNext()) { return 0; } int balance = 0; do { Token<GroovyTokenId> token = ts.token(); TokenId id = token.id(); if (id == up) { balance++; } else if (id == down) { balance--; } } while (ts.moveNext() && (ts.offset() <= end)); return balance; } catch (BadLocationException ble) { Exceptions.printStackTrace(ble); return 0; } }
Example #15
Source File: EditorUtils.java From netbeans with Apache License 2.0 | 5 votes |
public static Collection<String> getUsedFileInDefine(final Snapshot shanpshot, final int offset) { TokenSequence<? extends JsTokenId> ts = LexUtilities.getJsTokenSequence(shanpshot, offset); if (ts == null) { return Collections.emptyList(); } ts.move(0); if (!ts.moveNext()) { return Collections.emptyList(); } Token<? extends JsTokenId> token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.IDENTIFIER)); while (token.id() == JsTokenId.IDENTIFIER && !DEFINE.equals(token.text().toString()) && ts.moveNext()) { token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.IDENTIFIER)); } if (token.id() == JsTokenId.IDENTIFIER && DEFINE.equals(token.text().toString())) { // we are probably found the define method List<String> paths = new ArrayList<String>(); token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.BRACKET_LEFT_BRACKET, JsTokenId.KEYWORD_FUNCTION, JsTokenId.BRACKET_LEFT_CURLY, JsTokenId.BRACKET_RIGHT_PAREN)); if (token.id() == JsTokenId.BRACKET_LEFT_BRACKET) { do { token = LexUtilities.findNextToken(ts, Arrays.asList(JsTokenId.STRING, JsTokenId.OPERATOR_COMMA, JsTokenId.BRACKET_RIGHT_PAREN)); if (token.id() == JsTokenId.STRING) { paths.add(token.text().toString()); } } while ((token.id() != JsTokenId.BRACKET_RIGHT_PAREN && token.id() != JsTokenId.OPERATOR_SEMICOLON && !token.id().isKeyword()) && ts.moveNext()); return paths; } } return Collections.emptyList(); }
Example #16
Source File: ParameterInfoSupport.java From netbeans with Apache License 2.0 | 5 votes |
private static boolean skipWhitespaces(TokenSequence<PHPTokenId> tokenSequence) { Token<PHPTokenId> token = tokenSequence.token(); while (token != null && isWhiteSpace(token)) { boolean retval = tokenSequence.movePrevious(); token = tokenSequence.token(); if (!retval) { return false; } } return true; }
Example #17
Source File: JspELEmbeddingProvider.java From netbeans with Apache License 2.0 | 5 votes |
@Override public List<Embedding> getEmbeddings(Snapshot snapshot) { TokenHierarchy<?> th = snapshot.getTokenHierarchy(); TokenSequence<JspTokenId> sequence = th.tokenSequence(JspTokenId.language()); List<Embedding> embeddings = new ArrayList<Embedding>(); sequence.moveStart(); boolean inAttributeValueWithEL = false; while (sequence.moveNext()) { Token t = sequence.token(); if (t.id() == JspTokenId.ATTR_VALUE && t.length() == 1 && (t.text().charAt(0) == '"' || t.text().charAt(0) == '\'')) { //a quote before/after attribute value with EL inside inAttributeValueWithEL = !inAttributeValueWithEL; } if (t.id() == JspTokenId.EL) { embeddings.add(snapshot.create(sequence.offset(), t.length(), "text/x-el")); //NOI18N //XXX hack - there's a need to distinguish between ELs inside or outside of attribute values if(inAttributeValueWithEL) { embeddings.add(snapshot.create(ATTRIBUTE_EL_MARKER, "text/x-el")); //NOI18N } // just to separate expressions for easier handling in EL parser embeddings.add(snapshot.create(Constants.LANGUAGE_SNIPPET_SEPARATOR, "text/x-el")); //NOI18N } } if (embeddings.isEmpty()) { return Collections.emptyList(); } else { return Collections.singletonList(Embedding.create(embeddings)); } }
Example #18
Source File: FindLocalUsagesQuery.java From netbeans with Apache License 2.0 | 5 votes |
/** * Checks if {@code token} represents a comment. * * @param token the {@link Token} to check * @return {@code true} if {@code token} represents a line comment, block * comment; {@code false} otherwise or javadoc. */ private boolean isComment(final Token<JavaTokenId> token) { switch (token.id()) { case LINE_COMMENT: case BLOCK_COMMENT: return true; case JAVADOC_COMMENT: default: return false; } }
Example #19
Source File: JavaLexer.java From netbeans with Apache License 2.0 | 5 votes |
private Token<JavaTokenId> keywordOrIdentifier(JavaTokenId keywordId, int c) { // Check whether the given char is non-ident and if so then return keyword if (c == EOF || !Character.isJavaIdentifierPart(c = translateSurrogates(c))) { // For surrogate 2 chars must be backed up backup((c >= Character.MIN_SUPPLEMENTARY_CODE_POINT) ? 2 : 1); return token(keywordId); } else // c is identifier part return finishIdentifier(); }
Example #20
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
public static Token<?extends PHPTokenId> findPrevious(TokenSequence<?extends PHPTokenId> ts, List<PHPTokenId> ignores) { if (ignores.contains(ts.token().id())) { while (ts.movePrevious() && ignores.contains(ts.token().id())) { } } return ts.token(); }
Example #21
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
public static <T extends TokenId> Token<T> findPrevious(JoinedTokenSequence<T> ts, List<T> ignores) { if (ignores.contains(ts.token().id())) { while (ts.movePrevious() && ignores.contains(ts.token().id())) {} } if (ts.token() == null || ignores.contains(ts.token().id())) { return null; } return ts.token(); }
Example #22
Source File: InstantRenameActionTest.java From netbeans with Apache License 2.0 | 5 votes |
public void testIsInaccessibleOutsideOuterClassForStaticMethodOfNestedClass() throws Exception { boolean[] wasResolved = new boolean[1]; Collection<Token> changePoints = performTest("package test; public class Test { static class PPP { public static void method() {}}}", 73, wasResolved); assertNull(changePoints); assertTrue(wasResolved[0]); }
Example #23
Source File: JavadocCompletionUtils.java From netbeans with Apache License 2.0 | 5 votes |
public static boolean isWhiteSpaceFirst(Token<JavadocTokenId> token) { if (token == null || token.id() != JavadocTokenId.OTHER_TEXT || token.length() < 1) { return false; } CharSequence text = token.text(); char c = text.charAt(0); return c == ' ' || c == '\t'; }
Example #24
Source File: LexUtilities.java From netbeans with Apache License 2.0 | 5 votes |
/** Compute the balance of begin/end tokens on the line. * @param doc the document * @param offset The offset somewhere on the line * @param upToOffset If true, only compute the line balance up to the given offset (inclusive), * and if false compute the balance for the whole line */ public static int getBeginEndLineBalance(BaseDocument doc, int offset, boolean upToOffset) { try { int begin = Utilities.getRowStart(doc, offset); int end = upToOffset ? offset : Utilities.getRowEnd(doc, offset); TokenSequence<GroovyTokenId> ts = LexUtilities.getGroovyTokenSequence(doc, begin); if (ts == null) { return 0; } ts.move(begin); if (!ts.moveNext()) { return 0; } int balance = 0; do { Token<GroovyTokenId> token = ts.token(); TokenId id = token.id(); if (isBeginToken(id, doc, ts)) { balance++; } else if (id == GroovyTokenId.RBRACE) { balance--; } } while (ts.moveNext() && (ts.offset() <= end)); return balance; } catch (BadLocationException ble) { Exceptions.printStackTrace(ble); return 0; } }
Example #25
Source File: CalcLexer.java From netbeans with Apache License 2.0 | 5 votes |
/** * Fetch next token from underlying javacc tokenmanager. * <BR>The intId of the token that was found can be set * into the given tokenData parameter * by <CODE>TokenData.setTokenIntId()</CODE> in case there was * a valid token found. * <P>Token length of the fetched token can be set into tokenData * by <CODE>TokenData.setTokenLength()</CODE>. * If the token intId or length is not assigned in <CODE>fetchToken()</CODE> * it must be assigned later during either * {@link #ordinaryToken(OrdinaryTokenData)} * or {@link #extendedToken(ExtendedTokenData)} depending * which of these two gets called. * @param tokenData mutable info about the token being fetched. * @return true if a valid token was found or false * if there are no more tokens on the input (in which case a call * to <CODE>TokenData.setTokenIntId()</CODE> is not necessary). */ protected boolean fetchToken(TokenData tokenData) { try { // Get javacc token from tokenmanager org.netbeans.spi.lexer.javacc.Token javaccToken = tokenManager.getNextToken(); if (javaccToken != null) { int tokenKind = javaccToken.kind; tokenData.setTokenIntId(tokenKind); tokenData.setTokenLength(tokenData.getDefaultTokenLength()); return (tokenKind != CalcConstants.EOF); // EOF presents no characters } else { // javaccToken is null return false; // no more tokens from tokenManager } } catch (TokenMgrError e) { if (e.getErrorCode() == TokenMgrError.LEXICAL_ERROR) { if (tokenData.inExtendedToken()) { switch (tokenData.getExtendedTokenIntId()) { case CalcConstants.INCOMPLETE_ML_COMMENT: // This should only happen at the end of input tokenData.setTokenIntId(CalcConstants.EOF); // Lookahead will be non-zero (for no chars the lexical // error would not be thrown) tokenData.setTokenLength(tokenData.getTextLookahead()); return true; // there are chars -> valid token exists } } // Fallback for other ERRORS // System.out.println("msg=" + e.getMessage()); throw new IllegalStateException("Internal lexer error"); } else { // non-lexical type of error throw e; } } }
Example #26
Source File: InstantRenameActionTest.java From netbeans with Apache License 2.0 | 5 votes |
public void testNonPrivateClassWithPrivateConstructor() throws Exception { boolean[] wasResolved = new boolean[1]; Collection<Token> changePoints = performTest("package test; public class Test{public void run() {Test1 t = new Test1();} static class Test1{private Test1(){}}", 112 - 59, wasResolved); assertNull(changePoints); assertTrue(wasResolved[0]); }
Example #27
Source File: DTDLexer.java From netbeans with Apache License 2.0 | 5 votes |
private Token<DTDTokenId> nextProcessingContent() { int ch; boolean white = false; int whiteStart = -1; while ((ch = input.read()) != LexerInput.EOF) { if (ch == '?') { if (whiteStart > 0) { return tokenFactory.createToken(DTDTokenId.PI_CONTENT, whiteStart); } if (input.readLength() > 1) { input.backup(1); return tokenFactory.createToken(DTDTokenId.PI_CONTENT); } ch = input.read(); if (ch == '>') { return tokenFactory.createToken(DTDTokenId.SYMBOL); } // exit the processing content setState(ISI_INIT); return tokenFactory.createToken(DTDTokenId.ERROR); } if (Character.isWhitespace(ch)) { if (whiteStart == -1) { whiteStart = input.readLength(); } if (input.readLength() == 1) { white = true; } } else if (white) { // report whitespace & continue lexing in the same state: return tokenFactory.createToken(DTDTokenId.WS); } } return tokenFactory.createToken(DTDTokenId.PI_CONTENT); }
Example #28
Source File: SemanticHighlighterBase.java From netbeans with Apache License 2.0 | 5 votes |
@Override public Void visitUses(UsesTree tree, Void p) { tl.moveToOffset(sourcePositions.getStartPosition(info.getCompilationUnit(), tree)); Token t = firstIdentifierToken("uses"); //NOI18N if (t != null) { contextKeywords.add(t); } return super.visitUses(tree, p); }
Example #29
Source File: GspLexer.java From netbeans with Apache License 2.0 | 5 votes |
@Override public Token<GspTokenId> nextToken() { final GspTokenId tokenId = nextTokenId(); if (tokenId == null) { return null; // EOF } return tokenFactory.createToken(tokenId); }
Example #30
Source File: AngularJsDeclarationFinder.java From netbeans with Apache License 2.0 | 5 votes |
private OffsetRange isInObjectValueOfProperty(String propertyName, TokenSequence<? extends JsTokenId> ts, int caretOffset) { // e.g. check if "demo" is inside the object which is value of "components" property // components: { propA: 'a', propB: 'demo' } ts.move(caretOffset); if (ts.moveNext()) { JsTokenId id = ts.token().id(); if (id == JsTokenId.STRING) { OffsetRange result = new OffsetRange(ts.offset(), ts.offset() + ts.token().length()); ts.movePrevious(); Token<? extends JsTokenId> previous = LexUtilities.findPrevious(ts, Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL, JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT, JsTokenId.STRING_BEGIN, JsTokenId.STRING, JsTokenId.STRING_END, JsTokenId.OPERATOR_COMMA, JsTokenId.OPERATOR_COLON, JsTokenId.IDENTIFIER)); if (previous != null && previous.id() == JsTokenId.BRACKET_LEFT_CURLY && ts.movePrevious()) { previous = LexUtilities.findPrevious(ts, Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL, JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT)); if (previous != null && previous.id() == JsTokenId.OPERATOR_COLON && ts.movePrevious()) { previous = LexUtilities.findPrevious(ts, Arrays.asList(JsTokenId.WHITESPACE, JsTokenId.EOL, JsTokenId.BLOCK_COMMENT, JsTokenId.LINE_COMMENT)); if (previous != null && previous.id() == JsTokenId.IDENTIFIER && propertyName.equals(previous.text().toString())) { return result; } } } } } return null; }