Java Code Examples for com.intellij.openapi.editor.highlighter.HighlighterIterator#advance()

The following examples show how to use com.intellij.openapi.editor.highlighter.HighlighterIterator#advance() . You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may check out the related API usage on the sidebar.
Example 1
Source File: SimpleTokenSetQuoteHandler.java    From consulo with Apache License 2.0 6 votes vote down vote up
@Override
public boolean hasNonClosedLiteral(Editor editor, HighlighterIterator iterator, int offset) {
  int start = iterator.getStart();
  try {
    Document doc = editor.getDocument();
    CharSequence chars = doc.getCharsSequence();
    int lineEnd = doc.getLineEndOffset(doc.getLineNumber(offset));

    while (!iterator.atEnd() && iterator.getStart() < lineEnd) {
      IElementType tokenType = iterator.getTokenType();

      if (myLiteralTokenSet.contains(tokenType)) {
        if (isNonClosedLiteral(iterator, chars)) return true;
      }
      iterator.advance();
    }
  }
  finally {
    while(iterator.atEnd() || iterator.getStart() != start) iterator.retreat();
  }

  return false;
}
 
Example 2
Source File: CSharpQuoteHandler.java    From consulo-csharp with Apache License 2.0 6 votes vote down vote up
@Override
public boolean isOpeningQuote(HighlighterIterator iterator, int offset)
{
	boolean openingQuote = super.isOpeningQuote(iterator, offset);

	if(openingQuote)
	{
		// check escape next
		if(!iterator.atEnd())
		{
			iterator.retreat();

			if(!iterator.atEnd() && StringEscapesTokenTypes.STRING_LITERAL_ESCAPES.contains(iterator.getTokenType()))
			{
				openingQuote = false;
			}
			iterator.advance();
		}
	}
	return openingQuote;
}
 
Example 3
Source File: TodoItemNode.java    From consulo with Apache License 2.0 6 votes vote down vote up
private static void collectHighlights(@Nonnull List<? super HighlightedRegion> highlights, @Nonnull EditorHighlighter highlighter, int startOffset, int endOffset, int highlightOffsetShift) {
  HighlighterIterator iterator = highlighter.createIterator(startOffset);
  while (!iterator.atEnd()) {
    int start = Math.max(iterator.getStart(), startOffset);
    int end = Math.min(iterator.getEnd(), endOffset);
    if (start >= endOffset) break;

    TextAttributes attributes = iterator.getTextAttributes();
    int fontType = attributes.getFontType();
    if ((fontType & Font.BOLD) != 0) { // suppress bold attribute
      attributes = attributes.clone();
      attributes.setFontType(fontType & ~Font.BOLD);
    }
    HighlightedRegion region = new HighlightedRegion(highlightOffsetShift + start - startOffset, highlightOffsetShift + end - startOffset, attributes);
    highlights.add(region);
    iterator.advance();
  }
}
 
Example 4
Source File: XQueryQuoteHandler.java    From intellij-xquery with Apache License 2.0 6 votes vote down vote up
@Override
public boolean hasNonClosedLiteral(Editor editor, HighlighterIterator iterator, int offset) {
    int start = iterator.getStart();
    try {
        Document doc = editor.getDocument();
        CharSequence chars = doc.getCharsSequence();
        int lineEnd = doc.getLineEndOffset(doc.getLineNumber(offset));

        while (!iterator.atEnd() && iterator.getStart() < lineEnd) {
            IElementType tokenType = iterator.getTokenType();

            if (myLiteralTokenSet.contains(tokenType)) {
                if (isNonClosedLiteral(iterator, chars)) return true;
            }
            iterator.advance();
        }
    }
    finally {
        while(iterator.atEnd() || iterator.getStart() != start) iterator.retreat();
    }

    return false;
}
 
Example 5
Source File: HTMLTextPainter.java    From consulo with Apache License 2.0 6 votes vote down vote up
private void writeStyles(@NonNls final Writer writer) throws IOException {
  writer.write("<style type=\"text/css\">\n");
  writer.write(".ln { color: rgb(0,0,0); font-weight: normal; font-style: normal; }\n");
  HighlighterIterator hIterator = myHighlighter.createIterator(myOffset);
  while(!hIterator.atEnd()) {
    TextAttributes textAttributes = hIterator.getTextAttributes();
    if (!myStyleMap.containsKey(textAttributes)) {
      @NonNls String styleName = "s" + myStyleMap.size();
      myStyleMap.put(textAttributes, styleName);
      writer.write("." + styleName + " { ");
      final Color foreColor = textAttributes.getForegroundColor();
      if (foreColor != null) {
        writer.write("color: rgb(" + foreColor.getRed() + "," + foreColor.getGreen() + "," + foreColor.getBlue() + "); ");
      }
      if ((textAttributes.getFontType() & Font.BOLD) != 0) {
        writer.write("font-weight: bold; ");
      }
      if ((textAttributes.getFontType() & Font.ITALIC) != 0) {
        writer.write("font-style: italic; ");
      }
      writer.write("}\n");
    }
    hIterator.advance();
  }
  writer.write("</style>\n");
}
 
Example 6
Source File: ToggleCaseAction.java    From consulo with Apache License 2.0 6 votes vote down vote up
private static String toCase(Editor editor, int startOffset, int endOffset, final boolean lower) {
  CharSequence text = editor.getDocument().getImmutableCharSequence();
  EditorHighlighter highlighter;
  if (editor instanceof EditorEx) {
    highlighter = ((EditorEx)editor).getHighlighter();
  }
  else {
    highlighter = new EmptyEditorHighlighter(null);
    highlighter.setText(text);
  }
  HighlighterIterator iterator = highlighter.createIterator(startOffset);
  StringBuilder builder = new StringBuilder(endOffset - startOffset);
  while (!iterator.atEnd()) {
    int start = trim(iterator.getStart(), startOffset, endOffset);
    int end = trim(iterator.getEnd(), startOffset, endOffset);
    CharSequence fragment = text.subSequence(start, end);

    builder.append(iterator.getTokenType() == VALID_STRING_ESCAPE_TOKEN ? fragment :
                   lower ? fragment.toString().toLowerCase(Locale.getDefault()) :
                   fragment.toString().toUpperCase(Locale.getDefault()));

    if (end == endOffset) break;
    iterator.advance();
  }
  return builder.toString();
}
 
Example 7
Source File: HippieWordCompletionHandler.java    From consulo with Apache License 2.0 6 votes vote down vote up
private static void processWords(Editor editor, int startOffset, TokenProcessor processor) {
  CharSequence chars = editor.getDocument().getCharsSequence();
  HighlighterIterator iterator = ((EditorEx)editor).getHighlighter().createIterator(startOffset);
  while (!iterator.atEnd()) {
    int start = iterator.getStart();
    int end = iterator.getEnd();

    while (start < end) {
      int wordStart = start;
      while (wordStart < end && !isWordPart(chars.charAt(wordStart))) wordStart++;

      int wordEnd = wordStart;
      while (wordEnd < end && isWordPart(chars.charAt(wordEnd))) wordEnd++;

      if (wordEnd > wordStart && containsLetters(chars, wordStart, wordEnd) && !processor.processToken(wordStart, wordEnd)) {
        return;
      }
      start = wordEnd + 1;
    }
    iterator.advance();
  }
}
 
Example 8
Source File: BraceMatcherBasedSelectioner.java    From consulo with Apache License 2.0 5 votes vote down vote up
@Override
public List<TextRange> select(final PsiElement e, final CharSequence editorText, final int cursorOffset, final Editor editor) {
  final VirtualFile file = e.getContainingFile().getVirtualFile();
  final FileType fileType = file == null? null : file.getFileType();
  if (fileType == null) return super.select(e, editorText, cursorOffset, editor);
  final int textLength = editorText.length();
  final TextRange totalRange = e.getTextRange();
  final HighlighterIterator iterator = ((EditorEx)editor).getHighlighter().createIterator(totalRange.getStartOffset());
  final BraceMatcher braceMatcher = BraceMatchingUtil.getBraceMatcher(fileType, iterator);

  final ArrayList<TextRange> result = new ArrayList<TextRange>();
  final LinkedList<Trinity<Integer, Integer, IElementType>> stack = new LinkedList<Trinity<Integer, Integer, IElementType>>();
  while (!iterator.atEnd() && iterator.getStart() < totalRange.getEndOffset()) {
    final Trinity<Integer, Integer, IElementType> last;
    if (braceMatcher.isLBraceToken(iterator, editorText, fileType)) {
      stack.addLast(Trinity.create(iterator.getStart(), iterator.getEnd(), iterator.getTokenType()));
    }
    else if (braceMatcher.isRBraceToken(iterator, editorText, fileType)
        && !stack.isEmpty() && braceMatcher.isPairBraces((last = stack.getLast()).third, iterator.getTokenType())) {
      stack.removeLast();
      result.addAll(expandToWholeLine(editorText, new TextRange(last.first, iterator.getEnd())));
      int bodyStart = last.second;
      int bodyEnd = iterator.getStart();
      while (bodyStart < textLength && Character.isWhitespace(editorText.charAt(bodyStart))) bodyStart ++;
      while (bodyEnd > 0 && Character.isWhitespace(editorText.charAt(bodyEnd - 1))) bodyEnd --;
      result.addAll(expandToWholeLine(editorText, new TextRange(bodyStart, bodyEnd)));
    }
    iterator.advance();
  }
  result.add(e.getTextRange());
  return result;
}
 
Example 9
Source File: BraceMatchingUtilAdapter.java    From HighlightBracketPair with Apache License 2.0 5 votes vote down vote up
/**
 * find the right closest brace offset position
 *
 * @param iterator highlight iterator
 * @param rparenTokenType right token type to paired
 * @param fileText file text
 * @param fileType file type
 * @return offset
 */
public static int findRightRParen(HighlighterIterator iterator,
                                  IElementType rparenTokenType,
                                  CharSequence fileText,
                                  FileType fileType, boolean isBlockCaret) {
    int lastRbraceOffset = -1;
    int initOffset = iterator.atEnd() ? -1 : iterator.getStart();
    Stack<IElementType> braceStack = new Stack<>();
    for (; !iterator.atEnd(); iterator.advance()) {
        final IElementType tokenType = iterator.getTokenType();

        if (isRBraceToken(iterator, fileText, fileType)) {
            if (!braceStack.isEmpty()) {
                IElementType topToken = braceStack.pop();
                if (!isPairBraces(tokenType, topToken, fileType)) {
                    break; // unmatched braces
                }
            } else {
                if (tokenType == rparenTokenType) {
                    return iterator.getStart();
                } else {
                    break;
                }
            }
        } else if (isLBraceToken(iterator, fileText, fileType)) {
            if (isBlockCaret && initOffset == iterator.getStart())
                continue;
            else
                braceStack.push(iterator.getTokenType());
        }
    }

    return lastRbraceOffset;
}
 
Example 10
Source File: CustomFileTypeQuoteHandler.java    From consulo with Apache License 2.0 5 votes vote down vote up
@Override
public boolean hasNonClosedLiteral(Editor editor, HighlighterIterator iterator, int offset) {
  try {
    Document doc = editor.getDocument();
    CharSequence chars = doc.getCharsSequence();
    int lineEnd = doc.getLineEndOffset(doc.getLineNumber(offset));

    while (!iterator.atEnd() && iterator.getStart() < lineEnd) {
      IElementType tokenType = iterator.getTokenType();

      if (tokenType == CustomHighlighterTokenType.STRING ||
          tokenType == CustomHighlighterTokenType.SINGLE_QUOTED_STRING ||
          tokenType == CustomHighlighterTokenType.CHARACTER) {

        if (iterator.getStart() >= iterator.getEnd() - 1 ||
            chars.charAt(iterator.getEnd() - 1) != '\"' && chars.charAt(iterator.getEnd() - 1) != '\'') {
          return true;
        }
      }
      iterator.advance();
    }
  } finally {
    while (!iterator.atEnd() && iterator.getStart() != offset) iterator.retreat();
  }

  return false;
}
 
Example 11
Source File: BraceHighlightingHandler.java    From consulo with Apache License 2.0 5 votes vote down vote up
private void doHighlight(int offset, int originalOffset, @Nonnull FileType fileType) {
  if (myEditor.getFoldingModel().isOffsetCollapsed(offset)) return;

  HighlighterIterator iterator = getEditorHighlighter().createIterator(offset);
  final CharSequence chars = myDocument.getCharsSequence();

  if (BraceMatchingUtil.isLBraceToken(iterator, chars, fileType)) {
    IElementType tokenType = iterator.getTokenType();

    iterator.advance();
    if (!iterator.atEnd() && BraceMatchingUtil.isRBraceToken(iterator, chars, fileType)) {
      if (BraceMatchingUtil.isPairBraces(tokenType, iterator.getTokenType(), fileType) &&
          originalOffset == iterator.getStart()) return;
    }

    iterator.retreat();
    highlightLeftBrace(iterator, false, fileType);

    if (offset > 0) {
      iterator = getEditorHighlighter().createIterator(offset - 1);
      if (BraceMatchingUtil.isRBraceToken(iterator, chars, fileType)) {
        highlightRightBrace(iterator, fileType);
      }
    }
  }
  else if (BraceMatchingUtil.isRBraceToken(iterator, chars, fileType)) {
    highlightRightBrace(iterator, fileType);
  }
}
 
Example 12
Source File: BraceMatchingUtil.java    From consulo with Apache License 2.0 5 votes vote down vote up
public static int findRightmostRParen(HighlighterIterator iterator,
                                      IElementType rparenTokenType,
                                      CharSequence fileText,
                                      FileType fileType) {
  int lastRbraceOffset = -1;

  Stack<IElementType> braceStack = new Stack<IElementType>();
  for (; !iterator.atEnd(); iterator.advance()) {
    final IElementType tokenType = iterator.getTokenType();

    if (isRBraceToken(iterator, fileText, fileType)) {
      if (!braceStack.isEmpty()) {
        IElementType topToken = braceStack.pop();
        if (!isPairBraces(tokenType, topToken, fileType)) {
          break; // unmatched braces
        }
      }
      else {
        if (tokenType == rparenTokenType) {
          lastRbraceOffset = iterator.getStart();
        }
        else {
          break;
        }
      }
    }
    else if (isLBraceToken(iterator, fileText, fileType)) {
      braceStack.push(iterator.getTokenType());
    }
  }

  return lastRbraceOffset;
}
 
Example 13
Source File: EditorPainter.java    From consulo with Apache License 2.0 5 votes vote down vote up
private void paintBorderEffect(EditorHighlighter highlighter) {
  HighlighterIterator it = highlighter.createIterator(myStartOffset);
  while (!it.atEnd() && it.getStart() < myEndOffset) {
    TextAttributes attributes = it.getTextAttributes();
    EffectDescriptor borderDescriptor = getBorderDescriptor(attributes);
    if (borderDescriptor != null) {
      paintBorderEffect(it.getStart(), it.getEnd(), borderDescriptor);
    }
    it.advance();
  }
}
 
Example 14
Source File: EditorTestUtil.java    From consulo with Apache License 2.0 5 votes vote down vote up
public static List<IElementType> getAllTokens(EditorHighlighter highlighter) {
  List<IElementType> tokens = new ArrayList<IElementType>();
  HighlighterIterator iterator = highlighter.createIterator(0);
  while (!iterator.atEnd()) {
    tokens.add(iterator.getTokenType());
    iterator.advance();
  }
  return tokens;
}
 
Example 15
Source File: CSharpDocBraceMatcher.java    From consulo-csharp with Apache License 2.0 5 votes vote down vote up
private static boolean findEndTagStart(HighlighterIterator iterator)
{
	IElementType tokenType = iterator.getTokenType();
	int balance = 0;
	int count = 0;
	while(balance >= 0)
	{
		iterator.retreat();
		count++;
		if(iterator.atEnd())
		{
			break;
		}
		tokenType = iterator.getTokenType();
		if(tokenType == CSharpDocTokenType.XML_TAG_END || tokenType == CSharpDocTokenType.XML_EMPTY_ELEMENT_END)
		{
			balance++;
		}
		else if(tokenType == CSharpDocTokenType.XML_END_TAG_START || tokenType == CSharpDocTokenType.XML_START_TAG_START)
		{
			balance--;
		}
	}
	while(count-- > 0)
	{
		iterator.advance();
	}
	return tokenType == CSharpDocTokenType.XML_END_TAG_START;
}
 
Example 16
Source File: BashAnnotatorHighlightingTest.java    From BashSupport with Apache License 2.0 5 votes vote down vote up
private List<TextAttributes> collectComments() {
    List<TextAttributes> result = Lists.newArrayList();

    HighlighterIterator iterator = ((EditorEx) myFixture.getEditor()).getHighlighter().createIterator(0);
    while (!iterator.atEnd()) {
        if (iterator.getTokenType() == BashTokenTypes.COMMENT) {
            result.add(iterator.getTextAttributes());
        }
        iterator.advance();
    }
    return result;
}
 
Example 17
Source File: DustEnterHandler.java    From Intellij-Dust with MIT License 5 votes vote down vote up
/**
 * Checks to see if {@code Enter} has been typed while the caret is between an open and close tag pair.
 * @return true if between open and close tags, false otherwise
 */
private static boolean isBetweenHbTags(Editor editor, PsiFile file, int offset) {
  if (offset == 0) return false;
  CharSequence chars = editor.getDocument().getCharsSequence();
  if (chars.charAt(offset - 1) != '}') return false;

  EditorHighlighter highlighter = ((EditorEx)editor).getHighlighter();
  HighlighterIterator iterator = highlighter.createIterator(offset - 1);

  final PsiElement openerElement = file.findElementAt(iterator.getStart());

  PsiElement openTag = DustPsiUtil.findParentOpenTagElement(openerElement);

  if (openTag == null) {
    return false;
  }

  iterator.advance();

  if (iterator.atEnd()) {
    // no more tokens, so certainly no close tag
    return false;
  }

  final PsiElement closerElement = file.findElementAt(iterator.getStart());

  PsiElement closeTag = DustPsiUtil.findParentCloseTagElement(closerElement);

  // if we got this far, we're between open and close tags iff this is a close tag
  return closeTag != null;
}
 
Example 18
Source File: EnterAfterUnmatchedBraceHandler.java    From consulo with Apache License 2.0 4 votes vote down vote up
/**
 * Calculates number of unmatched left braces before the given offset.
 *
 * @param editor   target editor
 * @param offset   target offset
 * @param fileType target file type
 * @return number of unmatched braces before the given offset;
 * negative value if it's not possible to perform the calculation or if there are no unmatched left braces before
 * the given offset
 */
protected static int getUnmatchedLBracesNumberBefore(Editor editor, int offset, FileType fileType) {
  if (offset == 0) {
    return -1;
  }
  CharSequence chars = editor.getDocument().getCharsSequence();
  if (chars.charAt(offset - 1) != '{') {
    return -1;
  }

  EditorHighlighter highlighter = ((EditorEx)editor).getHighlighter();
  HighlighterIterator iterator = highlighter.createIterator(offset - 1);
  BraceMatcher braceMatcher = BraceMatchingUtil.getBraceMatcher(fileType, iterator);

  if (!braceMatcher.isLBraceToken(iterator, chars, fileType) || !braceMatcher.isStructuralBrace(iterator, chars, fileType)) {
    return -1;
  }

  Language language = iterator.getTokenType().getLanguage();

  iterator = highlighter.createIterator(0);
  int lBracesBeforeOffset = 0;
  int lBracesAfterOffset = 0;
  int rBracesBeforeOffset = 0;
  int rBracesAfterOffset = 0;
  for (; !iterator.atEnd(); iterator.advance()) {
    IElementType tokenType = iterator.getTokenType();
    if (!tokenType.getLanguage().equals(language) || !braceMatcher.isStructuralBrace(iterator, chars, fileType)) {
      continue;
    }

    boolean beforeOffset = iterator.getStart() < offset;

    if (braceMatcher.isLBraceToken(iterator, chars, fileType)) {
      if (beforeOffset) {
        lBracesBeforeOffset++;
      }
      else {
        lBracesAfterOffset++;
      }
    }
    else if (braceMatcher.isRBraceToken(iterator, chars, fileType)) {
      if (beforeOffset) {
        rBracesBeforeOffset++;
      }
      else {
        rBracesAfterOffset++;
      }
    }
  }

  return lBracesBeforeOffset - rBracesBeforeOffset - (rBracesAfterOffset - lBracesAfterOffset);
}
 
Example 19
Source File: TypedHandler.java    From consulo with Apache License 2.0 4 votes vote down vote up
private static void handleAfterLParen(@Nonnull Editor editor, @Nonnull FileType fileType, char lparenChar) {
  int offset = editor.getCaretModel().getOffset();
  HighlighterIterator iterator = ((EditorEx)editor).getHighlighter().createIterator(offset);
  boolean atEndOfDocument = offset == editor.getDocument().getTextLength();

  if (!atEndOfDocument) iterator.retreat();
  if (iterator.atEnd()) return;
  BraceMatcher braceMatcher = BraceMatchingUtil.getBraceMatcher(fileType, iterator);
  if (iterator.atEnd()) return;
  IElementType braceTokenType = iterator.getTokenType();
  final CharSequence fileText = editor.getDocument().getCharsSequence();
  if (!braceMatcher.isLBraceToken(iterator, fileText, fileType)) return;

  if (!iterator.atEnd()) {
    iterator.advance();

    if (!iterator.atEnd() && !BraceMatchingUtil.isPairedBracesAllowedBeforeTypeInFileType(braceTokenType, iterator.getTokenType(), fileType)) {
      return;
    }

    iterator.retreat();
  }

  int lparenOffset = BraceMatchingUtil.findLeftmostLParen(iterator, braceTokenType, fileText, fileType);
  if (lparenOffset < 0) lparenOffset = 0;

  iterator = ((EditorEx)editor).getHighlighter().createIterator(lparenOffset);
  boolean matched = BraceMatchingUtil.matchBrace(fileText, fileType, iterator, true, true);

  if (!matched) {
    String text;
    if (lparenChar == '(') {
      text = ")";
    }
    else if (lparenChar == '[') {
      text = "]";
    }
    else if (lparenChar == '<') {
      text = ">";
    }
    else if (lparenChar == '{') {
      text = "}";
    }
    else {
      throw new AssertionError("Unknown char " + lparenChar);
    }
    editor.getDocument().insertString(offset, text);
    TabOutScopesTracker.getInstance().registerEmptyScope(editor, offset);
  }
}
 
Example 20
Source File: InjectionRegistrarImpl.java    From consulo with Apache License 2.0 4 votes vote down vote up
@Nonnull
private static List<InjectedLanguageUtil.TokenInfo> obtainHighlightTokensFromLexer(@Nonnull LanguageVersion languageVersion,
                                                                                   @Nonnull CharSequence outChars,
                                                                                   @Nonnull VirtualFileWindow virtualFile,
                                                                                   @Nonnull Project project,
                                                                                   @Nonnull List<? extends PlaceInfo> placeInfos) {
  VirtualFile file = (VirtualFile)virtualFile;
  FileType fileType = file.getFileType();
  EditorHighlighterProvider provider = FileTypeEditorHighlighterProviders.INSTANCE.forFileType(fileType);
  EditorColorsScheme scheme = EditorColorsManager.getInstance().getGlobalScheme();
  EditorHighlighter highlighter = provider.getEditorHighlighter(project, fileType, file, scheme);
  highlighter.setText(outChars);
  HighlighterIterator iterator = highlighter.createIterator(0);
  int hostNum = -1;
  int prevHostEndOffset = 0;
  LiteralTextEscaper escaper = null;
  int prefixLength = 0;
  int suffixLength = 0;
  TextRange rangeInsideHost = null;
  int shredEndOffset = -1;
  List<InjectedLanguageUtil.TokenInfo> tokens = new ArrayList<>(outChars.length() / 5); // avg. token per 5 chars
  while (!iterator.atEnd()) {
    IElementType tokenType = iterator.getTokenType();
    TextRange range = new ProperTextRange(iterator.getStart(), iterator.getEnd());
    while (range != null && !range.isEmpty()) {
      if (range.getStartOffset() >= shredEndOffset) {
        hostNum++;
        PlaceInfo info = placeInfos.get(hostNum);
        shredEndOffset = info.rangeInDecodedPSI.getEndOffset();
        prevHostEndOffset = range.getStartOffset();
        escaper = info.myEscaper;
        rangeInsideHost = info.rangeInHostElement;
        prefixLength = info.prefix.length();
        suffixLength = info.suffix.length();
      }
      //in prefix/suffix or spills over to next fragment
      if (range.getStartOffset() < prevHostEndOffset + prefixLength) {
        range = new UnfairTextRange(prevHostEndOffset + prefixLength, range.getEndOffset());
      }
      TextRange spilled = null;
      if (range.getEndOffset() > shredEndOffset - suffixLength) {
        spilled = new UnfairTextRange(shredEndOffset, range.getEndOffset());
        range = new UnfairTextRange(range.getStartOffset(), shredEndOffset - suffixLength);
      }
      if (!range.isEmpty()) {
        int start = escaper.getOffsetInHost(range.getStartOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
        if (start == -1) start = rangeInsideHost.getStartOffset();
        int end = escaper.getOffsetInHost(range.getEndOffset() - prevHostEndOffset - prefixLength, rangeInsideHost);
        if (end == -1) {
          end = rangeInsideHost.getEndOffset();
          prevHostEndOffset = shredEndOffset;
        }
        ProperTextRange rangeInHost = new ProperTextRange(start, end);
        tokens.add(new InjectedLanguageUtil.TokenInfo(tokenType, rangeInHost, hostNum, iterator.getTextAttributes()));
      }
      range = spilled;
    }
    iterator.advance();
  }
  return tokens;
}