Python token.ERRORTOKEN Examples

def tokeneater(self, toktype, toktext, xxx_todo_changeme, xxx_todo_changeme1, line):
        (srow, scol) = xxx_todo_changeme
        (erow, ecol) = xxx_todo_changeme1
        if toktype == token.ERRORTOKEN:
            raise RuntimeError("ErrorToken occured")
        if toktype in [token.NEWLINE, tokenize.NL]:
            self.output.write('\n')
            self.col = 0
        else:
            # map token type to a color group
            if token.LPAR <= toktype and toktype <= token.OP:
                toktype = token.OP
            elif toktype == token.NAME and keyword.iskeyword(toktext):
                toktype = _KEYWORD

            # restore whitespace
            assert scol >= self.col
            self.output.write(" "*(scol-self.col))

            try:
                tokclass = tokclasses[toktype]
            except KeyError:
                tokclass = None
            if self.tokclass is not None and tokclass != self.tokclass:
                self.output.write('</span>')
            if tokclass is not None and tokclass != self.tokclass:
                self.output.write('<span class="%s">' % tokclass)
            self.output.write(cgi.escape(toktext))
            self.tokclass = tokclass

            # calculate new column position
            self.col = scol + len(toktext)
            newline = toktext.rfind("\n")
            if newline != -1:
                self.col = len(toktext) - newline - 1 

def visit_Module(self, node):
    try:
      self.attr(
          node, 'bom',
          [lambda: self.tokens.eat_tokens(lambda t: t.type == token.ERRORTOKEN)],
          default='')
    except:
      pass
    self.generic_visit(node) 

def _sendHTMLText(self, toktype, toktext):
        numberlinks = self.numberlinks
        
        # If it is an error, set a red box around the bad tokens
        # older browsers should ignore it
        if toktype == ERRORTOKEN:
            style = ' style="border: solid 1.5pt #FF0000;"'
        else:
            style = ''
        # Get styles
        starttag, endtag, color = self._getHTMLStyles(toktype, toktext)
        # This is a hack to 'fix' multi-line  strings.
        # Multi-line strings are treated as only one token 
        # even though they can be several physical lines.
        # That makes it hard to spot the start of a line,
        # because at this level all we know about are tokens.
        
        if toktext.count(self.LINENUMHOLDER):
            # rip apart the string and separate it by line.
            # count lines and change all linenum token to line numbers.
            # embedded all the new font tags inside the current one.
            # Do this by ending the tag first then writing our new tags,
            # then starting another font tag exactly like the first one.
            if toktype == LINENUMBER:
                splittext = toktext.split(self.LINENUMHOLDER)
            else:    
                splittext = toktext.split(self.LINENUMHOLDER+' ')
            store = []
            store.append(splittext.pop(0))
            lstarttag, lendtag, lcolor = self._getHTMLStyles(LINENUMBER, toktext)
            count = len(splittext)
            for item in splittext:
                num =  self._getLineNumber()
                if numberlinks:
                    numstrip = num.strip()
                    content = '<a name="%s" href="#%s">%s</a>' \
                              %(numstrip,numstrip,num)
                else:
                    content = num
                if count <= 1:
                    endtag,starttag = '',''
                linenumber = ''.join([endtag,'<font color=', lcolor, '>',
                            lstarttag, content, lendtag, '</font>' ,starttag])
                store.append(linenumber+item)
            toktext = ''.join(store)
        # send text
        ## Output optimization
        # skip font tag if black text, but styles will still be sent. (b,u,i)
        if color !='#000000':
            startfont = '<font color="%s"%s>'%(color, style)
            endfont = '</font>'
        else:
            startfont, endfont = ('','')
        if toktype != LINENUMBER:
            self.out.write(''.join([startfont,starttag,
                                     toktext,endtag,endfont]))
        else:
            self.out.write(toktext)
        return 

def __init__(self, error_info):
        import tokenize

        super().__init__(error_info)

        self.tokens = []
        self.token_error = None

        if self.error_info["message"] == "EOL while scanning string literal":
            self.intro_text = (
                "You haven't properly closed the string on line %s." % self.error_info["lineno"]
                + "\n(If you want a multi-line string, then surround it with"
                + " `'''` or `\"\"\"` at both ends.)"
            )

        elif self.error_info["message"] == "EOF while scanning triple-quoted string literal":
            # lineno is not useful, as it is at the end of the file and user probably
            # didn't want the string to end there
            self.intro_text = "You haven't properly closed a triple-quoted string"

        else:
            if self.error_info["filename"] and os.path.isfile(self.error_info["filename"]):
                with open(self.error_info["filename"], mode="rb") as fp:
                    try:
                        for t in tokenize.tokenize(fp.readline):
                            self.tokens.append(t)
                    except tokenize.TokenError as e:
                        self.token_error = e
                    except IndentationError as e:
                        self.indentation_error = e

                if not self.tokens or self.tokens[-1].type not in [
                    token.ERRORTOKEN,
                    token.ENDMARKER,
                ]:
                    self.tokens.append(tokenize.TokenInfo(token.ERRORTOKEN, "", None, None, ""))
            else:
                self.tokens = []

            unbalanced = self._sug_unbalanced_parens()
            if unbalanced:
                self.intro_text = (
                    "Unbalanced parentheses, brackets or braces:\n\n" + unbalanced.body
                )
                self.intro_confidence = 5
            else:
                self.intro_text = "Python doesn't know how to read your program."

                if "^" in str(self.error_info):
                    self.intro_text += (
                        "\n\nSmall `^` in the original error message shows where it gave up,"
                        + " but the actual mistake can be before this."
                    )

                self.suggestions = [self._sug_missing_or_misplaced_colon()]