Python pygments.lexers.PythonLexer() Examples

def __init__(self, **kwargs):
        stylename = kwargs.get('style_name', 'default')
        style = kwargs['style'] if 'style' in kwargs \
            else styles.get_style_by_name(stylename)
        self.formatter = BBCodeFormatter(style=style)
        self.lexer = lexers.PythonLexer()
        self.text_color = '#000000'
        self._label_cached = Label()
        self.use_text_color = True

        super(CodeInput, self).__init__(**kwargs)

        self._line_options = kw = self._get_line_options()
        self._label_cached = Label(**kw)
        # use text_color as foreground color
        text_color = kwargs.get('foreground_color')
        if text_color:
            self.text_color = get_hex_from_color(text_color)
        # set foreground to white to allow text colors to show
        # use text_color as the default color in bbcodes
        self.use_text_color = False
        self.foreground_color = [1, 1, 1, .999]
        if not kwargs.get('background_color'):
            self.background_color = [.9, .92, .92, 1] 

def __init__(self, **kwargs):
        stylename = kwargs.get('style_name', 'default')
        style = kwargs['style'] if 'style' in kwargs \
            else styles.get_style_by_name(stylename)
        self.formatter = BBCodeFormatter(style=style)
        self.lexer = lexers.PythonLexer()
        self.text_color = '#000000'
        self._label_cached = Label()
        self.use_text_color = True

        super(CodeInput, self).__init__(**kwargs)

        self._line_options = kw = self._get_line_options()
        self._label_cached = Label(**kw)
        # use text_color as foreground color
        text_color = kwargs.get('foreground_color')
        if text_color:
            self.text_color = get_hex_from_color(text_color)
        # set foreground to white to allow text colors to show
        # use text_color as the default color in bbcodes
        self.use_text_color = False
        self.foreground_color = [1, 1, 1, .999]
        if not kwargs.get('background_color'):
            self.background_color = [.9, .92, .92, 1] 

def __init__(self, *args, **kwargs):
        super(InterpreterInput, self).__init__(*args, **kwargs)

        self.register_event_type('on_request_completions')
        self.register_event_type('on_clear_completions')
        self.register_event_type('on_get_completions')

        if platform != 'android':
            from pygments.lexers import PythonLexer
            self.lexer = PythonLexer()

        App.get_running_app().bind(on_pause=self.on_pause)

    #     self.text = '''for i in range(5):
    # print(i)
    # time.sleep(1)''' 

def test_formatter_unicode_handling(cls):
    # test that the formatter supports encoding and Unicode
    tokens = list(lexers.PythonLexer(encoding='utf-8').
                  get_tokens("def f(): 'ä'"))

    try:
        inst = cls(encoding=None)
    except (ImportError, FontNotFound) as e:
        # some dependency or font not installed
        pytest.skip(str(e))

    if cls.name != 'Raw tokens':
        out = format(tokens, inst)
        if cls.unicodeoutput:
            assert type(out) is str, '%s: %r' % (cls, out)

        inst = cls(encoding='utf-8')
        out = format(tokens, inst)
        assert type(out) is bytes, '%s: %r' % (cls, out)
        # Cannot test for encoding, since formatters may have to escape
        # non-ASCII characters.
    else:
        inst = cls()
        out = format(tokens, inst)
        assert type(out) is bytes, '%s: %r' % (cls, out) 

def table_definition(table_name):
    """
    Get the source of a table function.

    If a table is registered DataFrame and not a function then all that is
    returned is {'type': 'dataframe'}.

    If the table is a registered function then the JSON returned has keys
    "type", "filename", "lineno", "text", and "html". "text" is the raw
    text of the function, "html" has been marked up by Pygments.

    """
    if orca.table_type(table_name) == 'dataframe':
        return jsonify(type='dataframe')

    filename, lineno, source = \
        orca.get_raw_table(table_name).func_source_data()

    html = highlight(source, PythonLexer(), HtmlFormatter())

    return jsonify(
        type='function', filename=filename, lineno=lineno, text=source,
        html=html) 

def display(cells):
    output = []

    for cell in cells:
        execution_count = (
            cell["execution_count"] if cell["execution_count"] is not None else " "
        )
        prompt = (
            Fore.GREEN
            + Style.BRIGHT
            + "In [{}]: ".format(execution_count)
            + Style.RESET_ALL
        )
        code = highlight(cell.source, PythonLexer(), TerminalTrueColorFormatter())
        output.append(prompt + code)

    return output 

def column_definition(table_name, col_name):
    """
    Get the source of a column function.

    If a column is a registered Series and not a function then all that is
    returned is {'type': 'series'}.

    If the column is a registered function then the JSON returned has keys
    "type", "filename", "lineno", "text", and "html". "text" is the raw
    text of the function, "html" has been marked up by Pygments.

    """
    col_type = orca.get_table(table_name).column_type(col_name)

    if col_type != 'function':
        return jsonify(type=col_type)

    filename, lineno, source = \
        orca.get_raw_column(table_name, col_name).func_source_data()

    html = highlight(source, PythonLexer(), HtmlFormatter())

    return jsonify(
        type='function', filename=filename, lineno=lineno, text=source,
        html=html) 

def injectable_definition(inj_name):
    """
    Get the source of an injectable function.

    If an injectable is a registered Python variable and not a function
    then all that is returned is {'type': 'variable'}.

    If the column is a registered function then the JSON returned has keys
    "type", "filename", "lineno", "text", and "html". "text" is the raw
    text of the function, "html" has been marked up by Pygments.

    """
    inj_type = orca.injectable_type(inj_name)

    if inj_type == 'variable':
        return jsonify(type='variable')
    else:
        filename, lineno, source = \
            orca.get_injectable_func_source_data(inj_name)
        html = highlight(source, PythonLexer(), HtmlFormatter())
        return jsonify(
            type='function', filename=filename, lineno=lineno, text=source,
            html=html) 

def code(self, session=None):
        all_errors = ""

        try:
            dag_id = request.args.get('dag_id')
            dag_orm = DagModel.get_dagmodel(dag_id, session=session)
            code = DagCode.get_code_by_fileloc(dag_orm.fileloc)
            html_code = Markup(highlight(
                code, lexers.PythonLexer(), HtmlFormatter(linenos=True)))

        except Exception as e:
            all_errors += (
                "Exception encountered during " +
                "dag_id retrieval/dag retrieval fallback/code highlighting:\n\n{}\n".format(e)
            )
            html_code = Markup('<p>Failed to load file.</p><p>Details: {}</p>').format(
                escape(all_errors))

        return self.render_template(
            'airflow/dag_code.html', html_code=html_code, dag=dag_orm, title=dag_id,
            root=request.args.get('root'),
            demo_mode=conf.getboolean('webserver', 'demo_mode'),
            wrapped=conf.getboolean('webserver', 'default_wrap')) 

def stacktraces():
    code = []
    for threadId, stack in sys._current_frames().items():
        code.append("\n# ThreadID: %s" % threadId)
        for filename, lineno, name, line in traceback.extract_stack(stack):
            code.append('File: "%s", line %d, in %s' % (filename, lineno, name))
            if line:
                code.append("  %s" % (line.strip()))
 
    return highlight("\n".join(code), PythonLexer(), HtmlFormatter(
      full=False,
      # style="native",
      noclasses=True,
    ))


# This part was made by nagylzs 

def testPython(self):
        """ Does the CompletionLexer work for Python?
        """
        lexer = CompletionLexer(PythonLexer())

        # Test simplest case.
        self.assertEqual(lexer.get_context("foo.bar.baz"),
                          [ "foo", "bar", "baz" ])

        # Test trailing period.
        self.assertEqual(lexer.get_context("foo.bar."), [ "foo", "bar", "" ])

        # Test with prompt present.
        self.assertEqual(lexer.get_context(">>> foo.bar.baz"),
                          [ "foo", "bar", "baz" ])

        # Test spacing in name.
        self.assertEqual(lexer.get_context("foo.bar. baz"), [ "baz" ])

        # Test parenthesis.
        self.assertEqual(lexer.get_context("foo("), []) 

def __init__(self, **kwargs):
        stylename = kwargs.get('style_name', 'default')
        style = kwargs['style'] if 'style' in kwargs \
            else styles.get_style_by_name(stylename)
        self.formatter = BBCodeFormatter(style=style)
        self.lexer = lexers.PythonLexer()
        self.text_color = '#000000'
        self._label_cached = Label()
        self.use_text_color = True

        super(CodeInput, self).__init__(**kwargs)

        self._line_options = kw = self._get_line_options()
        self._label_cached = Label(**kw)
        # use text_color as foreground color
        text_color = kwargs.get('foreground_color')
        if text_color:
            self.text_color = get_hex_from_color(text_color)
        # set foreground to white to allow text colors to show
        # use text_color as the default color in bbcodes
        self.use_text_color = False
        self.foreground_color = [1, 1, 1, .999]
        if not kwargs.get('background_color'):
            self.background_color = [.9, .92, .92, 1] 

def test_get_lexers():
    # test that the lexers functions work
    for func, args in [(lexers.get_lexer_by_name, ("python",)),
                       (lexers.get_lexer_for_filename, ("test.py",)),
                       (lexers.get_lexer_for_mimetype, ("text/x-python",)),
                       (lexers.guess_lexer, ("#!/usr/bin/python3 -O\nprint",)),
                       (lexers.guess_lexer_for_filename, ("a.py", "<%= @foo %>"))
                       ]:
        x = func(opt='val', *args)
        assert isinstance(x, lexers.PythonLexer)
        assert x.options["opt"] == "val"

    for cls, (_, lname, aliases, _, mimetypes) in lexers.LEXERS.items():
        assert cls == lexers.find_lexer_class(lname).__name__

        for alias in aliases:
            assert cls == lexers.get_lexer_by_name(alias).__class__.__name__

        for mimetype in mimetypes:
            assert cls == lexers.get_lexer_for_mimetype(mimetype).__class__.__name__

    try:
        lexers.get_lexer_by_name(None)
    except ClassNotFound:
        pass
    else:
        raise Exception 

def test_formatter_public_api(cls):
    # test that every formatter class has the correct public API
    ts = list(lexers.PythonLexer().get_tokens("def f(): pass"))
    string_out = StringIO()
    bytes_out = BytesIO()

    info = formatters.FORMATTERS[cls.__name__]
    assert len(info) == 5
    assert info[1], "missing formatter name"
    assert info[2], "missing formatter aliases"
    assert info[4], "missing formatter docstring"

    try:
        inst = cls(opt1="val1")
    except (ImportError, FontNotFound) as e:
        pytest.skip(str(e))

    try:
        inst.get_style_defs()
    except NotImplementedError:
        # may be raised by formatters for which it doesn't make sense
        pass

    if cls.unicodeoutput:
        inst.format(ts, string_out)
    else:
        inst.format(ts, bytes_out) 

def test_codetag(self):
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# BUG: text'
        tokens = list(lx.get_tokens(text))
        assert '# ' == tokens[0][1]
        assert 'BUG' == tokens[1][1] 

def test_bare_class_handler():
    from pygments.formatters import HtmlFormatter
    from pygments.lexers import PythonLexer
    try:
        lex('test\n', PythonLexer)
    except TypeError as e:
        assert 'lex() argument must be a lexer instance' in str(e)
    else:
        assert False, 'nothing raised'
    try:
        format([], HtmlFormatter)
    except TypeError as e:
        assert 'format() argument must be a formatter instance' in str(e)
    else:
        assert False, 'nothing raised' 

def test_basic(self):
        filters_args = [
            ('whitespace', {'spaces': True, 'tabs': True, 'newlines': True}),
            ('whitespace', {'wstokentype': False, 'spaces': True}),
            ('highlight', {'names': ['isinstance', 'lexers', 'x']}),
            ('codetagify', {'codetags': 'API'}),
            ('keywordcase', {'case': 'capitalize'}),
            ('raiseonerror', {}),
            ('gobble', {'n': 4}),
            ('tokenmerge', {}),
            ('symbols', {'lang': 'isabelle'}),
        ]
        for x, args in filters_args:
            lx = lexers.PythonLexer()
            lx.add_filter(x, **args)
            # We don't read as binary and decode, but instead read as text, as
            # we need consistent line endings. Otherwise we'll get \r\n on
            # Windows
            with open(TESTFILE, 'r', encoding='utf-8') as fp:
                text = fp.read()
            tokens = list(lx.get_tokens(text))
            assert all(isinstance(t[1], str) for t in tokens), \
                '%s filter did not return Unicode' % x
            roundtext = ''.join([t[1] for t in tokens])
            if x not in ('whitespace', 'keywordcase', 'gobble'):
                # these filters change the text
                assert roundtext == text, \
                    "lexer roundtrip with %s filter failed" % x 

def test_whitespace(self):
        lx = lexers.PythonLexer()
        lx.add_filter('whitespace', spaces='%')
        with open(TESTFILE, 'rb') as fp:
            text = fp.read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        assert ' ' not in lxtext 

def test_keywordcase(self):
        lx = lexers.PythonLexer()
        lx.add_filter('keywordcase', case='capitalize')
        with open(TESTFILE, 'rb') as fp:
            text = fp.read().decode('utf-8')
        lxtext = ''.join([t[1] for t in list(lx.get_tokens(text))])
        assert 'Def' in lxtext and 'Class' in lxtext 

def after_init(self):
        """Called after the original ``init`` is used to set extra fields."""
        self._lexer = PythonLexer()
        self._formatter = TerminalFormatter()

        # Logic:
        self._proccessed_filenames: List[str] = []
        self._error_count = 0 

def test_codetag_boundary(self):
        # ticket #368
        lx = lexers.PythonLexer()
        lx.add_filter('codetagify')
        text = u'# DEBUG: text'
        tokens = list(lx.get_tokens(text))
        assert '# DEBUG: text' == tokens[0][1] 

def test_normal():
    # test that cmdline gives the same output as library api
    from pygments.lexers import PythonLexer
    from pygments.formatters import HtmlFormatter
    filename = TESTFILE
    with open(filename, 'rb') as fp:
        code = fp.read()

    output = highlight(code, PythonLexer(), HtmlFormatter())

    o = check_success('-lpython', '-fhtml', filename)
    assert o == output 

def test_valid_output():
    with open(TESTFILE) as fp:
        tokensource = list(PythonLexer().get_tokens(fp.read()))
    fmt = LatexFormatter(full=True, encoding='latin1')

    handle, pathname = tempfile.mkstemp('.tex')
    # place all output files in /tmp too
    old_wd = os.getcwd()
    os.chdir(os.path.dirname(pathname))
    tfile = os.fdopen(handle, 'wb')
    fmt.format(tokensource, tfile)
    tfile.close()
    try:
        import subprocess
        po = subprocess.Popen(['latex', '-interaction=nonstopmode',
                               pathname], stdout=subprocess.PIPE)
        ret = po.wait()
        output = po.stdout.read()
        po.stdout.close()
    except OSError as e:
        # latex not available
        pytest.skip(str(e))
    else:
        if ret:
            print(output)
        assert not ret, 'latex run reported errors'

    os.unlink(pathname)
    os.chdir(old_wd) 

def _wiki_code_callback(matchobj):
    m = matchobj.groups()
    content = m[1]
    content = re.sub(r'^\s*', '', content, re.S)
    content = re.sub(r'\s*$', '', content, re.S)
    # code_class = m[0]
    formatter = HtmlFormatter(noclasses=True)
    code = highlight(content, PythonLexer(), formatter)
    return code 

def _pygmentsDemo(self):

        from pygments import highlight
        from pygments.lexers import PythonLexer
        from pygments.formatters import HtmlFormatter

        code = 'def foo(x="bar"): return True'

        lexer = PythonLexer()
        formatter = HtmlFormatter()
        codeHtml = highlight(code, lexer, formatter)

        doc = self.textEdit.document
        doc.defaultStyleSheet = formatter.get_style_defs()
        doc.setHtml(codeHtml) 

def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        self._document = self.document()
        self._formatter = HtmlFormatter(nowrap=True)
        self.set_style('default')
        if lexer is not None:
            self._lexer = lexer
        else:
            if PY3:
                self._lexer = Python3Lexer()
            else:
                self._lexer = PythonLexer() 

def print_contributer_lines(contributer, diff_infos):
    output = []
    for diff_info in diff_infos:
        lines = diff_info["reviewers"].get(contributer)
        if not lines:
            continue

        shl.print_section(shl.BOLD, diff_info["from_hash"], diff_info["file"], file=output)

        prev_line = None
        for line in lines:
            try:
                from pygments import highlight
                from pygments.lexers import PythonLexer
                from pygments.formatters import TerminalFormatter

                code = highlight(line["code_line"], PythonLexer(), TerminalFormatter())
            except ImportError:
                code = line["code_line"]

            cur_line = int(line["line_num"])

            if prev_line and prev_line + 1 < cur_line:
                output.append("    .")
                output.append("    .")
                output.append("    .")
            output.append("{line_num: >5}|\t{code_line}".format(line_num=line["line_num"], code_line=code.rstrip()))

            prev_line = cur_line

        output.append("\n\n")

    pydoc.pager("\n".join(output)) 

def show_code(func):
	if type(func) is str :
		code = func
	else :
		code = inspect.getsourcelines(func)[0]
		code = ''.join(code)
	print(highlight(code, PythonLexer(), Terminal256Formatter())) 

def __init__(self, parent, lexer=None):
        super(PygmentsHighlighter, self).__init__(parent)

        self._document = self.document()
        self._formatter = HtmlFormatter(nowrap=True)
        self.set_style('default')
        if lexer is not None:
            self._lexer = lexer
        else:
            if PY3:
                self._lexer = Python3Lexer()
            else:
                self._lexer = PythonLexer() 

def get_pygments():
    try:
        import pygments
        from pygments.lexers import PythonLexer
        from pygments.formatters import Terminal256Formatter
    except ImportError:  # pragma: no cover
        return None, None, None
    else:
        return pygments, PythonLexer(), Terminal256Formatter(style='vim')