Python ast.parse() Examples

def compile_expression(exp):
    cached = cache.get(exp)
    if cached is not None:
        return cached
    _exp = ast.parse(exp)
    nodes = [node for node in ast.walk(_exp)]
    if len(nodes) < 2 or not isinstance(nodes[1], ast.Expr):
        raise ExpressionError("%s is not Expression" % exp)
    for node in nodes:
        if isinstance(node, ast.Call):
            raise ExpressionError("Call method is forbidden")
        if isinstance(node, ast.Lambda):
            raise ExpressionError("Lambda is strongly forbidden")
    result = compile(exp, '<string>', mode='eval')
    cache[exp] = result
    return result 

def _makeAST(f):
    # Need to look at the flags used to compile the original function f and
    # pass these same flags to the compile() function. This ensures that
    # syntax-changing __future__ imports like print_function work correctly.
    orig_f_co_flags = f.__code__.co_flags
    # co_flags can contain various internal flags that we can't pass to
    # compile(), so strip them out here
    valid_flags = 0
    for future_feature in __future__.all_feature_names:
        feature = getattr(__future__, future_feature)
        valid_flags |= feature.compiler_flag
    s = inspect.getsource(f)
    s = _dedent(s)
    # use compile instead of ast.parse so that additional flags can be passed
    flags = ast.PyCF_ONLY_AST | (orig_f_co_flags & valid_flags)
    tree = compile(s, filename='<unknown>', mode='exec',
        flags=flags, dont_inherit=True)
    # tree = ast.parse(s)
    tree.sourcefile = inspect.getsourcefile(f)
    tree.lineoffset = inspect.getsourcelines(f)[1] - 1
    return tree 

def _analyse_ast(str_code: str) -> Tuple[bool, Set[str]]:
    """
    Analyse the AST built from `str_definition`.

    Parameters
    ----------
    str_code: str
        A string containing a piece of valid python code : statement, expression or
         function definition (but without the `def ....` line).

    Returns
    -------
    has_return: bool
        True is the expression contains at least one return statement.
    variables: Set of str
        A set containing the identifiers of all variables used but not declared
        in `str_code`.

    """
    node = ast.parse(str_code)
    visitor = VarCounterVisitor()
    visitor.visit(node)
    return visitor.has_return, visitor.get_vars() 

def preprocess_method_body(source_code_obj):

    src = inspect.getsource(source_code_obj)

    ast_tree = ast.parse(src)

    visitor = RewriteDicts()
    ast_tree = visitor.visit(ast_tree)

    ast.fix_missing_locations(ast_tree)
    updated_code = compile(ast_tree, filename='<ast>', mode='exec')
    bc = Bytecode.from_code(updated_code)

    dlist = visitor.updated_dicts
    RewriteDicts.updated_dicts = []
    RewriteDicts.last_store_name = None

    block_code = get_code_block(bc)
    return block_code.arg, dlist 

def _grab_ast(repo, abspath):
  """Parses the Python file indicated by `abspath`.

  Args:
    * repo (RecipeRepo) - The repo which contains `abspath`. Used for error
      reporting.
    * abspath (str) - The absolute (native) path to the Python file to parse.

  Returns the Python AST object if the file exists and is parsable. Otherwise
  logs an error and returns None.
  """
  assert isinstance(repo, RecipeRepo), type(repo)
  relpath = os.path.relpath(abspath, repo.path)
  assert '..' not in relpath
  try:
    with open(abspath, 'rb') as f:
      return ast.parse(f.read(), relpath)
  except SyntaxError as ex:
    LOGGER.warn('skipping %s: bad syntax: %s', _to_posix(relpath), ex)
  except OSError as ex:
    LOGGER.warn('skipping %s: %s', _to_posix(relpath), ex)
  return None 

def parse_parameter(param):
  """Parses a recipe parameter into a Doc.Parameter.

  Args:
    * param (recipe_api.Property) - The parameter to parse.

  Returns Doc.Parameter.
  """
  assert isinstance(param, recipe_api.Property), type(param)
  default = None
  if param._default is not recipe_api.PROPERTY_SENTINEL:
    default = json.dumps(param._default)

  return doc.Doc.Parameter(
    docstring=param.help,
    kind=param.kind.schema_proto() if param.kind else None,
    default_json=default) 

def test_afterSubMult(self):
        'Tests after SubToMult pre-processing'

        tests = [("1 + 2 - 3", ast.BoolOp(ast.Add(), [ast.Num(1), ast.Num(2),
                                                      ast.BinOp(ast.Num(-1),
                                                                ast.Mult(),
                                                                ast.Num(3))])),
                 ("1 + 2 - 3 + 4", ast.BoolOp(ast.Add(),
                                              [ast.Num(1),
                                               ast.Num(2),
                                               ast.BinOp(ast.Num(-1),
                                                         ast.Mult(),
                                                         ast.Num(3)),
                                               ast.Num(4)])),
                 ("(1 + 2) - (3 + 4)",
                  ast.BoolOp(ast.Add(),
                             [ast.Num(1), ast.Num(2),
                              ast.BinOp(ast.Num(-1), ast.Mult(),
                                        ast.BinOp(ast.Num(3), ast.Add(),
                                                  ast.Num(4)))]))]
        for teststring, ref_ast in tests:
            test_ast = ast.parse(teststring, mode="eval").body
            test_ast = pre_processing.all_preprocessings(test_ast)
            test_ast = Flattening(ast.Add).visit(test_ast)
            self.assertTrue(Comparator().visit(test_ast, ref_ast)) 

def _get_modules(self, content) -> Set[str]:
        imports = set()
        tree = ast.parse(content)
        for node in ast.walk(tree):
            if isinstance(node, ast.Import):
                for subnode in node.names:
                    imports.add(subnode.name)
            elif isinstance(node, ast.ImportFrom) and node.level == 0:
                imports.add(node.module)
        modules = set()
        for module in imports:
            if not module:
                continue
            module = module.split('.', maxsplit=1)[0]
            if module in self.stdlib:
                continue
            module = self.aliases.get(module, module)
            modules.add(module)
        return modules 

def evaluate(self, node, filename=None):
        """
        Evaluate a source string or node, using ``filename`` when
        displaying errors.
        """
        if isinstance(node, string_types):
            self.source = node
            kwargs = {'mode': 'eval'}
            if filename:
                kwargs['filename'] = filename
            try:
                node = ast.parse(node, **kwargs)
            except SyntaxError as e:
                s = self.get_fragment(e.offset)
                raise SyntaxError('syntax error %s' % s)
        node_type = node.__class__.__name__.lower()
        handler = self.get_handler(node_type)
        if handler is None:
            if self.source is None:
                s = '(source not available)'
            else:
                s = self.get_fragment(node.col_offset)
            raise SyntaxError("don't know how to evaluate %r %s" % (
                node_type, s))
        return handler(node) 

def test_warning_calls():
        # combined "ignore" and stacklevel error
        base = Path(numpy.__file__).parent

        for path in base.rglob("*.py"):
            if base / "testing" in path.parents:
                continue
            if path == base / "__init__.py":
                continue
            if path == base / "random" / "__init__.py":
                continue
            # use tokenize to auto-detect encoding on systems where no
            # default encoding is defined (e.g. LANG='C')
            with tokenize.open(str(path)) as file:
                tree = ast.parse(file.read())
                FindFuncs(path).visit(tree) 

def _inferWaiter(gen):
    f = gen.gi_frame
    s = inspect.getsource(f)
    s = _dedent(s)
    root = ast.parse(s)
    root.symdict = f.f_globals.copy()
    root.symdict.update(f.f_locals)
    # print ast.dump(root)
    v = _YieldVisitor(root)
    v.visit(root)
    if v.kind == _kind.EDGE_TUPLE:
        return _EdgeTupleWaiter(gen)
    if v.kind == _kind.SIGNAL_TUPLE:
        return _SignalTupleWaiter(gen)
    if v.kind == _kind.DELAY:
        return _DelayWaiter(gen)
    if v.kind == _kind.EDGE:
        return _EdgeWaiter(gen)
    if v.kind == _kind.SIGNAL:
        return _SignalWaiter(gen)
    # default
    return _Waiter(gen) 

def get_top_imported_names(file: str) -> Set[str]:
    """Collect names imported in given file.

    We only collect top-level names, i.e. `from foo.bar import baz`
    will only add `foo` to the list.
    """
    if not file.endswith(".pyi"):
        return set()
    with open(os.path.join(file), "rb") as f:
        content = f.read()
    parsed = ast.parse(content)
    top_imported = set()
    for node in ast.walk(parsed):
        if isinstance(node, ast.Import):
            for name in node.names:
                top_imported.add(name.name.split('.')[0])
        elif isinstance(node, ast.ImportFrom):
            if node.level > 0:
                # Relative imports always refer to the current package.
                continue
            assert node.module
            top_imported.add(node.module.split('.')[0])
    return top_imported 

def evaluate(self, node, filename=None):
        """
        Evaluate a source string or node, using ``filename`` when
        displaying errors.
        """
        if isinstance(node, string_types):
            self.source = node
            kwargs = {'mode': 'eval'}
            if filename:
                kwargs['filename'] = filename
            try:
                node = ast.parse(node, **kwargs)
            except SyntaxError as e:
                s = self.get_fragment(e.offset)
                raise SyntaxError('syntax error %s' % s)
        node_type = node.__class__.__name__.lower()
        handler = self.get_handler(node_type)
        if handler is None:
            if self.source is None:
                s = '(source not available)'
            else:
                s = self.get_fragment(node.col_offset)
            raise SyntaxError("don't know how to evaluate %r %s" % (
                node_type, s))
        return handler(node) 

def easy_debug(code: str, should_exec=False, ctx=None):
    res = to_tagged_ast(parse(code).result)
    c = py_compile(res)
    print("-----------code")
    print(code)
    print("-----------Python")
    print(dis.dis(code))
    print("-----------YaPyPy")
    print(dis.dis(c))
    print("-----------astpretty")
    astpretty.pprint(ast.parse(code))
    print("----------- Python exec result")
    exec(code, ctx or {})
    print("-----------YaPyPy exec result")
    if should_exec:
        exec(c, ctx or {})
    else:
        print("\t(skip)") 

def compile_ast(self, source, filename = "<input>", symbol = "single"):
        # Here, we try to compile the relevant code. It may throw an
        # exception indicating that the command is not complete, in
        # which case we cannot proceed, but a full command will be
        # supplied eventually.
        compiled = code.compile_command(source, filename, symbol)
        # If the compilation succeeded, as indicated by its object not being
        # None, and no exception having occurred, parse it with AST and
        # store that.
        if compiled != None:
            self.latest_parsed = ast.parse(source, filename, symbol)
            CaptureExprs().visit(self.latest_parsed)
            # Since latest_parsed has been altered to capture values computed
            # but not assigned, store an unaltered copy for testing.
            self.clean_parsed = ast.parse(source, filename, symbol)
            
            return compile(self.latest_parsed, filename, symbol) 

def explodeCode(string):
    lines = string.splitlines()
    total = len(lines)
    if total == 0:
        return [None, None]
    a = ast.parse(string)
    forms = []
    totalForms = len(a.body)
    for i in range(totalForms):
        start = a.body[i].lineno
        if i >= totalForms - 1:
            end = total
        else:
            end = a.body[i+1].lineno - 1
        forms.append(toForm(lines, {"start": start, "end": end}))
    return forms 

def __init__(self, agenda):
        """ Create include from argument.

        Args:

            agenda (str, Agenda): Argument to the INCLUDE statement. This can
                either be a string or an Agenda object.
        """
        if type(agenda) == str:
            if not agenda in imports:
                self.agenda = Agenda.parse(agenda)
                imports[agenda] = self.agenda
            else:
                self.agenda = imports[agenda]
        elif type(agenda) == Agenda:
            self.agenda = agenda
        else:
            raise Exception("agenda argument must be either a controlfile"
            " name or a typhon.arts.workspace.agenda.Agenda object.") 

def mi_parameters(code, count_multi=True):
    '''Given a source code snippet, compute the necessary parameters to
    compute the Maintainability Index metric. These include:

        * the Halstead Volume
        * the Cyclomatic Complexity
        * the number of LLOC (Logical Lines of Code)
        * the percent of lines of comment

    :param multi: If True, then count multiline strings as comment lines as
        well. This is not always safe because Python multiline strings are not
        always docstrings.
    '''
    ast_node = ast.parse(code)
    raw = analyze(code)
    comments_lines = raw.comments + (raw.multi if count_multi else 0)
    comments = comments_lines / float(raw.sloc) * 100 if raw.sloc != 0 else 0
    return (h_visit_ast(ast_node).volume,
            ComplexityVisitor.from_ast(ast_node).total_complexity, raw.lloc,
            comments) 

def _wrap_parse(code: str, filename: str) -> 'ast.Call':
        """
        async wrapper is required to avoid await calls raising a SyntaxError
        """
        import ast
        from textwrap import indent

        code = 'async def wrapper():\n' + indent(code, ' ')
        return ast.parse(code, filename=filename).body[0].body[0].value 

def exec_then_eval(code, namespace=None):
    """Exec a code block & return evaluation of the last line"""
    # TODO: make this less brittle.
    namespace = namespace or {}

    block = ast.parse(code, mode='exec')
    last = ast.Expression(block.body.pop().value)

    exec(compile(block, '<string>', mode='exec'), namespace)
    return eval(compile(last, '<string>', mode='eval'), namespace) 

def test_simple(self):
        'Some basics tests'

        nbits = 8
        tests = [("x", "x"), ("x + 3 - 3", "x"), ("x + x*y - x*y", "x"),
                 ("x + 45 + 243", "x + 32")]
        for input_string, ref_string in tests:
            input_ast = ast.parse(input_string, mode='eval')
            ref_ast = ast.parse(ref_string, mode='eval')
            self.generic_test(input_ast, ref_ast, nbits)
        # test with ast.Module
        for input_string, ref_string in tests:
            input_ast = ast.parse(input_string)
            ref_ast = ast.parse(ref_string)
            self.generic_test(input_ast, ref_ast, nbits) 

def test_basicops(expr_string, refgraph):
    'Test if classic operators are correctly processed into DAG'
    random.seed(124)
    expr_ast = ast.parse(expr_string)
    visitor = dag_translator.DAGTranslator(expr_ast)
    visitor.visit(expr_ast)
    graph = visitor.graph
    assert str(graph.string()) == refgraph 

def test_withFunc(self):
        'Test simplification of arguments'

        nbits = 8
        tests = [("foo(x + x)", "foo(2*x)"), ("foo(x - x)", "foo(0)"),
                 ("foo(1 + 1)", "foo(2)"),
                 ("foo(x + 3*x + 45 - 4)", "foo(4*x + 41)"),
                 ("f(x + x + g(z + 3 + z) + x)", "f(3*x + g(2*z + 3))")]
        for input_string, ref_string in tests:
            input_ast = ast.parse(input_string, mode='eval')
            ref_ast = ast.parse(ref_string, mode='eval')
            self.generic_test(input_ast, ref_ast, nbits) 

def execute_controlfile(self, name):
        """ Execute controlfile or agenda on workspace.

        This method looks recursively for a controlfile with the given name in the current
        directory and the arts include path. If such a file has been found it will be parsed
        and executed on the workspace.

        Args:

            name(str): Name of the controlfile

        Raises:

            Exception: If parsing of the controlfile fails.

        Returns:

            The controlfile as parsed Agenda object.

        """

        if not name in imports:
            agenda = Agenda.parse(name)
            imports[name] = agenda
        else:
            agenda = imports[name]

        self.execute_agenda(agenda)

        return agenda 

def safe_eval(expr):
    """Safely evaluate string that may contain basic arithmetic
    """

    return _safe_eval_node(ast.parse(expr, mode="eval").body) 

def get_function_return_names(fct):
    """ Return variable name(s) or return statement of the given function """
    lines = inspect.getsourcelines(fct)
    outputs = None
    for line in lines[0][::-1]:
        stripped = line.strip()
        if "def" in stripped:
            # NOTE: This work as def is a reserved keyword which will trigger
            # invalid syntax if misused
            msg = "No return statement found in {}"
            raise PyungoError(msg.format(fct.__name__))
        ast_tree = ast.parse(stripped)
        for ast_node in ast.walk(ast_tree):
            if isinstance(ast_node, ast.Return):
                if isinstance(ast_node.value, ast.Name):
                    outputs = [ast_node.value.id]
                elif isinstance(ast_node.value, ast.Tuple):
                    outputs = [
                        elt.id
                        for elt in ast_node.value.elts
                        if isinstance(elt, ast.Name)
                    ]
                else:
                    name = ast_node.value.__class__.__name__
                    msg = (
                        "Variable name or Tuple of variable names are "
                        "expected, got {}".format(name)
                    )
                    raise PyungoError(msg)
                break
        if outputs:
            break
    return outputs 

def _parse_expr(token: Tokenizer):
    expr = ast.parse(token.value).body[0].value
    expr.lineno = token.lineno
    expr.col_offset = token.colno
    return expr 

def yapypy_debug(code: str, should_exec=False, ctx=None):
    res = to_tagged_ast(parse(code).result)
    c = py_compile(res)
    print("-----------Code")
    print(code)
    print("-----------YaPyPy")
    print(dis.dis(c))
    print("-----------YaPyPy exec result")
    if should_exec:
        exec(c, ctx or {})
    else:
        print("\t(skip)") 

def yapypy_test_code(code: str, should_exec=False, ctx=None):
    res = to_tagged_ast(parse(code).result)
    c = py_compile(res)
    if should_exec:
        exec(c, ctx or {}) 

def handleAnnotation(self, annotation, node):
        if isinstance(annotation, ast.Str):
            # Defer handling forward annotation.
            def handleForwardAnnotation():
                try:
                    tree = ast.parse(annotation.s)
                except SyntaxError:
                    self.report(
                        messages.ForwardAnnotationSyntaxError,
                        node,
                        annotation.s,
                    )
                    return

                body = tree.body
                if len(body) != 1 or not isinstance(body[0], ast.Expr):
                    self.report(
                        messages.ForwardAnnotationSyntaxError,
                        node,
                        annotation.s,
                    )
                    return

                parsed_annotation = tree.body[0].value
                for descendant in ast.walk(parsed_annotation):
                    ast.copy_location(descendant, annotation)

                self.handleNode(parsed_annotation, node)

            self.deferFunction(handleForwardAnnotation)
        else:
            self.handleNode(annotation, node)