Python ast 模块，NodeVisitor() 实例源码

def get_decorators(cls):
    decorators = {}

    def visit_FunctionDef(node):
        decorators[node.name] = []
        for n in node.decorator_list:
            name = ''
            if isinstance(n, ast.Call):
                name = n.func.attr if isinstance(n.func, ast.Attribute) else n.func.id
            else:
                name = n.attr if isinstance(n, ast.Attribute) else n.id

            args = [a.s for a in n.args] if hasattr(n, 'args') else []
            decorators[node.name].append((name, args))

    node_iter = ast.NodeVisitor()
    node_iter.visit_FunctionDef = visit_FunctionDef
    _cls = cls if inspect.isclass(cls) else cls.__class__
    node_iter.visit(ast.parse(inspect.getsource(_cls)))
    return decorators

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def parse(self):
        """
        The main functionality. It parses a tree from :py:data:`self.path_or_stream`
        and uses :py:data:`ast.NodeVisitor` to iterate over the tree.

        :returns: self
        """
        self.stream.seek(0)
        try:
            with closing(self.stream) as stream:
                tree = ast.parse(stream.read())
        except (SyntaxError,):
            raise ParserException('File {file_} is not a python file'.format(
                file_=self.stream.name
            ))
        else:
            self.visit(tree)
        return self

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name:
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def get_set_vars(root):
    class GetSetVarsVisitor(ast.NodeVisitor):
        def __init__(self):
            self.set_vars = set()

        def visit_Call(self, node):
            if u.is_set_to_call(node):
                self.set_vars.add(get_var_name(node.func.value))

    vis = GetSetVarsVisitor()
    if isinstance(root, list):
        for stmt in root:
            vis.visit(stmt)
    else:
        vis.visit(root)
    return vis.set_vars

def get_decorators(cls):
    target = cls
    decorators = {}

    def visit_FunctionDef(node):
        decorators[node.name] = []
        for n in node.decorator_list:
            name = ''
            if isinstance(n, ast.Call):
                name = n.func.attr if isinstance(n.func, ast.Attribute) else n.func.id
            else:
                name = n.attr if isinstance(n, ast.Attribute) else n.id

            decorators[node.name].append(name)

    node_iter = ast.NodeVisitor()
    node_iter.visit_FunctionDef = visit_FunctionDef
    node_iter.visit(ast.parse(inspect.getsource(target)))
    return decorators

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].

    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def decorate_ast_with_source(_ast, code):

    class MyVisitor(ast.NodeVisitor):
        def visit(self, node):
            self.generic_visit(node)
            node.source_code = code

    MyVisitor().visit(_ast)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].
    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name:
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def visit(self, node):
        # type: (ast.AST) -> None
        inferred_type = self._binder.get_type_for_node(node)
        if isinstance(inferred_type, Boto3ClientMethodCallType):
            self.api_calls.setdefault(inferred_type.service_name, set()).add(
                inferred_type.method_name)
        ast.NodeVisitor.visit(self, node)

def visit(self, node):
        # type: (Any) -> None
        return ast.NodeVisitor.visit(self, node)

def parse_file(file_base, file_path, file_name):
    """Parse a python's files contents to extract API documentation."""
    src_path = os.path.join(file_base, file_path, file_name)
    print(src_path)

    with open(src_path) as fp:
        source = fp.read()
        tree = ast.parse(source)

        output_folder = "./docs/"
        dst_path = os.path.join(output_folder, file_path, file_name + ".md")

        os.makedirs(os.path.join(output_folder, file_path), exist_ok=True)

        with open(dst_path, "w") as output_file:

            class FuncLister(ast.NodeVisitor):
                def visit_FunctionDef(self, node):
                    """Function Visitor"""
                    output_file.write(get_function_markdown(node))
                    self.generic_visit(node)

                def visit_ClassDef(self, node):
                    """Class Visitor"""
                    output_file.write(get_class_markdown(node))
                    self.generic_visit(node)

            FuncLister().visit(tree)

def _get_import_cache(ipython):
    """Load a mapping of names to import statements from the IPython history.
    """

    import_cache = {}

    def _format_alias(alias):
        return ("import {0.name} as {0.asname}" if alias.asname
                else "import {0.name}").format(alias)

    class Visitor(ast.NodeVisitor):
        def visit_Import(self, node):
            for alias in node.names:
                (import_cache.setdefault(alias.asname or alias.name, set())
                 .add(_format_alias(alias)))

        def visit_ImportFrom(self, node):
            if node.level:  # Skip relative imports.
                return
            for alias in node.names:
                (import_cache.setdefault(alias.asname or alias.name, set())
                 .add("from {} {}".format(node.module, _format_alias(alias))))

    for _, _, entry in (
            ipython.history_manager.get_tail(
                ipython.history_load_length, raw=False)):
        try:
            parsed = ast.parse(entry)
        except SyntaxError:
            continue
        Visitor().visit(parsed)

    return import_cache

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def visit_Assign(self, node):
        for target in node.targets:
            if(isinstance(target,ast.Tuple)):#this is to handle variables declared on the same line
                for el in target.elts:
                    self.grabVar(el)
            else:
                self.grabVar(target)

        ast.NodeVisitor.generic_visit(self, node)#Make sure to run the original method so the AST module can do its thing

def visit_For(self,node):
        #Iterator variables don't get picked up in the Assign nodes, so we have to find them by looking at for loops
        self.grabVar(node.target)
        #We also keep track of where this for loop starts and ends
        self.forLoops[node.lineno] = {'start':node.lineno,'end':node.body[len(node.body)-1].lineno,'is_arg':False}

        ast.NodeVisitor.generic_visit(self, node)

def visit_FunctionDef(self,node):
        funcStart = node.lineno
        funcEnd = node.body[len(node.body)-1].lineno
        self.functions[node.name] = {'start':funcStart,'end':funcEnd}
        #Grab the arguments
        argList = node.args.args
        for argNode in argList:
            argName = argNode.arg
            varObj = {'name':argName,'line':node.lineno,'start':node.lineno,'col':-1,'end':funcEnd,'is_arg':True}
            self.varList.append(varObj)

        ast.NodeVisitor.generic_visit(self, node)

def visit_ClassDef(self, node):
        classStart = node.lineno
        classEnd = node.body[len(node.body)-1].lineno
        self.classes[node.name] = {'start':classStart,'end':classEnd}

        ast.NodeVisitor.generic_visit(self, node)

def visit_Assign(self, node):
        for target in node.targets:
            if(isinstance(target,ast.Tuple)):#this is to handle variables declared on the same line
                for el in target.elts:
                    self.grabVar(el)
            else:
                self.grabVar(target)
        ast.NodeVisitor.generic_visit(self, node)#Make sure to run the original method so the AST module can do its thing

def visit_FunctionDef(self,node):
        funcStart = node.lineno
        funcEnd = node.body[len(node.body)-1].lineno
        self.functions[node.name] = {'start':funcStart,'end':funcEnd}
        #Grab the arguments
        argList = node.args.args
        for argNode in argList:
            argName = argNode.arg
            varObj = {'name':argName,'line':node.lineno,'start':node.lineno,'col':-1,'end':funcEnd,'is_arg':True}
            self.varList.append(varObj)
        ast.NodeVisitor.generic_visit(self, node)

def visit_ImportFrom(self, node):
        if node.module and node.module.split(".")[0] == self.name:
            self.result = True

    # Based on ast.NodeVisitor.visit

def visit(self, node):
        # One change from ast.NodeVisitor.visit: do not fallback to generic_visit (we only care about top-level)
        method = 'visit_' + node.__class__.__name__
        visitor = getattr(self, method, None)
        if visitor is not None:
            visitor(node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def visit_Attribute(self, node):  # pylint: disable=invalid-name
    """Handle bare Attributes i.e. [tf.foo, tf.bar].
    Args:
      node: Node that is of type ast.Attribute
    """
    full_name = self._get_attribute_full_path(node)
    if full_name and full_name.startswith("tf."):
      self._rename_functions(node, full_name)
    if full_name in self._api_change_spec.change_to_function:
      if not hasattr(node, "is_function_for_call"):
        new_text = full_name + "()"
        self._file_edit.add("Changed %r to %r"%(full_name, new_text),
                            node.lineno, node.col_offset, full_name, new_text)

    ast.NodeVisitor.generic_visit(self, node)

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def ast_uses_varset(root, varset):
    class UsesVarsetVisitor(ast.NodeVisitor):
        def __init__(self):
            self.uses_varset = False

        def visit_Name(self, node):
            if node.id in varset:
                self.uses_varset = True

    vis = UsesVarsetVisitor()
    vis.visit(root)
    return vis.uses_varset

def get_method_by_name(module_node, name):
    class Visitor(ast.NodeVisitor):
        def visit_Module(self, node):
            self.result = None
            self.generic_visit(node)

        def visit_FunctionDef(self, node):
            if node.name == name:
                assert self.result is None, "More than one result in module"
                self.result = node
    v = Visitor()
    v.visit(module_node)
    return v.result

def get_class_node(module_node):
    class Visitor(ast.NodeVisitor):
        def visit_Module(self, node):
            self.result = None
            self.generic_visit(node)

        def visit_ClassDef(self, node):
            assert self.result is None, "More than one class in module"
            self.result = node
    v = Visitor()
    v.visit(module_node)
    return v.result

def get_variables(root):
    class Visitor(ast.NodeVisitor):
        def __init__(self):
            self.vars = set()

        def visit_Name(self, node):
            self.vars.add(node.id)

    var_visitor = Visitor()
    var_visitor.visit(root)
    return var_visitor.vars

def split_params_from_model(self, root):
        class CollectParamsVisitor(ast.NodeVisitor):
            def visit_Module(self_c, node):
                self_c.param_statements = []
                self_c.param_dict = None
                self_c.generic_visit(node)

            def visit_Assign(self_c, node):
                if u.is_param_declaration(node):
                    self_c.param_statements.append(node)
                elif u.is_self_params_assignment(node):
                    self_c.param_dict = node
        cpv = CollectParamsVisitor()
        cpv.visit(root)

        class RemoveParamsTransformer(self.MyTransformer):
            def visit_FunctionDef(self_r, node):
                """ Don't recurse into user defined functions """
                return node

            def visit_Assign(self_r, node):
                if u.is_param_declaration(node):
                    return []
                elif u.is_self_params_assignment(node):
                    return []
                else:
                    return node
        rpt = RemoveParamsTransformer()
        root = rpt.visit(root)

        return root, cpv.param_statements, cpv.param_dict

def scan_opcodes_cli(self, co):
        import ast
        with open(co.co_filename, 'rU') as f:
            nodes = ast.parse(f.read(), co.co_filename)

        items = []

        class ModuleFinderVisitor(ast.NodeVisitor):
            def visit_Assign(self, node):
                for x in node.targets:
                    if isinstance(x, ast.Subscript):
                        if isinstance(x.value, ast.Name):
                            items.append(("store", (x.value.id, )))
                        elif isinstance(x.value, ast.Attribute):
                            items.append(("store", (x.value.attr, )))
                        else:
                            print 'Unknown in store: %s' % type(x.value).__name__
                    elif isinstance(x, ast.Name):
                        items.append(("store", (x.id, )))

            def visit_Import(self, node):
                items.extend([("import", (None, x.name)) for x in node.names])

            def visit_ImportFrom(self, node):
                if node.level == 1:
                    items.append(("relative_import", (node.level, [x.name for x in node.names], node.module)))
                else:
                    items.extend([("import", ([x.name for x in node.names], node.module))])

        v = ModuleFinderVisitor()
        v.visit(nodes)

        for what, args in items:
            yield what, args

def _find_decorators(cls):
        result = dict()

        def visit_FunctionDef(node):
            result[node.name] = [ast.dump(e) for e in node.decorator_list]

        v = ast.NodeVisitor()

        v.visit_FunctionDef = visit_FunctionDef
        v.visit(compile(inspect.getsource(cls), '?', 'exec', ast.PyCF_ONLY_AST))

        return result

def visit_Attribute(self, node):
            ast.NodeVisitor.generic_visit(self, node)
            self.ls.append(node.attr)

def visit_Call(self, node):
            p = ParseCall()
            p.visit(node.func)
            ast.NodeVisitor.generic_visit(self, node)

            if p.ls[-1] == 'simplefilter' or p.ls[-1] == 'filterwarnings':
                if node.args[0].s == "ignore":
                    raise AssertionError(
                        "ignore filter should not be used; found in "
                        "{} on line {}".format(self.__filename, node.lineno))

            if p.ls[-1] == 'warn' and (
                    len(p.ls) == 1 or p.ls[-2] == 'warnings'):

                if "testing/tests/test_warnings.py" is self.__filename:
                    # This file
                    return

                # See if stacklevel exists:
                if len(node.args) == 3:
                    return
                args = {kw.arg for kw in node.keywords}
                if "stacklevel" in args:
                    return
                raise AssertionError(
                    "warnings should have an appropriate stacklevel; found in "
                    "{} on line {}".format(self.__filename, node.lineno))

def __init__(self, fname, cache):
        ast.NodeVisitor.__init__(self)
        self.fname = fname
        self.cache = cache
        self.class_stack = []

def generic_visit(self, node):
    ast.NodeVisitor.generic_visit(self, node)

def visit_node(self, node):
        self.path.append(node.name)
        node_name = ".".join(self.path)
        if node_name in self.filters:
            self.nodes[node_name] = node
        ast.NodeVisitor.generic_visit(self, node)
        self.path.pop()
        self.ended_nodes.append(node)

def __init__(self, calculator):

        super(NodeVisitor, self).__init__()
        self._calculator = calculator
        self._dependencies = []

    ##############################################

def dependencies(self):
        return self._dependencies

    ##############################################

    # def visit(self, node):

    #     print(node)
    #     super(NodeVisitor, self).visit(node)

    ##############################################

def _compile(self):

        expression = self._expression
        self._logger.info("expression '{}'".format(expression))

        # Python don't accept identifier starting with @
        # https://docs.python.org/3.5/reference/lexical_analysis.html#identifiers
        if '@' in expression:
            custom_measurements = []
            start = 0
            while True:
                name, start = self._find_name('@', start)
                if name is None:
                    break
                else:
                    custom_measurements.append(name)
            for measurement in custom_measurements:
                expression = self.expression.replace(measurement, self._calculator.measurements[measurement].name)

        functions = []
        for function in (
                'Angle1Spl_',
                'Angle2Spl_',
                'AngleLine_',
                'CurrentLength',
                'C1LengthSpl_',
                'C2LengthSpl_',
                'Line_',
                'Spl_',
        ):
            start = 0
            while True:
                name, start = self._find_name(function, start)
                if name is None:
                    break
                else:
                    functions.append(name)
        # self._logger.info('Functions ' + str(functions))
        for function_call in functions:
            parts = function_call.split('_')
            function = parts[0]
            args = parts[1:]
            pythonised_function = '__calculator__._function_' + function + '(' + ', '.join(["'{}'".format(x) for x in args]) + ')'
            # self._logger.info('Function {} {} -> {}'.format(function, args, pythonised_function))
            expression = expression.replace(function_call, pythonised_function)

        self._logger.info("Pythonised expression '{}'".format(expression))

        # Fixme: What is the (supported) grammar ?
        # http://beltoforion.de/article.php?a=muparser
        # http://beltoforion.de/article.php?a=muparserx
        self._ast = ast.parse(expression, mode='eval')
        node_visitor = NodeVisitor(self._calculator)
        node_visitor.generic_visit(self._ast)
        self._dependencies = node_visitor.dependencies
        self._code = compile(self._ast, '<string>', mode='eval')
        # print('AST', astunparse.dump(self._ast))
        # print('AST -> Python', astunparse.unparse(self._ast))
        ## print('AST -> Python', astor.to_source(self._ast.body))

    ##############################################