Python pkgutil.iter_modules() Examples

def _execute_performance(self, naarad_obj):
    """
    Executes naarad

    :param naarad_obj:
    :return:
    """
    naarad_obj.analyze(self._logs_dir, self._output_dir)

    if ('matplotlib' in [tuple_[1] for tuple_ in iter_modules()]) and len(self.configs) > 1:
      prevConfig = self.configs[0]
      if naarad_obj._output_directory is None:
        naarad_obj._output_directory = self._output_dir
      for curConfig in self.configs[1:]:
        if not curConfig.naarad_id is None:
          naarad_obj.diff(curConfig.naarad_id, prevConfig.naarad_id)
          prevConfig = curConfig

    tests = [test for test in self.tests if not isinstance(test, list)] +\
            [individual_test for test in self.tests if isinstance(test, list) for individual_test in test]
    for test in tests:
      if test.naarad_id is not None:
        test.naarad_stats = naarad_obj.get_stats_data(test.naarad_id)
        test.sla_objs = self._convert_naarad_slas_to_list(naarad_obj.get_sla_data(test.naarad_id)) 

def list_available_algorithms() -> List[str]:
    """
    The list of available DCOP algorithms.

    Returns
    -------
    a list of str
    """
    exclude_list = {'generic_computations', 'graphs', 'objects'}
    algorithms = []

    root_algo = import_module('pydcop.algorithms')
    for importer, modname, ispkg in pkgutil.iter_modules(root_algo.__path__,
                                                         ''):
        if modname not in exclude_list:
            algorithms.append(modname)

    return algorithms 

def dynamic_load_return_mods():
#---------------------------------------

#    path = os.path.join(os.getcwd(), "plugins/policies/select")
    path = os.path.join(os.path.dirname(__file__), "plugins/policies/select")
    path = path.replace("src/pluginSystem/", "")
    path = path.replace("src\pluginSystem\\", "")

    modules = pkgutil.iter_modules(path=[path])
    
#    print ""
        
    mode_names = []        
    for loader, mod_name, ispkg in modules:
        mode_names.append(mod_name)
        
    return mode_names


#--------------------------------------- 

def dynamic_load_return_mods():
#---------------------------------------

    #---- we want to return mod names so we can control output
    
#    path = os.path.join(os.path.dirname(__file__), "applicationSignatures")
#    path = os.path.join(os.getcwd(), "plugins/policies/mutate")
    path = os.path.join(os.path.dirname(__file__), "plugins/policies/mutate")
    path = path.replace("src/pluginSystem/", "")
    path = path.replace("src\pluginSystem\\", "")

    modules = pkgutil.iter_modules(path=[path])
            
    mode_names = []        
    for loader, mod_name, ispkg in modules:
        mode_names.append(mod_name)
        
    return mode_names


#--------------------------------------- 

def dynamic_load_return_mods():
#---------------------------------------

#    path = os.path.join(os.getcwd(), "plugins/substitutions")
    path = os.path.join(os.path.dirname(__file__), "plugins/substitutions")
    path = path.replace("src/pluginSystem/", "")
    path = path.replace("src\pluginSystem\\", "")

    modules = pkgutil.iter_modules(path=[path])
    
    mode_names = []        
    for loader, mod_name, ispkg in modules:
        mode_names.append(mod_name)
        
    return mode_names


#--------------------------------------- 

def load_submodules(base_pkg=None):
    if base_pkg is None:
        import dionaea as base_pkg

    prefix = base_pkg.__name__ + "."
    for importer, modname, ispkg in pkgutil.iter_modules(base_pkg.__path__, prefix):
        if modname in loaded_submodules:
            continue

        logger.info("Import module %s", modname)
        try:
            __import__(modname, fromlist="dummy")
        except Exception as e:
            logger.warning("Error loading module: {}".format(str(e)))

            for msg in traceback.format_exc().split("\n"):
                logger.warning(msg.rstrip())

        loaded_submodules.append(modname) 

def list_fuzzers():
    """ Dump all of the fuzzers currently loadable
    """

    msg("Currently loaded fuzzers")
    try:
        load = importlib.import_module("src.fuzzers")
        modules = list(pkgutil.iter_modules(load.__path__))
        for mod in modules:

            dp = mod[0].find_module(mod[1]).load_module(mod[1])
            for e in dir(dp):
                x = getattr(dp, e)
                if isclass(x) and e != "Fuzzer" and issubclass(x, src.core.fuzzer.Fuzzer):
                    msg("  %s (%s)" % (x().name, mod[1]))

    except Exception, e:
        msg("Failed to list modules: %s" % e, ERROR) 

def load_backend(backend_name):
    # Look for a fully qualified database backend name
    try:
        return import_module('%s.base' % backend_name)
    except ImportError as e_user:
        # The database backend wasn't found. Display a helpful error message
        # listing all possible (built-in) database backends.
        backend_dir = os.path.join(os.path.dirname(upath(__file__)), 'backends')
        try:
            builtin_backends = [
                name for _, name, ispkg in pkgutil.iter_modules([backend_dir])
                if ispkg and name != 'dummy']
        except EnvironmentError:
            builtin_backends = []
        if backend_name not in ['django.db.backends.%s' % b for b in
                                builtin_backends]:
            backend_reprs = map(repr, sorted(builtin_backends))
            error_msg = ("%r isn't an available database backend.\n"
                         "Try using 'django.db.backends.XXX', where XXX "
                         "is one of:\n    %s\nError was: %s" %
                         (backend_name, ", ".join(backend_reprs), e_user))
            raise ImproperlyConfigured(error_msg)
        else:
            # If there's some other error, this must be an error in Django
            raise 

def load_fuzzer():
    """ Dynamically load the specified fuzzer 
    """

    load = importlib.import_module("src.fuzzers")
    modules = list(pkgutil.iter_modules(load.__path__))
    for mod in modules:
        
        # pull up the module and iterate over its components, 
        # stop once we find the correct class to invoke.
        dp = mod[0].find_module(mod[1]).load_module(mod[1])
        for e in dir(dp):

            x = getattr(dp, e)
            if e and e != "Fuzzer" and config.SESSION_FUZZER in e and\
               issubclass(x, Fuzzer):
                utility.msg("Loaded fuzzer %s" % e, LOG)
                return x()

    return None 

def load_plugins(self, path):
        """Attempt to load plugins from the path specified.

        :param path: full path to a directory where to look for plugins

        """
        for loader, name, ispkg in pkgutil.iter_modules([path]):
            file, pathname, desc = imp.find_module(name, [path])
            # set the full plugin module name
            module_name = "streamlink.plugin.{0}".format(name)

            try:
                self.load_plugin(module_name, file, pathname, desc)
            except Exception:
                sys.stderr.write("Failed to load plugin {0}:\n".format(name))
                print_small_exception("load_plugin")

                continue 

def getunpackers():
    """Scans the unpackers dir, finds unpackers and add them to UNPACKERS list.
    An unpacker will be loaded only if it is a valid python module (name must
    adhere to naming conventions) and it is not blacklisted (i.e. inserted
    into BLACKLIST."""
    path = __path__
    prefix = __name__ + '.'
    unpackers = []
    interface = ['unpack', 'detect', 'PRIORITY']
    for _importer, modname, _ispkg in pkgutil.iter_modules(path, prefix):
        if 'tests' not in modname and modname not in BLACKLIST:
            try:
                module = __import__(modname, fromlist=interface)
            except ImportError:
                raise UnpackingError('Bad unpacker: %s' % modname)
            else:
                unpackers.append(module)

    return sorted(unpackers, key = lambda mod: mod.PRIORITY) 

def _iter_all_modules(package, prefix=""):
    """
    Iterates over the names of all modules that can be found in the given
    package, recursively.
    Example:
        _iter_all_modules(_pytest) ->
            ['_pytest.assertion.newinterpret',
             '_pytest.capture',
             '_pytest.core',
             ...
            ]
    """
    import os
    import pkgutil

    if type(package) is not str:
        path, prefix = package.__path__[0], package.__name__ + "."
    else:
        path = package
    for _, name, is_package in pkgutil.iter_modules([path]):
        if is_package:
            for m in _iter_all_modules(os.path.join(path, name), prefix=name + "."):
                yield prefix + m
        else:
            yield prefix + name 

def _iter_all_modules(package, prefix=""):
    """
    Iterates over the names of all modules that can be found in the given
    package, recursively.
    Example:
        _iter_all_modules(_pytest) ->
            ['_pytest.assertion.newinterpret',
             '_pytest.capture',
             '_pytest.core',
             ...
            ]
    """
    import os
    import pkgutil

    if type(package) is not str:
        path, prefix = package.__path__[0], package.__name__ + "."
    else:
        path = package
    for _, name, is_package in pkgutil.iter_modules([path]):
        if is_package:
            for m in _iter_all_modules(os.path.join(path, name), prefix=name + "."):
                yield prefix + m
        else:
            yield prefix + name 

def _get_module_names(self, search_path=None):
        """
        Get the names of all modules in the search_path. This means file names
        and not names defined in the files.
        """

        names = []
        # add builtin module names
        if search_path is None:
            names += [self._generate_name(name) for name in sys.builtin_module_names]

        if search_path is None:
            search_path = self._importer.sys_path_with_modifications()
        for module_loader, name, is_pkg in pkgutil.iter_modules(search_path):
            names.append(self._generate_name(name))
        return names 

def init_plugins(config):
    import imp, pkgutil, __builtin__, os
    global plugins

    if __builtin__.use_local_modules:
        fp, pathname, description = imp.find_module('plugins')
        plugin_names = [name for a, name, b in pkgutil.iter_modules([pathname])]
        plugin_names = filter( lambda name: os.path.exists(os.path.join(pathname,name+'.py')), plugin_names)
        imp.load_module('chainkey_plugins', fp, pathname, description)
        plugin_modules = map(lambda name: imp.load_source('chainkey_plugins.'+name, os.path.join(pathname,name+'.py')), plugin_names)
    else:
        import chainkey_plugins
        plugin_names = [name for a, name, b in pkgutil.iter_modules(chainkey_plugins.__path__)]
        plugin_modules = [ __import__('chainkey_plugins.'+name, fromlist=['chainkey_plugins']) for name in plugin_names]

    for name, p in zip(plugin_names, plugin_modules):
        try:
            plugins.append( p.Plugin(config, name) )
        except Exception:
            print_msg(_("Error: cannot initialize plugin"),p)
            traceback.print_exc(file=sys.stdout) 

def _sub_modules(self):
        """
        Lists modules in the directory of this module (if this module is a
        package).
        """
        path = self._module.path
        names = []
        if path is not None and path.endswith(os.path.sep + '__init__.py'):
            mods = pkgutil.iter_modules([os.path.dirname(path)])
            for module_loader, name, is_pkg in mods:
                name = helpers.FakeName(name)
                # It's obviously a relative import to the current module.
                imp = helpers.FakeImport(name, self, level=1)
                name.parent = imp
                names.append(name)
        return names 

def get_modules(module_path):

    modules = [name for _, name, _ in pkgutil.iter_modules([module_path])]
    if "templates" in modules:
        modules.pop(modules.index("templates"))

    return sorted(modules) 

def _import_submodules_from_package(package):
    import pkgutil

    modules = []
    for importer, modname, ispkg in pkgutil.iter_modules(package.__path__,
                                                         prefix=package.__name__ + "."):
        modules.append(__import__(modname, fromlist="dummy"))
    return modules 

def load_backend(backend_name):
    """
    Return a database backend's "base" module given a fully qualified database
    backend name, or raise an error if it doesn't exist.
    """
    # This backend was renamed in Django 1.9.
    if backend_name == 'django.db.backends.postgresql_psycopg2':
        backend_name = 'django.db.backends.postgresql'

    try:
        return import_module('%s.base' % backend_name)
    except ImportError as e_user:
        # The database backend wasn't found. Display a helpful error message
        # listing all built-in database backends.
        backend_dir = os.path.join(os.path.dirname(__file__), 'backends')
        builtin_backends = [
            name for _, name, ispkg in pkgutil.iter_modules([backend_dir])
            if ispkg and name not in {'base', 'dummy', 'postgresql_psycopg2'}
        ]
        if backend_name not in ['django.db.backends.%s' % b for b in builtin_backends]:
            backend_reprs = map(repr, sorted(builtin_backends))
            raise ImproperlyConfigured(
                "%r isn't an available database backend.\n"
                "Try using 'django.db.backends.XXX', where XXX is one of:\n"
                "    %s" % (backend_name, ", ".join(backend_reprs))
            ) from e_user
        else:
            # If there's some other error, this must be an error in Django
            raise 

def __new__(mcs, name, bases, dict):
        plugin_path = os.path.dirname(streamlink.plugins.__file__)
        plugins = []
        for loader, pname, ispkg in pkgutil.iter_modules([plugin_path]):
            module = load_module(pname, plugin_path)
            if hasattr(module, "__plugin__"):
                plugins.append((pname))

        session = Streamlink()

        def gentest(pname):
            def load_plugin_test(self):
                # Reset file variable to ensure it is still open when doing
                # load_plugin else python might open the plugin source .py
                # using ascii encoding instead of utf-8.
                # See also open() call here: imp._HackedGetData.get_data
                file, pathname, desc = imp.find_module(pname, [plugin_path])
                session.load_plugin(pname, file, pathname, desc)
                # validate that can_handle_url does not fail
                session.plugins[pname].can_handle_url("http://test.com")

            return load_plugin_test

        for pname in plugins:
            dict['test_{0}_load'.format(pname)] = gentest(pname)

        return type.__new__(mcs, name, bases, dict) 

def for_path_items(cls, prefix, path_items):
    pkg_names = frozenset(pkg_name for _, pkg_name, _ in pkgutil.iter_modules(path=path_items))
    return cls(prefix=prefix, packages=pkg_names) 

def _find_clazzes(directory, base_class):
    found_classes = set()
    for module_finder, name, ispkg in pkgutil.iter_modules([directory]):
        if ispkg:
            continue

        relative_path = os.path.relpath(module_finder.path, AIRFLOW_ROOT)
        package_name = relative_path.replace("/", ".")
        full_module_name = package_name + "." + name
        try:
            mod = import_module(full_module_name)
        except ModuleNotFoundError:
            print(f"Module {full_module_name} can not be loaded.", file=sys.stderr)
            continue

        clazzes = inspect.getmembers(mod, inspect.isclass)
        integration_clazzes = [
            clazz
            for name, clazz in clazzes
            if issubclass(clazz, base_class) and clazz.__module__.startswith(package_name)
        ]

        for clazz in integration_clazzes:
            found_classes.add("{}.{}".format(clazz.__module__, clazz.__name__))

    return found_classes 

def get_all_fix_names(fixer_pkg, remove_prefix=True):
    """Return a sorted list of all available fix names in the given package."""
    pkg = __import__(fixer_pkg, [], [], ["*"])
    fix_names = []
    for finder, name, ispkg in pkgutil.iter_modules(pkg.__path__):
        if name.startswith("fix_"):
            if remove_prefix:
                name = name[4:]
            fix_names.append(name)
    return fix_names 

def dynamic_load_module(module, prefix=None):
    """Load submodules inside a module, mainly used for model loading, not robust!!!"""
    # loading all models under directory `models` dynamically
    if not isinstance(module, str):
        module = module.__path__
    for importer, modname, ispkg in pkgutil.iter_modules(module):
        if prefix is None:
            __import__(modname)
        else:
            __import__("{}.{}".format(prefix, modname)) 

def get_checks():
    """Loads and returns the available lint checks"""
    global _checks_loaded
    if not _checks_loaded:
        for _loader, name, _ispkg in pkgutil.iter_modules(__path__):
            if name.startswith('check_'):
                importlib.import_module(__name__ + '.' + name)
        _checks_loaded = True
    return LintCheckMeta.registry 

def get_ngraph_op_cls(op_type):
    """ Walk over python modules in ngraph.op_graph and look for op_type class. """
    for importer, modname, ispkg in pkgutil.iter_modules(ngraph.op_graph.__path__):
        imported_mod = importlib.import_module('ngraph.op_graph.' + modname)
        if hasattr(imported_mod, op_type):
            return getattr(imported_mod, op_type)
    raise ValueError("Cannot find op_type of {} in any ngraph.op_graph modules.".format(op_type)) 

def _list_module_names():
    module_names = []
    package_path = os.path.dirname(os.path.abspath(__file__))
    for _, modname, ispkg in pkgutil.iter_modules(path=[package_path]):
        if modname == "opts" or ispkg:
            continue
        else:
            module_names.append(modname)
    return module_names 

def init_parser(self):
        self.parser = PdmParser(
            prog="pdm",
            description="PDM - Python Development Master",
            formatter_class=PdmFormatter,
        )
        self.parser.is_root = True
        self.parser.add_argument(
            "-V",
            "--version",
            action="version",
            version="{}, version {}".format(
                click.style("pdm", bold=True), self.version
            ),
            help="show the version and exit",
        )
        verbose_option.add_to_parser(self.parser)

        self.subparsers = self.parser.add_subparsers()
        for _, name, _ in pkgutil.iter_modules(COMMANDS_MODULE_PATH):
            module = importlib.import_module(f"pdm.cli.commands.{name}", __name__)
            try:
                klass = module.Command  # type: Type[BaseCommand]
            except AttributeError:
                continue
            self.register_command(klass, klass.name or name) 

def index(self, dir, shadowed=None):
        """Generate an HTML index for a directory of modules."""
        modpkgs = []
        if shadowed is None: shadowed = {}
        for importer, name, ispkg in pkgutil.iter_modules([dir]):
            modpkgs.append((name, '', ispkg, name in shadowed))
            shadowed[name] = 1

        modpkgs.sort()
        contents = self.multicolumn(modpkgs, self.modpkglink)
        return self.bigsection(dir, '#ffffff', '#ee77aa', contents)

# -------------------------------------------- text documentation generator 

def all_names():
    for _, modname, _ in pkgutil.iter_modules(__path__):
        if modname.startswith('test_'):
            yield 'stripe.test.' + modname