Python imp.load_source() Examples

def load_module(module_name, file_path):
    """
    Load a module by name and search path

    This function should work with python 2.7 and 3.x

    Returns None if Module could not be loaded.
    """
    if sys.version_info >= (3,5,):
        import importlib.util

        spec = importlib.util.spec_from_file_location(module_name, file_path)
        if not spec:
            return

        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)

        return module
    else:
        import imp
        mod = imp.load_source(module_name, file_path)
        return mod 

def test_imp_load_source(self):
        import os
        try:
            _x_mod = os.path.join(self.test_dir, "x.py")
            self.write_to_file(_x_mod, """
'''some pydoc'''
X = 3.14
    """)
            with open(_x_mod, "r") as f:
                x = imp.load_source("test_imp_load_source_x",
                                    _x_mod,
                                    f)
            self.assertEqual(x.__name__, "test_imp_load_source_x")
            self.assertEqual(x.X, 3.14)
            self.assertEqual(x.__doc__, '''some pydoc''')
        finally:
            os.unlink(_x_mod) 

def load_functions(filepath=None):
    if not filepath:
        return None

    if not os.path.isfile(filepath):
        raise IOError("'{}' not found".format(filepath))

    mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])

    if file_ext.lower() == '.py':
        py_mod = imp.load_source(mod_name, filepath)

    elif file_ext.lower() == '.pyc':
        py_mod = imp.load_compiled(mod_name, filepath)

    else:
        raise ValueError("'{}' does not have the .py or .pyc extension".format(filepath))

    return py_mod 

def main():
    dn_merge = imp.load_source('dn_merge', '../detection/merge_results.py')

    file_paths = []
    for split_id in range(settings.TEST_SPLIT_NUM):
        result_file_path = darknet_tools.append_before_ext(settings.TEST_RESULTS_OUT, '.{}'.format(split_id))
        file_paths.append(result_file_path)

    print('loading ssd outputs')
    unmerged = read(file_paths)

    print('doing nms sort')
    nms_sorted = dn_merge.do_nms_sort(unmerged, .5)

    print('writing results')
    dn_merge.write(nms_sorted, os.path.join(settings.PRODUCTS_ROOT, 'proposals.jsonl' if proposal_output else 'detections.jsonl')) 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        """

        :param ag_params: agent parameter dictionary
        :param policyparams: policy parameter dict
        :param gpu_id: gpu id
        :param ngpu: number of gpus to use
        """
        CEMBaseController.__init__(self, ag_params, policyparams)

        params = imp.load_source('params', ag_params['current_dir'] + '/conf.py')
        netconf = params.configuration
        self.predictor = netconf['setup_predictor'](ag_params, netconf, gpu_id, ngpu, self._logger)

        self._net_bsize = netconf['batch_size']
        self._net_context = netconf['context_frames']
        self._hp.start_planning = self._net_context
        self._n_cam = netconf['ncam']

        self._images, self._verbose_worker = None, None
        self._save_actions = None 

def get_model(args):
    model_fn = os.path.basename(args.model)
    model = imp.load_source(model_fn.split('.')[0], args.model).model

    if 'result_dir' in args:
        dst = '%s/%s' % (args.result_dir, model_fn)
        if not os.path.exists(dst):
            shutil.copy(args.model, dst)

        dst = '%s/%s' % (args.result_dir, os.path.basename(__file__))
        if not os.path.exists(dst):
            shutil.copy(__file__, dst)

    # load model
    if args.resume_model is not None:
        serializers.load_hdf5(args.resume_model, model)

    # prepare model
    if args.gpu >= 0:
        model.to_gpu()

    return model 

def load_module(module_path):
    if "/" not in module_path:
        import importlib

        return importlib.import_module("%s" % module_path, package="armory")
    else:
        module_name = module_path.split("/")[-1]
        if sys.version_info.major == 2:
            import imp

            return imp.load_source(module_name, module_path + ".py")
        else:
            import importlib.util

            spec = importlib.util.spec_from_file_location(
                module_name, module_path + ".py"
            )
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            return module 

def import_module_by_path(path):
    name = os.path.splitext(os.path.basename(path))[0]
    if sys.version_info[0] == 2:
        import imp
        return imp.load_source(name, path)
    elif sys.version_info[:2] <= (3, 4):
        from importlib.machinery import SourceFileLoader
        return SourceFileLoader(name, path).load_module()
    else:
        import importlib.util
        spec = importlib.util.spec_from_file_location(name, path)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)
        return mod


# ===================================================================
# --- others
# =================================================================== 

def main(options):
    feature_conf = imp.load_source("", 
        os.path.join(config.FEAT_CONF_DIR, options.feature_conf+".py"))
    if options.feature_level == 1:
        combiner = Combiner(feature_dict=feature_conf.feature_dict,
                            feature_name=options.feature_name,
                            feature_suffix=options.feature_suffix,
                            corr_threshold=options.corr_threshold)
    elif options.feature_level > 1:
        feature_conf_meta = imp.load_source("", 
            os.path.join(config.FEAT_CONF_DIR, options.feature_conf_meta+".py"))
        combiner = StackingCombiner(feature_list=feature_conf.feature_list,
                                    feature_name=options.feature_name,
                                    feature_suffix=options.feature_suffix,
                                    feature_level=options.feature_level,
                                    meta_feature_dict=feature_conf_meta.feature_dict,
                                    corr_threshold=options.corr_threshold)

    combiner.combine()
    combiner.save() 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        super(Register_Gtruth_Controller, self).__init__(ag_params, policyparams, gpu_id, ngpu)
        
        self._hp = self._default_hparams()
        self._override_defaults(policyparams)

        self.reg_tradeoff = np.ones([self._n_cam, self._n_desig]) / self._n_cam / self._n_desig

        params = imp.load_source('params', ag_params['current_dir'] + '/gdnconf.py')
        self.gdnconf = params.configuration
        self.goal_image_warper = setup_gdn(self.gdnconf, gpu_id)

        num_reg_images = len(self._hp.register_gtruth)

        self.ntask = self._n_desig // num_reg_images

        self.visualizer = CEM_Visual_Preparation_Registration() 

def update(self, source):
        """
        Updates an existing datatype in the arches db

        """

        import imp

        dt_source = imp.load_source("", source)
        details = dt_source.details
        self.validate_details(details)

        instance = models.DDataType.objects.get(datatype=details["datatype"])
        instance.iconclass = details["iconclass"]
        instance.modulename = os.path.basename(source)
        instance.classname = details["classname"]
        instance.defaultwidget = details["defaultwidget"]
        instance.defaultconfig = details["defaultconfig"]
        instance.configcomponent = details["configcomponent"]
        instance.configname = details["configname"]
        instance.isgeometric = details["isgeometric"]
        instance.issearchable = details["issearchable"]

        instance.save() 

def import_module_by_path(path):
    name = os.path.splitext(os.path.basename(path))[0]
    if sys.version_info[0] == 2:
        import imp
        return imp.load_source(name, path)
    elif sys.version_info[:2] <= (3, 4):
        from importlib.machinery import SourceFileLoader
        return SourceFileLoader(name, path).load_module()
    else:
        import importlib.util
        spec = importlib.util.spec_from_file_location(name, path)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)
        return mod


# ===================================================================
# --- others
# =================================================================== 

def _loadFilterFromPath(self, path, functionName):
        """
        _loadFilterFromPath("path/to/a/python/file/withAPen.py", "myFilterPen")
        """
        try:
            f = open(path, "r")
            try:
                moduleName = "externalPenBallWizard{0}".format(path.split('/')[-1][:-3])
                if moduleName not in sys.modules:
                    module = imp.load_source(moduleName, path, f)
                else:
                    module = __import__(moduleName, fromlist=[functionName])
                result = getattr(module, functionName)
            except:
                result = None
            f.close()
            return result
        except IOError as e:
            print 'Couldn’t load file {0}'.format(e) 

def load_module(self, name, stuff):
        file, filename, info = stuff
        (suff, mode, type) = info
        try:
            if type == BUILTIN_MODULE:
                return self.hooks.init_builtin(name)
            if type == FROZEN_MODULE:
                return self.hooks.init_frozen(name)
            if type == C_EXTENSION:
                m = self.hooks.load_dynamic(name, filename, file)
            elif type == PY_SOURCE:
                m = self.hooks.load_source(name, filename, file)
            elif type == PY_COMPILED:
                m = self.hooks.load_compiled(name, filename, file)
            elif type == PKG_DIRECTORY:
                m = self.hooks.load_package(name, filename, file)
            else:
                raise ImportError, "Unrecognized module type (%r) for %s" % \
                      (type, name)
        finally:
            if file: file.close()
        m.__file__ = filename
        return m 

def load_module(self, name, stuff):
        file, filename, info = stuff
        (suff, mode, type) = info
        try:
            if type == BUILTIN_MODULE:
                return self.hooks.init_builtin(name)
            if type == FROZEN_MODULE:
                return self.hooks.init_frozen(name)
            if type == C_EXTENSION:
                m = self.hooks.load_dynamic(name, filename, file)
            elif type == PY_SOURCE:
                m = self.hooks.load_source(name, filename, file)
            elif type == PY_COMPILED:
                m = self.hooks.load_compiled(name, filename, file)
            elif type == PKG_DIRECTORY:
                m = self.hooks.load_package(name, filename, file)
            else:
                raise ImportError, "Unrecognized module type (%r) for %s" % \
                      (type, name)
        finally:
            if file: file.close()
        m.__file__ = filename
        return m 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        """

        :param ag_params: agent parameter dictionary
        :param policyparams: policy parameter dict
        :param gpu_id: gpu id
        :param ngpu: number of gpus to use
        """
        CEMBaseController.__init__(self, ag_params, policyparams)

        params = imp.load_source('params', ag_params['current_dir'] + '/conf.py')
        netconf = params.configuration
        self.predictor = netconf['setup_predictor'](ag_params, netconf, gpu_id, ngpu, self._logger)

        self._net_bsize = netconf['batch_size']
        self._net_seqlen = netconf['sequence_length']

        self._net_context = netconf['context_frames']
        self._hp.start_planning = self._net_context

        self._n_desig = netconf.get('ndesig', None)
        self._img_height, self._img_width = netconf['orig_size']

        self._n_cam = netconf['ncam']

        self._desig_pix = None
        self._goal_pix = None
        self._images = None

        if self._hp.predictor_propagation:
            self._rec_input_distrib = []  # record the input distributions 

def import_file(path):
    """ Returns imported file """
    if sys.version_info[0] == 2 or sys.version_info[1] < 3:
        import imp

        return imp.load_source("tmp", path)
    elif sys.version_info[0] == 3:
        import importlib.machinery

        return importlib.machinery.SourceFileLoader("tmp", path).load_module("tmp") 

def load_module(self, module_path):
        try:
            module = imp.load_source('payload_module', module_path).CMEModule()
            if self.module_is_sane(module, module_path):
                return module
        except Exception as e:
            self.logger.error('Failed loading module at {}: {}'.format(module_path, e))

        return None 

def import_file(path):
    """ Returns imported file """
    if sys.version_info[0] == 2 or sys.version_info[1] < 3:
        import imp

        return imp.load_source("tmp", path)
    elif sys.version_info[0] == 3:
        import importlib.machinery

        return importlib.machinery.SourceFileLoader("tmp", path).load_module("tmp") 

def load_protocol(self, protocol_path):
        protocol = imp.load_source('protocol', protocol_path)
        #if self.module_is_sane(module, module_path):
        return protocol 

def import_file(path):
    """ Returns imported file """
    if sys.version_info[0] == 2 or sys.version_info[1] < 3:
        import imp

        return imp.load_source("tmp", path)
    elif sys.version_info[0] == 3:
        import importlib.machinery

        return importlib.machinery.SourceFileLoader("tmp", path).load_module("tmp") 

def load_commands_from_file(path):
    module = imp.load_source('custom_commands', path)
    commands = [obj for name, obj in getmembers(module) if isinstance(obj, click.core.Command)]
    return commands 

def __init__(self, path, batchable=None):
        self.path = path
        self.batchable = batchable
        self.name = os.path.split(path)[1].rstrip('.py')
        self.module = imp.load_source(self.name, path) 

def python_file_run_with_helper(self, test_path):
        """
        Call run() function at external python module.

        :param test_path: Path to python file. Call run() at this module.
        :return: module.run().return
        """
        assert os.path.isfile(test_path)
        module = imp.load_source('ext_test', test_path)
        assert hasattr(module, "run")
        run = getattr(module, "run")
        helper = Helper(self)
        return run(helper) 

def _load_module(self, dirpath, filename):
        mod_name = filename.split('.')[0]
        mod_category = re.search('/modules/([^/]*)', dirpath).group(1)
        mod_dispname = '/'.join(re.split('/modules/', dirpath)[-1].split('/') + [mod_name])
        mod_loadname = mod_dispname.replace('/', '_')
        mod_loadpath = os.path.join(dirpath, filename)
        mod_file = open(mod_loadpath)
        try:
            # import the module into memory
            mod = imp.load_source(mod_loadname, mod_loadpath, mod_file)
            __import__(mod_loadname)
            # add the module to the framework's loaded modules
            self._loaded_modules[mod_dispname] = sys.modules[mod_loadname].Module(mod_dispname)
            self._categorize_module(mod_category, mod_dispname)
            # return indication of success to support module reload
            return True
        except ImportError as e:
            # notify the user of missing dependencies
            self.error(f"Module '{mod_dispname}' disabled. Dependency required: '{self.to_unicode_str(e)[16:]}'")
        except:
            # notify the user of errors
            self.print_exception()
            self.error(f"Module '{mod_dispname}' disabled.")
        # remove the module from the framework's loaded modules
        self._loaded_modules.pop(mod_dispname, None)
        self._categorize_module('disabled', mod_dispname) 

def test_windows_os():
    sys.platform = "Windows Vista"
    try:
        imp.load_source("phpsploit", "./phpsploit")
        import phpsploit
    except SystemExit as err:
        errmsg = str(err)
        assert "you should be a TARGET rather than an attacker" in errmsg 

def load_module(type_, path):
        module = None
        try:
            module = load_source(
                "platformio.platforms.%s" % type_, path)
        except ImportError:
            raise exception.UnknownPlatform(type_)
        return module 

def run(self):
        """
        this method will:
        1) clone git repository with test files into temp location
        2) execute tests
        3) clear repository

        """
        if not self.image_id:
            raise RuntimeError("no image_id specified (build probably failed)")
        tmpdir = tempfile.mkdtemp()
        g = LazyGit(self.git_uri, self.git_commit, tmpdir)
        with g:
            tests_file = os.path.abspath(os.path.join(g.git_path, self.tests_git_path))
            self.log.debug("loading file with tests: '%s'", tests_file)
            module_name, dummy_module_ext = os.path.splitext(self.tests_git_path)
            tests_module = imp.load_source(module_name, tests_file)

            results, passed = tests_module.run(image_id=self.image_id, tests=self.tests,
                                               git_repo_path=tmpdir, logger=self.log,
                                               results_dir=self.results_dir, **self.kwargs)

        shutil.rmtree(tmpdir)
        if not passed:
            self.log.error("tests failed: %s", results)
            raise RuntimeError("Tests didn't pass!")
        return results 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        """

        :param ag_params: agent parameters
        :param policyparams: policy parameters
        :param gpu_id: starting gpu id
        :param ngpu: number of gpus
        """
        CEMBaseController.__init__(self, ag_params, policyparams)

        params = imp.load_source('params', ag_params['current_dir'] + '/conf.py')
        net_conf = params.configuration

        if ngpu > 1:
            vpred_ngpu = ngpu - 1
        else: vpred_ngpu = ngpu

        self._predictor = net_conf['setup_predictor'](ag_params, net_conf, gpu_id, vpred_ngpu, self._logger)
        self._scoring_func = control_embedding.deploy_simple_model(self._hp.classifier_conf_path, batch_size=self._hp.classifier_batch_size,
                                                                   restore_path=self._hp.classifier_restore_path,
                                                                   device_id=gpu_id + ngpu - 1)

        self._vpred_bsize = net_conf['batch_size']

        self._seqlen = net_conf['sequence_length']
        self._net_context = net_conf['context_frames']
        self._hp.start_planning = self._net_context            # skip steps so there are enough context frames
        self._n_pred = self._seqlen - self._net_context
        assert self._n_pred > 0, "context_frames must be larger than sequence_length"

        self._img_height, self._img_width = net_conf['orig_size']

        self._n_cam = net_conf['ncam']

        self._images = None
        self._expert_images = None
        self._expert_score = None
        self._goal_image = None
        self._start_image = None
        self._verbose_worker = None 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        """

        :param ag_params: agent parameters
        :param policyparams: policy parameters
        :param gpu_id: starting gpu id
        :param ngpu: number of gpus
        """
        CEMBaseController.__init__(self, ag_params, policyparams)

        params = imp.load_source('params', ag_params['current_dir'] + '/conf.py')
        net_conf = params.configuration

        if ngpu > 1:
            vpred_ngpu = ngpu - 1
        else: vpred_ngpu = ngpu

        self._predictor = net_conf['setup_predictor'](ag_params, net_conf, gpu_id, vpred_ngpu, self._logger)
        self._scoring_func = control_embedding.deploy_model(self._hp.nce_conf_path, batch_size=self._hp.nce_batch_size,
                                                            restore_path=self._hp.nce_restore_path,
                                                            device_id=gpu_id + ngpu - 1)

        self._vpred_bsize = net_conf['batch_size']

        self._seqlen = net_conf['sequence_length']
        self._net_context = net_conf['context_frames']
        self._hp.start_planning = self._net_context            # skip steps so there are enough context frames
        self._n_pred = self._seqlen - self._net_context
        assert self._n_pred > 0, "context_frames must be larger than sequence_length"

        self._img_height, self._img_width = net_conf['orig_size']

        self._n_cam = net_conf['ncam']

        self._images = None
        self._expert_images = None
        self._expert_score = None
        self._goal_image = None
        self._start_image = None
        self._verbose_worker = None