Python imp.load_source() Examples

def load_module(module_name, file_path):
    """
    Load a module by name and search path

    This function should work with python 2.7 and 3.x

    Returns None if Module could not be loaded.
    """
    if sys.version_info >= (3,5,):
        import importlib.util

        spec = importlib.util.spec_from_file_location(module_name, file_path)
        if not spec:
            return

        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)

        return module
    else:
        import imp
        mod = imp.load_source(module_name, file_path)
        return mod 

def update(self, source):
        """
        Updates an existing datatype in the arches db

        """

        import imp

        dt_source = imp.load_source("", source)
        details = dt_source.details
        self.validate_details(details)

        instance = models.DDataType.objects.get(datatype=details["datatype"])
        instance.iconclass = details["iconclass"]
        instance.modulename = os.path.basename(source)
        instance.classname = details["classname"]
        instance.defaultwidget = details["defaultwidget"]
        instance.defaultconfig = details["defaultconfig"]
        instance.configcomponent = details["configcomponent"]
        instance.configname = details["configname"]
        instance.isgeometric = details["isgeometric"]
        instance.issearchable = details["issearchable"]

        instance.save() 

def get_model(args):
    model_fn = os.path.basename(args.model)
    model = imp.load_source(model_fn.split('.')[0], args.model).model

    if 'result_dir' in args:
        dst = '%s/%s' % (args.result_dir, model_fn)
        if not os.path.exists(dst):
            shutil.copy(args.model, dst)

        dst = '%s/%s' % (args.result_dir, os.path.basename(__file__))
        if not os.path.exists(dst):
            shutil.copy(__file__, dst)

    # load model
    if args.resume_model is not None:
        serializers.load_hdf5(args.resume_model, model)

    # prepare model
    if args.gpu >= 0:
        model.to_gpu()

    return model 

def main():
    dn_merge = imp.load_source('dn_merge', '../detection/merge_results.py')

    file_paths = []
    for split_id in range(settings.TEST_SPLIT_NUM):
        result_file_path = darknet_tools.append_before_ext(settings.TEST_RESULTS_OUT, '.{}'.format(split_id))
        file_paths.append(result_file_path)

    print('loading ssd outputs')
    unmerged = read(file_paths)

    print('doing nms sort')
    nms_sorted = dn_merge.do_nms_sort(unmerged, .5)

    print('writing results')
    dn_merge.write(nms_sorted, os.path.join(settings.PRODUCTS_ROOT, 'proposals.jsonl' if proposal_output else 'detections.jsonl')) 

def load_functions(filepath=None):
    if not filepath:
        return None

    if not os.path.isfile(filepath):
        raise IOError("'{}' not found".format(filepath))

    mod_name, file_ext = os.path.splitext(os.path.split(filepath)[-1])

    if file_ext.lower() == '.py':
        py_mod = imp.load_source(mod_name, filepath)

    elif file_ext.lower() == '.pyc':
        py_mod = imp.load_compiled(mod_name, filepath)

    else:
        raise ValueError("'{}' does not have the .py or .pyc extension".format(filepath))

    return py_mod 

def load_module(self, name, stuff):
        file, filename, info = stuff
        (suff, mode, type) = info
        try:
            if type == BUILTIN_MODULE:
                return self.hooks.init_builtin(name)
            if type == FROZEN_MODULE:
                return self.hooks.init_frozen(name)
            if type == C_EXTENSION:
                m = self.hooks.load_dynamic(name, filename, file)
            elif type == PY_SOURCE:
                m = self.hooks.load_source(name, filename, file)
            elif type == PY_COMPILED:
                m = self.hooks.load_compiled(name, filename, file)
            elif type == PKG_DIRECTORY:
                m = self.hooks.load_package(name, filename, file)
            else:
                raise ImportError, "Unrecognized module type (%r) for %s" % \
                      (type, name)
        finally:
            if file: file.close()
        m.__file__ = filename
        return m 

def test_imp_load_source(self):
        import os
        try:
            _x_mod = os.path.join(self.test_dir, "x.py")
            self.write_to_file(_x_mod, """
'''some pydoc'''
X = 3.14
    """)
            with open(_x_mod, "r") as f:
                x = imp.load_source("test_imp_load_source_x",
                                    _x_mod,
                                    f)
            self.assertEqual(x.__name__, "test_imp_load_source_x")
            self.assertEqual(x.X, 3.14)
            self.assertEqual(x.__doc__, '''some pydoc''')
        finally:
            os.unlink(_x_mod) 

def import_module_by_path(path):
    name = os.path.splitext(os.path.basename(path))[0]
    if sys.version_info[0] == 2:
        import imp
        return imp.load_source(name, path)
    elif sys.version_info[:2] <= (3, 4):
        from importlib.machinery import SourceFileLoader
        return SourceFileLoader(name, path).load_module()
    else:
        import importlib.util
        spec = importlib.util.spec_from_file_location(name, path)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)
        return mod


# ===================================================================
# --- others
# =================================================================== 

def _loadFilterFromPath(self, path, functionName):
        """
        _loadFilterFromPath("path/to/a/python/file/withAPen.py", "myFilterPen")
        """
        try:
            f = open(path, "r")
            try:
                moduleName = "externalPenBallWizard{0}".format(path.split('/')[-1][:-3])
                if moduleName not in sys.modules:
                    module = imp.load_source(moduleName, path, f)
                else:
                    module = __import__(moduleName, fromlist=[functionName])
                result = getattr(module, functionName)
            except:
                result = None
            f.close()
            return result
        except IOError as e:
            print 'Couldn’t load file {0}'.format(e) 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        super(Register_Gtruth_Controller, self).__init__(ag_params, policyparams, gpu_id, ngpu)
        
        self._hp = self._default_hparams()
        self._override_defaults(policyparams)

        self.reg_tradeoff = np.ones([self._n_cam, self._n_desig]) / self._n_cam / self._n_desig

        params = imp.load_source('params', ag_params['current_dir'] + '/gdnconf.py')
        self.gdnconf = params.configuration
        self.goal_image_warper = setup_gdn(self.gdnconf, gpu_id)

        num_reg_images = len(self._hp.register_gtruth)

        self.ntask = self._n_desig // num_reg_images

        self.visualizer = CEM_Visual_Preparation_Registration() 

def __init__(self, ag_params, policyparams, gpu_id, ngpu):
        """

        :param ag_params: agent parameter dictionary
        :param policyparams: policy parameter dict
        :param gpu_id: gpu id
        :param ngpu: number of gpus to use
        """
        CEMBaseController.__init__(self, ag_params, policyparams)

        params = imp.load_source('params', ag_params['current_dir'] + '/conf.py')
        netconf = params.configuration
        self.predictor = netconf['setup_predictor'](ag_params, netconf, gpu_id, ngpu, self._logger)

        self._net_bsize = netconf['batch_size']
        self._net_context = netconf['context_frames']
        self._hp.start_planning = self._net_context
        self._n_cam = netconf['ncam']

        self._images, self._verbose_worker = None, None
        self._save_actions = None 

def load_module(module_path):
    if "/" not in module_path:
        import importlib

        return importlib.import_module("%s" % module_path, package="armory")
    else:
        module_name = module_path.split("/")[-1]
        if sys.version_info.major == 2:
            import imp

            return imp.load_source(module_name, module_path + ".py")
        else:
            import importlib.util

            spec = importlib.util.spec_from_file_location(
                module_name, module_path + ".py"
            )
            module = importlib.util.module_from_spec(spec)
            spec.loader.exec_module(module)
            return module 

def import_module_by_path(path):
    name = os.path.splitext(os.path.basename(path))[0]
    if sys.version_info[0] == 2:
        import imp
        return imp.load_source(name, path)
    elif sys.version_info[:2] <= (3, 4):
        from importlib.machinery import SourceFileLoader
        return SourceFileLoader(name, path).load_module()
    else:
        import importlib.util
        spec = importlib.util.spec_from_file_location(name, path)
        mod = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(mod)
        return mod


# ===================================================================
# --- others
# =================================================================== 

def main(options):
    feature_conf = imp.load_source("", 
        os.path.join(config.FEAT_CONF_DIR, options.feature_conf+".py"))
    if options.feature_level == 1:
        combiner = Combiner(feature_dict=feature_conf.feature_dict,
                            feature_name=options.feature_name,
                            feature_suffix=options.feature_suffix,
                            corr_threshold=options.corr_threshold)
    elif options.feature_level > 1:
        feature_conf_meta = imp.load_source("", 
            os.path.join(config.FEAT_CONF_DIR, options.feature_conf_meta+".py"))
        combiner = StackingCombiner(feature_list=feature_conf.feature_list,
                                    feature_name=options.feature_name,
                                    feature_suffix=options.feature_suffix,
                                    feature_level=options.feature_level,
                                    meta_feature_dict=feature_conf_meta.feature_dict,
                                    corr_threshold=options.corr_threshold)

    combiner.combine()
    combiner.save() 

def load_module(self, name, stuff):
        file, filename, info = stuff
        (suff, mode, type) = info
        try:
            if type == BUILTIN_MODULE:
                return self.hooks.init_builtin(name)
            if type == FROZEN_MODULE:
                return self.hooks.init_frozen(name)
            if type == C_EXTENSION:
                m = self.hooks.load_dynamic(name, filename, file)
            elif type == PY_SOURCE:
                m = self.hooks.load_source(name, filename, file)
            elif type == PY_COMPILED:
                m = self.hooks.load_compiled(name, filename, file)
            elif type == PKG_DIRECTORY:
                m = self.hooks.load_package(name, filename, file)
            else:
                raise ImportError, "Unrecognized module type (%r) for %s" % \
                      (type, name)
        finally:
            if file: file.close()
        m.__file__ = filename
        return m 

def get_settings(local_config_file_path=None):
    swagger_settings = SWAGGER_SETTINGS.copy()
    plugin_settings = {}

    if local_config_file_path:
        custom_config = imp.load_source('config', local_config_file_path)
        swagger_settings.update(getattr(custom_config, 'SWAGGER_SETTINGS', dict()))
        plugin_settings.update(getattr(custom_config, 'PLUGIN_SETTINGS', dict()))

    return swagger_settings, plugin_settings 

def get_caffe_pb():
    """
    Get caffe protobuf.

    Returns:
        The imported caffe protobuf module.
    """
    dir = get_dataset_path('caffe')
    caffe_pb_file = os.path.join(dir, 'caffe_pb2.py')
    if not os.path.isfile(caffe_pb_file):
        download(CAFFE_PROTO_URL, dir)
        assert os.path.isfile(os.path.join(dir, 'caffe.proto'))

        cmd = "protoc --version"
        version, ret = subproc_call(cmd, timeout=3)
        if ret != 0:
            sys.exit(1)
        try:
            version = version.decode('utf-8')
            version = float('.'.join(version.split(' ')[1].split('.')[:2]))
            assert version >= 2.7, "Require protoc>=2.7 for Python3"
        except Exception:
            logger.exception("protoc --version gives: " + str(version))
            raise

        cmd = 'cd {} && protoc caffe.proto --python_out .'.format(dir)
        ret = os.system(cmd)
        assert ret == 0, \
            "Command `{}` failed!".format(cmd)
        assert os.path.isfile(caffe_pb_file), caffe_pb_file
    import imp
    return imp.load_source('caffepb', caffe_pb_file) 

def dataset_walker(dataset=None, dataroot=None, labels=None):
    # we assume that the dataset_walker class in dataroot/../scripts
    # is the one to use
    scripts_folder = os.path.join(dataroot, '../..', "scripts")
    # print(scripts_folder)
    _dw = imp.load_source('dataset_walker', os.path.join(scripts_folder, "dataset_walker.py"))
    return _dw.dataset_walker(dataset, dataroot=dataroot, labels=labels) 

def _load_module_from_file(name, filename):
    if _PY2:
        module = imp.load_source(name, filename)
    else:
        spec = importlib.util.spec_from_file_location(name, filename)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
    return module 

def compile_file(source, globals_=None):
  """Compile by saving to file and importing that.

  Compiling the AST/source code this way ensures that the source code is
  readable by e.g. `pdb` or `inspect`.

  Args:
    source: The code to compile, either as a string or as an AST.
    globals_: A dictionary of variables that should be available as globals in
        the compiled module. They will be monkey patched after importing the
        module.

  Returns:
    A module object containing the compiled source code.
  """
  if isinstance(source, gast.AST):
    source = quoting.to_source(source)

  # Write source to temporary file
  tempdir = tempfile.mkdtemp()
  uuid = str(uuid4().hex[:4])
  tmpname = os.path.join(tempdir, 'tangent_%s.py' % uuid)
  with open(tmpname, 'w') as f:
    f.write(source)

  # Load the temporary file as a module
  module_name = 'tangent_%s' % uuid
  if six.PY3:
    spec = util.spec_from_file_location(module_name, tmpname)
    m = util.module_from_spec(spec)
    spec.loader.exec_module(m)
  else:
    m = imp.load_source(module_name, tmpname)

  # Update the modules namespace
  if globals_:
    m.__dict__.update(globals_)
  return m 

def version():
    module = imp.load_source("hirlite.version", "hirlite/version.py")
    return module.__version__

# Patch "install_lib" command to run build_clib before build_ext
# to properly work with easy_install.
# See: http://bugs.python.org/issue5243 

def load_module(module_id, path):
        fp = open(path, "rb")
        try:
            return imp.load_source(module_id, path, fp)
        finally:
            fp.close() 

def load_module(module_id, path):
        fp = open(path, "rb")
        try:
            return imp.load_source(module_id, path, fp)
        finally:
            fp.close() 

def _import_module(pyfilepath):
    module_name = os.path.splitext(os.path.basename(pyfilepath))[0]

    if sys.version_info > (3, 4):
        import importlib.util
        spec = importlib.util.spec_from_file_location(module_name, pyfilepath)
        module = importlib.util.module_from_spec(spec)
        spec.loader.exec_module(module)
    else:
        import imp
        module = imp.load_source(module_name, pyfilepath)

    return module 

def import_file(module_name, file_path, make_importable=None):
        module = imp.load_source(module_name, file_path)
        return module 

def _loadConfig(self, pyData, basedir):
        config = defaultconfig.DefaultConfig.copy()

        pyData["basedir"] = basedir
        pyData["projdir"] = os.getcwd() + "/"

        for key in pyData:
            config[key] = pyData[key]

        # cleanup. Damn this is ugly.
        config["target_bin"] = config["projdir"] + config["target_bin"]
        config["target_dir"] = os.path.dirname(os.path.realpath(config['target_bin']))
        config["input_dir"] = config["projdir"] + config["input_dir"]
        config["temp_dir"] = config["projdir"] + config["temp_dir"]
        config["outcome_dir"] = config["projdir"] + config["outcome_dir"]
        config["verified_dir"] = config["projdir"] + config["verified_dir"]
        config["grammars"] = config["projdir"] + config["grammars"]

        if 'use_protocol' in config and config['use_protocol']:
            foo = imp.load_source('Protocol', config["projdir"] + 'protocol.py')
            proto = foo.Protocol()
            config['protocolInstance'] = proto
        else:
            config['protocolInstance'] = None

        self.config = config

        return config 

def register(self, source):
        """
        Inserts a datatype into the arches db

        """

        import imp

        dt_source = imp.load_source("", source)
        details = dt_source.details
        self.validate_details(details)

        dt = models.DDataType(
            datatype=details["datatype"],
            iconclass=details["iconclass"],
            modulename=os.path.basename(source),
            classname=details["classname"],
            defaultwidget=details["defaultwidget"],
            defaultconfig=details["defaultconfig"],
            configcomponent=details["configcomponent"],
            configname=details["configname"],
            isgeometric=details["isgeometric"],
            issearchable=details["issearchable"],
        )

        if len(models.DDataType.objects.filter(datatype=dt.datatype)) == 0:
            dt.save()
        else:
            print("{0} already exists".format(dt.datatype)) 

def register(self, source):
        """
        Inserts a function into the arches db

        """

        import imp

        fn_config = imp.load_source("", source)
        details = fn_config.details

        try:
            uuid.UUID(details["functionid"])
        except:
            details["functionid"] = str(uuid.uuid4())
            print("Registering function with functionid: {}".format(details["functionid"]))

        fn = models.Function(
            functionid=details["functionid"],
            name=details["name"],
            functiontype=details["type"],
            description=details["description"],
            defaultconfig=details["defaultconfig"],
            modulename=os.path.basename(source),
            classname=details["classname"],
            component=details["component"],
        )

        fn.save() 

def register(self, source):
        """
        Inserts a search component into the arches db

        """

        dt_source = imp.load_source("", source)

        if getattr(dt_source, "details", None):
            details = dt_source.details

            try:
                uuid.UUID(details["searchcomponentid"])
            except:
                details["searchcomponentid"] = str(uuid.uuid4())
            print("Registering the search component, %s, with componentid: %s" % (details["name"], details["searchcomponentid"]))

            instance = models.SearchComponent(
                searchcomponentid=details["searchcomponentid"],
                name=details["name"],
                icon=details["icon"],
                modulename=details["modulename"],
                classname=details["classname"],
                type=details["type"],
                componentpath=details["componentpath"],
                componentname=details["componentname"],
                sortorder=details["sortorder"],
                enabled=details["enabled"],
            )

            instance.save() 

def update(self, source):
        """
        Updates an existing search component in the arches db

        """

        dt_source = imp.load_source("", source)
        name = dt_source.details["componentname"]

        self.unregister(name)
        self.register(source)