Python pkgutil.get_data() Examples

The following are code examples for showing how to use pkgutil.get_data(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: openhatch   Author: campbe13   File: py26.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _get_data(package, resource):
    loader = pkgutil.get_loader(package)
    if loader is None or not hasattr(loader, 'get_data'):
        return None
    mod = sys.modules.get(package) or loader.load_module(package)
    if mod is None or not hasattr(mod, '__file__'):
        return None

    # Modify the resource name to be compatible with the loader.get_data
    # signature - an os.path format "filename" starting with the dirname of
    # the package's __file__
    parts = resource.split('/')
    parts.insert(0, os.path.dirname(mod.__file__))
    resource_name = os.path.join(*parts)
    return loader.get_data(resource_name)

# pkgutil.get_data() not available in python 2.5
# see http://docs.python.org/release/2.5/lib/module-pkgutil.html 
Example 2
Project: pyPRMS   Author: paknorton   File: ValidParams.py    MIT License 6 votes vote down vote up
def filename(self, filename=None):
        """Set the XML file name.

        If no filename is specified an library-internal XML file is read.

        :param filename: name of XML parameter file
        :type filename: str or None
        """

        self.__filename = filename

        if filename is not None:
            self.__xml_tree = xmlET.parse(self.__filename)
        else:
            # Use the package parameters.xml by default
            xml_fh = io.StringIO(pkgutil.get_data('pyPRMS', 'xml/parameters.xml').decode('utf-8'))
            self.__xml_tree = xmlET.parse(xml_fh)

        self.__isloaded = False
        self._read()
        self.__isloaded = True 
Example 3
Project: NiujiaoDebugger   Author: MrSrc   File: driver.py    GNU General Public License v3.0 6 votes vote down vote up
def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 
Example 4
Project: adversarial-policies   Author: HumanCompatibleAI   File: gym_compete.py    MIT License 6 votes vote down vote up
def load_zoo_agent_params(tag, env_name, index):
    """Loads parameters for the gym_compete zoo agent, but does not restore them.
    :param tag: (str) version of the zoo agent (e.g. '1', '2', '3').
    :param env_name: (str) Gym environment ID
    :param index: (int) the player ID of the agent we want to load ('0' or '1')
    :return a NumPy array of policy weights."""
    # Load parameters
    canonical_env = env_name_to_canonical(env_name)
    agent_dir = os.path.join("agent_zoo", canonical_env)

    if is_symmetric(env_name):  # asymmetric version, parameters tagged with agent id
        symmetric_fname = f"agent_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, symmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)
    else:  # symmetric version, parameters not associated with a specific agent
        asymmetric_fname = f"agent{index + 1}_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, asymmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)

    pylog.info(f"Loaded zoo parameters from '{path}'")

    return pickle.loads(params_pkl) 
Example 5
Project: onering   Author: panyam   File: templates.py    Apache License 2.0 6 votes vote down vote up
def get_source(self, environment, template_name):
        final_path = template_name
        if not template_name.startswith("/"):
            for tdir in self.template_dirs:
                full_path = os.path.join(tdir, template_name)
                if os.path.isfile(full_path):
                    final_path = full_path
                    break
                else:
                    full_path = os.path.join(tdir, template_name + self.template_extension)
                    if os.path.isfile(full_path):
                        final_path = full_path
                        break
            else:
                # See if parent can return it
                if self.parent_loader:
                    return self.parent_loader.get_source(environment, template_name)
                else:
                    source = pkgutil.get_data("onering", "data/templates/" + template_name).decode('utf-8')
                    return source, final_path, lambda: True

        with file(final_path) as f:
            source = f.read().decode('utf-8')
        return source, final_path, lambda: mtime == getmtime(final_path) 
Example 6
Project: onering   Author: panyam   File: templates.py    Apache License 2.0 6 votes vote down vote up
def get_source(self, environment, template_name):
        final_path = template_name
        if not template_name.startswith("/"):
            for tdir in self.template_dirs:
                full_path = os.path.join(tdir, template_name)
                if os.path.isfile(full_path):
                    final_path = full_path
                    break
                else:
                    full_path = os.path.join(tdir, template_name + self.template_extension)
                    if os.path.isfile(full_path):
                        final_path = full_path
                        break
            else:
                # See if parent can return it
                if self.parent_loader:
                    return self.parent_loader.get_source(environment, template_name)
                else:
                    source = pkgutil.get_data("onering", "data/templates/" + template_name).decode('utf-8')
                    return source, final_path, lambda: True

        with file(final_path) as f:
            source = f.read().decode('utf-8')
        return source, final_path, lambda: mtime == getmtime(final_path) 
Example 7
Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    Apache License 2.0 6 votes vote down vote up
def get_data():
		'''
		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                    plot
		filename                 subjectivity_html/obj/2002/Abandon.html
		text           A senior at an elite college (Katie Holmes), a...
		movie_name                                               abandon
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 
Example 8
Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    Apache License 2.0 6 votes vote down vote up
def get_full_data():
		'''
		Returns all plots and reviews, not just the ones that appear in movies with both plot descriptions and reviews.

		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                             plot
		text                    Vijay Singh Rajput (Amitabh Bachchan) is a qui...
		movie_name                                                        aankhen
		has_plot_and_reviews                                                False
		Name: 0, dtype: object
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus_full.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 
Example 9
Project: scattertext   Author: JasonKessler   File: SampleLexicons.py    Apache License 2.0 6 votes vote down vote up
def get_data():
        '''
        Returns
        -------
        pd.DataFrame

        I.e.,
        >>> print(WhissellsLexicon.get_data())
                     activation  imagery  pleasantness
        word
        a                1.3846      1.0        2.0000
        abandon          2.3750      2.4        1.0000
        abandoned        2.1000      3.0        1.1429
        abandonment      2.0000      1.4        1.0000
        abated           1.3333      1.2        1.6667
        '''
        data_stream = pkgutil.get_data('scattertext', 'data/whissells_df.csv')
        return pd.read_csv(
            io.StringIO(data_stream.decode('utf8'))
        ).set_index('word').astype(np.float64) 
Example 10
Project: ironpython2   Author: IronLanguages   File: driver.py    Apache License 2.0 6 votes vote down vote up
def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 
Example 11
Project: unhuman-resources   Author: agajdosi   File: tldextract.py    GNU General Public License v3.0 6 votes vote down vote up
def _cache_tlds(self, tlds):
        '''Logs a diff of the new TLDs and caches them on disk, according to
        settings passed to __init__.'''
        if LOG.isEnabledFor(logging.DEBUG):
            import difflib
            snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
            snapshot = sorted(json.loads(snapshot_data.decode('utf-8')))
            new = sorted(tlds)
            LOG.debug('computed TLD diff:\n%s', '\n'.join(difflib.unified_diff(
                snapshot,
                new,
                fromfile=".tld_set_snapshot",
                tofile=self.cache_file
            )))

        if self.cache_file:
            try:
                with open(self.cache_file, 'w') as cache_file:
                    json.dump(tlds, cache_file)
            except IOError as ioe:
                LOG.warning("unable to cache TLDs in file %s: %s", self.cache_file, ioe) 
Example 12
Project: unhuman-resources   Author: agajdosi   File: tldextract.py    GNU General Public License v3.0 6 votes vote down vote up
def _cache_tlds(self, tlds):
        '''Logs a diff of the new TLDs and caches them on disk, according to
        settings passed to __init__.'''
        if LOG.isEnabledFor(logging.DEBUG):
            import difflib
            snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
            snapshot = sorted(json.loads(snapshot_data.decode('utf-8')))
            new = sorted(tlds)
            LOG.debug('computed TLD diff:\n%s', '\n'.join(difflib.unified_diff(
                snapshot,
                new,
                fromfile=".tld_set_snapshot",
                tofile=self.cache_file
            )))

        if self.cache_file:
            try:
                with open(self.cache_file, 'w') as cache_file:
                    json.dump(tlds, cache_file)
            except IOError as ioe:
                LOG.warning("unable to cache TLDs in file %s: %s", self.cache_file, ioe) 
Example 13
Project: koku   Author: project-koku   File: 0001_initial_squashed_0007_auto_20190208_0316.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def load_report_map_data(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')

    data = pkgutil.get_data('reporting_common',
                            'data/aws_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0003_auto_20180928_1732 
Example 14
Project: koku   Author: project-koku   File: 0001_initial_squashed_0007_auto_20190208_0316.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def load_ocp_report_map_data(apps, schema_editor):
    """Load OCP Usage report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0005_auto_20181127_2046 
Example 15
Project: koku   Author: project-koku   File: 0001_initial_squashed_0007_auto_20190208_0316.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def reload_ocp_map_0005(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ocp_items = ReportColumnMap.objects.filter(provider_type='OCP')
    ocp_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0007_auto_20190208_0316 
Example 16
Project: pyblish-win   Author: pyblish   File: test_pkgutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 
Example 17
Project: pyblish-win   Author: pyblish   File: test_pkgutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)
        del sys.path[0]

        del sys.modules[pkg] 
Example 18
Project: pyblish-win   Author: pyblish   File: test_pkgutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_data(self, path):
            return "Hello, world!" 
Example 19
Project: pyblish-win   Author: pyblish   File: test_pkgutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_getdata_pep302(self):
        # Use a dummy importer/loader
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        del sys.modules['foo'] 
Example 20
Project: Splunk_CBER_App   Author: MHaggis   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, _ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 21
Project: AshsSDK   Author: thehappydinoa   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 22
Project: ngo-addons-backport   Author: camptocamp   File: test_load.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def test_bigfile(self):
        data = json.loads(pkgutil.get_data(self.__module__, 'contacts_big.json'))
        result = self.registry('res.partner').load(
            self.cr, openerp.SUPERUSER_ID,
            ['name', 'mobile', 'email', 'image'],
            data)
        self.assertFalse(result['messages'])
        self.assertEqual(len(result['ids']), len(data)) 
Example 23
Project: ngo-addons-backport   Author: camptocamp   File: test_load.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def test_backlink(self):
        data = json.loads(pkgutil.get_data(self.__module__, 'contacts.json'))
        result = self.registry('res.partner').load(
            self.cr, openerp.SUPERUSER_ID,
            ["name", "type", "street", "city", "country_id", "category_id",
             "supplier", "customer", "is_company", "parent_id"],
            data)
        self.assertFalse(result['messages'])
        self.assertEqual(len(result['ids']), len(data)) 
Example 24
Project: pyPRMS   Author: paknorton   File: ValidParams.py    MIT License 5 votes vote down vote up
def __init__(self, filename=None):
        """Create ValidParams object.

        Read an XML file of parameters to use as a master of valid PRMS
        parameters. If no filename is specified an internal library XML file
        is read.

        :param filename: name of XML parameter file
        :type filename: str or None
        """

        super(ValidParams, self).__init__()

        self.__filename = filename

        if filename:
            self.__xml_tree = xmlET.parse(self.__filename)
        else:
            # Use the package file, parameters.xml, by default
            xml_fh = io.StringIO(pkgutil.get_data('pyPRMS', 'xml/parameters.xml').decode('utf-8'))
            self.__xml_tree = xmlET.parse(xml_fh)

        # TODO: need more robust logic here; currently no way to handle failures
        self.__isloaded = False
        self._read()
        self.__isloaded = True 
Example 25
Project: misp42splunk   Author: remg427   File: _utils.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 
Example 26
Project: core   Author: getavalon   File: _utils.py    MIT License 5 votes vote down vote up
def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 
Example 27
Project: NiujiaoDebugger   Author: MrSrc   File: test_pkgutil.py    GNU General Public License v3.0 5 votes vote down vote up
def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 
Example 28
Project: NiujiaoDebugger   Author: MrSrc   File: test_pkgutil.py    GNU General Public License v3.0 5 votes vote down vote up
def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        names = []
        for moduleinfo in pkgutil.iter_modules([zip_file]):
            self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo)
            names.append(moduleinfo.name)
        self.assertEqual(names, ['test_getdata_zipfile'])

        del sys.path[0]

        del sys.modules[pkg] 
Example 29
Project: NiujiaoDebugger   Author: MrSrc   File: test_pkgutil.py    GNU General Public License v3.0 5 votes vote down vote up
def get_data(self, path):
            return "Hello, world!" 
Example 30
Project: NiujiaoDebugger   Author: MrSrc   File: test_pkgutil.py    GNU General Public License v3.0 5 votes vote down vote up
def test_alreadyloaded(self):
        # Ensure that get_data works without reloading - the "loads" module
        # variable in the example loader should count how many times a reload
        # occurs.
        import foo
        self.assertEqual(foo.loads, 1)
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        self.assertEqual(foo.loads, 1)
        del sys.modules['foo']


# These tests, especially the setup and cleanup, are hideous. They
# need to be cleaned up once issue 14715 is addressed. 
Example 31
Project: tempo-cnn   Author: hendriks73   File: classifier.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _extract_from_package(resource):
    data = pkgutil.get_data('tempocnn', resource)
    with tempfile.NamedTemporaryFile(prefix='model', suffix='.h5', delete=False) as f:
        f.write(data)
        name = f.name
    return name 
Example 32
Project: cloudify-agent   Author: cloudify-cosmo   File: utils.py    Apache License 2.0 5 votes vote down vote up
def get_agent_version():
    data = pkgutil.get_data('cloudify_agent', 'VERSION')
    return json.loads(data)['version'] 
Example 33
Project: cloudify-agent   Author: cloudify-cosmo   File: test_main.py    Apache License 2.0 5 votes vote down vote up
def test_version(self):
        mock_logger = mock.Mock()
        with mock.patch('cloudify_agent.shell.main.get_logger',
                        return_value=mock_logger):
            self._run('cfy-agent --version')
        self.assertEqual(1, len(mock_logger.mock_calls))
        version = json.loads(
            pkgutil.get_data('cloudify_agent', 'VERSION'))['version']
        log_args = mock_logger.mock_calls[0][1]
        logged_output = log_args[0]
        self.assertIn(version, logged_output) 
Example 34
Project: recruit   Author: Frank-qlu   File: __init__.py    Apache License 2.0 5 votes vote down vote up
def get_resource(self, request, filename):
        """Return a static resource from the shared folder."""
        filename = join("shared", basename(filename))
        try:
            data = pkgutil.get_data(__package__, filename)
        except OSError:
            data = None
        if data is not None:
            mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
            return Response(data, mimetype=mimetype)
        return Response("Not Found", status=404) 
Example 35
Project: recruit   Author: Frank-qlu   File: __init__.py    Apache License 2.0 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 36
Project: adversarial-policies   Author: HumanCompatibleAI   File: train.py    MIT License 5 votes vote down vote up
def load_default(env_name, config_dir):
    default_config = DEFAULT_CONFIGS.get(env_name, "default.json")
    fname = os.path.join("configs", config_dir, default_config)
    config = pkgutil.get_data("aprl", fname)
    return json.loads(config) 
Example 37
Project: aws-cfn-plex   Author: lordmuffin   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 38
Project: onering   Author: panyam   File: context.py    Apache License 2.0 5 votes vote down vote up
def load_core_schemas(self):
        from onering import dsl
        source = pkgutil.get_data("onering", "data/schemas/core.schema").decode('utf-8')
        parser = dsl.parser.Parser(source, self)
        parser.parse()
        set_trace() 
Example 39
Project: onering   Author: panyam   File: context.py    Apache License 2.0 5 votes vote down vote up
def load_core_schemas(self):
        from onering import dsl
        source = pkgutil.get_data("onering", "data/schemas/core.schema").decode('utf-8')
        parser = dsl.parser.Parser(source, self)
        parser.parse()
        set_trace() 
Example 40
Project: grand_central   Author: amiracle   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 41
Project: alexa-apple-calendar   Author: zanderxyz   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 42
Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    Apache License 2.0 5 votes vote down vote up
def _convention_speech_iter():
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/political_data.json').decode('utf-8')
		except:
			url = POLITICAL_DATA_URL
			data_stream = urlopen(url).read().decode('utf-8')
		return json.loads(data_stream) 
Example 43
Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    Apache License 2.0 5 votes vote down vote up
def get_data():
		clean = ConventionData2012._clean_function_factory()
		get_speaker_name = ConventionData2012._speaker_name_factory()
		data = []
		for party, speech in ConventionData2012._iter_party_speech_pairs():
			cleaned_speech = clean(speech)
			speaker_name = get_speaker_name(speech)
			if cleaned_speech and cleaned_speech != '' and speaker_name != '':
				data.append({'party': party,
				             'text': cleaned_speech,
				             'speaker': speaker_name})
		return pd.DataFrame(data) 
Example 44
Project: scattertext   Author: JasonKessler   File: HTMLSemioticSquareViz.py    Apache License 2.0 5 votes vote down vote up
def _get_template(self):
		return pkgutil.get_data('scattertext', SEMIOTIC_SQUARE_HTML_PATH).decode('utf-8') 
Example 45
Project: scattertext   Author: JasonKessler   File: BasicHTMLFromScatterplotStructure.py    Apache License 2.0 5 votes vote down vote up
def get_packaged_html_template_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/' + file_name).decode('utf-8') 
Example 46
Project: scattertext   Author: JasonKessler   File: DefaultBackgroundFrequencies.py    Apache License 2.0 5 votes vote down vote up
def get_background_frequency_df(frequency_path=None):
		if frequency_path:
			unigram_freq_table_buf = open(frequency_path)
		else:
			unigram_freq_table_buf = StringIO(pkgutil.get_data('scattertext', 'data/count_1w.txt')
			                                  .decode('utf-8'))
		to_ret = (pd.read_csv(unigram_freq_table_buf,
                              sep='\t',
		                      names=['word', 'background'])
		          .sort_values(ascending=False, by='background')
		          .drop_duplicates(['word'])
		          .set_index('word'))
		return to_ret 
Example 47
Project: scattertext   Author: JasonKessler   File: test_TermDocMat.py    Apache License 2.0 5 votes vote down vote up
def get_hamlet_docs():
    try:
        cwd = os.path.dirname(os.path.abspath(__file__))
        path = os.path.join(cwd, '..', 'data', 'hamlet.txt')
        buf = open(path).read()
    except:
        buf = pkgutil.get_data('scattertext', os.path.join('data', 'hamlet.txt'))
    hamlet_docs = buf.split('\n\n')
    return hamlet_docs 
Example 48
Project: scattertext   Author: JasonKessler   File: test_CorpusFromParsedDocuments.py    Apache License 2.0 5 votes vote down vote up
def convention_speech_iter():
	relative_path = os.path.join('../scattertext/data', 'political_data.json')
	try:
		cwd = os.path.dirname(os.path.abspath(__file__))
		path = os.path.join(cwd, relative_path)
		return json.load(open(path))
	except:
		return json.loads(pkgutil.get_data('scattertext', relative_path).decode('utf-8')) 
Example 49
Project: oa_qian   Author: sunqb   File: __init__.py    Apache License 2.0 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 50
Project: onedrive-d   Author: miqui   File: __init__.py    GNU General Public License v3.0 5 votes vote down vote up
def get_content(file_name, pkg_name='onedrive_d', is_text=True):
    """
    Read a resource file in data/.
    :param str file_name:
    :param str pkg_name:
    :param True | False is_text: True to indicate the text is UTF-8 encoded.
    :return str | bytes: Content of the file.
    """
    content = pkgutil.get_data(pkg_name, 'data/' + file_name)
    if is_text:
        content = content.decode('utf-8')
    return content 
Example 51
Project: lingvo   Author: tensorflow   File: py_utils.py    Apache License 2.0 5 votes vote down vote up
def ReadFileLines(file_path):
  """Read a text file and return the lines.

  If the file cannot be found at the given path, attempt to load it from the
  Lingvo package (useful for data dependencies in par files).

  Args:
    file_path: path to file, either absolute or relative to the bazel workspace.

  Returns:
    A list of lines from the file.
  """
  if not tf.io.gfile.exists(file_path):
    try:
      lines = pkgutil.get_data(
          'lingvo', file_path.replace('lingvo/', '',
                                      1)).splitlines(True)
    except IOError:
      # If pkgutil can't find the file, continue and let GFile raise the error.
      lines = None
  else:
    lines = None

  if not lines:
    with tf.io.gfile.GFile(file_path, 'r') as f:
      lines = f.readlines()

  return lines


# Partially borrowed from
# https://github.com/tensorflow/tensor2tensor/blob/32929305e1a4ec926eff24123758b794df35492b/tensor2tensor/layers/common_layers.py#L349 
Example 52
Project: FUTU_Stop_Loss   Author: BigtoC   File: __init__.py    MIT License 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example 53
Project: gfwlist2dnsmasq   Author: mysqto   File: main.py    MIT License 5 votes vote down vote up
def combine_rules(content, user_rule = None):
	builtin_rules = pkgutil.get_data('gfwlist2dnsmasq', 
							'resources/builtin.txt').decode().splitlines(False)
	gfwlist = content.decode().splitlines(False)
	gfwlist.extend(builtin_rules)

	if user_rule:
		gfwlist.extend(user_rule.decode().splitlines(False))

	return gfwlist 
Example 54
Project: gfwlist2dnsmasq   Author: mysqto   File: main.py    MIT License 5 votes vote down vote up
def reduce_domains(domains):
	# reduce 'www.google.com' to 'google.com'
	# remove invalid domains
	tld_content = pkgutil.get_data('gfwlist2dnsmasq', 'resources/tld.txt')
	try:
		tld_content = tld_content.decode()
	except:
		tld_content = tld_content
	tlds = set(tld_content.splitlines(False))
	new_domains = set()
	for domain in domains:
		domain_parts = domain.split('.')
		last_root_domain = None
		for i in xrange(0, len(domain_parts)):
			root_domain = '.'.join(domain_parts[len(domain_parts) - i - 1:])
			if i == 0:
				if not tlds.__contains__(root_domain):
					# root_domain is not a valid tld
					break
			last_root_domain = root_domain
			if tlds.__contains__(root_domain):
				continue
			else:
				break
		if last_root_domain is not None:
			new_domains.add(last_root_domain)
	return sorted(new_domains) 
Example 55
Project: cloudbase-init-ci   Author: cloudbase   File: util.py    Apache License 2.0 5 votes vote down vote up
def get_resource(resource):
    """Get the given resource from the list of known resources."""
    return pkgutil.get_data('argus.resources', resource) 
Example 56
Project: pyPro4Sail   Author: hectornieto   File: spectral_library.py    GNU General Public License v3.0 5 votes vote down vote up
def get_spectra():
    """Reads the spectral information and stores is for future use."""

    # PROSPECT-D
    prospect_d_spectraf = pkgutil.get_data('pyPro4Sail', 'prospect_d_spectra.txt')
    wl, nr, kab, kcar, kant, kbrown, kw, km= np.loadtxt(
        BytesIO(prospect_d_spectraf), unpack=True)
    #prospect_d_spectra = ProspectDSpectra(wl, nr, kab, kcar, kbrown, kw, km, kant)

    return wl, nr, kab, kcar, kbrown, kw, km, kant 
Example 57
Project: ironpython2   Author: IronLanguages   File: test_pkgutil.py    Apache License 2.0 5 votes vote down vote up
def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 
Example 58
Project: ironpython2   Author: IronLanguages   File: test_pkgutil.py    Apache License 2.0 5 votes vote down vote up
def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)
        del sys.path[0]

        del sys.modules[pkg] 
Example 59
Project: ironpython2   Author: IronLanguages   File: test_pkgutil.py    Apache License 2.0 5 votes vote down vote up
def get_data(self, path):
            return "Hello, world!" 
Example 60
Project: ironpython2   Author: IronLanguages   File: test_pkgutil.py    Apache License 2.0 5 votes vote down vote up
def test_alreadyloaded(self):
        # Ensure that get_data works without reloading - the "loads" module
        # variable in the example loader should count how many times a reload
        # occurs.
        import foo
        self.assertEqual(foo.loads, 1)
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        self.assertEqual(foo.loads, 1)
        del sys.modules['foo'] 
Example 61
Project: unhuman-resources   Author: agajdosi   File: tldextract.py    GNU General Public License v3.0 5 votes vote down vote up
def _get_snapshot_tld_extractor():
        snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
        return json.loads(snapshot_data.decode('utf-8')) 
Example 62
Project: unhuman-resources   Author: agajdosi   File: tldextract.py    GNU General Public License v3.0 5 votes vote down vote up
def _get_snapshot_tld_extractor():
        snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
        return json.loads(snapshot_data.decode('utf-8')) 
Example 63
Project: maze-builder   Author: kcsaff   File: illustrators.py    MIT License 5 votes vote down vote up
def resource(resource_name):
    return pkgutil.get_data(RESOURCE_PACKAGE, resource_name).decode('utf-8') 
Example 64
Project: maze-builder   Author: kcsaff   File: template.py    MIT License 5 votes vote down vote up
def resource(resource_name):
    return pkgutil.get_data(RESOURCE_PACKAGE, resource_name).decode('utf-8') 
Example 65
Project: koku   Author: project-koku   File: 0001_initial_squashed_0007_auto_20190208_0316.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def reload_ocp_map_0007(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ocp_items = ReportColumnMap.objects.filter(provider_type='OCP')
    ocp_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 
Example 66
Project: koku   Author: project-koku   File: 0001_initial_squashed_0007_auto_20190208_0316.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def reload_aws_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    aws_items = ReportColumnMap.objects.filter(provider_type='AWS')
    aws_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/aws_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 
Example 67
Project: koku   Author: project-koku   File: 0016_auto_20190829_2053.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def reload_azure_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    azure_items = ReportColumnMap.objects.filter(provider_type='AZURE')
    azure_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/azure_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 
Example 68
Project: koku   Author: project-koku   File: 0018_auto_20190923_1838.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def reload_azure_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    azure_items = ReportColumnMap.objects.filter(provider_type='AZURE')
    azure_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/azure_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 
Example 69
Project: koku   Author: project-koku   File: 0022_add_gcp_column_maps.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def load_gcp_column_map(apps, schema_editor):
    """Load GCP column map to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ReportColumnMap.objects.filter(provider_type='GCP').delete()

    data = pkgutil.get_data('reporting_common',
                            'data/gcp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 
Example 70
Project: koku   Author: project-koku   File: 0040_auto_20191121_2154.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def load_openshift_metric_map(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    CostModelMetricsMap = apps.get_model('api', 'CostModelMetricsMap')
    CostModelMetricsMap.objects.all().delete()

    data = pkgutil.get_data('api',
                            'metrics/data/cost_models_metric_map.json')

    data = json.loads(data)

    for entry in data:
        map = CostModelMetricsMap(**entry)
        map.save() 
Example 71
Project: koku   Author: project-koku   File: 0010_costmodelmetricsmap.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def load_openshift_metric_map(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    CostModelMetricsMap = apps.get_model('api', 'CostModelMetricsMap')

    data = pkgutil.get_data('api',
                            'metrics/data/cost_models_metric_map.json')

    data = json.loads(data)

    for entry in data:
        map = CostModelMetricsMap(**entry)
        map.save() 
Example 72
Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def populate_storage_line_item_daily_table(self, start_date, end_date, cluster_id):
        """Populate the daily storage aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            cluster_id (String) Cluster Identifier

        Returns
            (None)

        """
        # Cast string to date object
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
            end_date = end_date.date()
        table_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily']

        daily_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpstoragelineitem_daily.sql'
        )
        daily_sql = daily_sql.decode('utf-8')
        daily_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'cluster_id': cluster_id,
            'schema': self.schema
        }
        daily_sql, daily_sql_params = self.jinja_sql.prepare_query(
            daily_sql, daily_sql_params
        )
        self._execute_raw_sql_query(
            table_name, daily_sql, start_date, end_date, bind_params=list(daily_sql_params)) 
Example 73
Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def populate_pod_charge(self, cpu_temp_table, mem_temp_table):
        """Populate the memory and cpu charge on daily summary table.

        Args:
            cpu_temp_table (String) Name of cpu charge temp table
            mem_temp_table (String) Name of mem charge temp table

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        daily_charge_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpusagelineitem_daily_pod_charge.sql'
        )
        charge_line_sql = daily_charge_sql.decode('utf-8')
        charge_line_sql_params = {
            'cpu_temp': cpu_temp_table,
            'mem_temp': mem_temp_table,
            'schema': self.schema
        }
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(
            charge_line_sql, charge_line_sql_params
        )
        self._execute_raw_sql_query(
            table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 
Example 74
Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def populate_storage_charge(self, temp_table_name):
        """Populate the storage charge into the daily summary table.

        Args:
            storage_charge (Float) Storage charge.

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        daily_charge_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocp_storage_charge.sql'
        )
        charge_line_sql = daily_charge_sql.decode('utf-8')
        charge_line_sql_params = {
            'temp_table': temp_table_name,
            'schema': self.schema
        }
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(
            charge_line_sql, charge_line_sql_params
        )
        self._execute_raw_sql_query(
            table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 
Example 75
Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def populate_line_item_daily_summary_table(self, start_date, end_date, cluster_id):
        """Populate the daily aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            cluster_id (String) Cluster Identifier

        Returns
            (None)

        """
        # Cast start_date to date
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
            end_date = end_date.date()
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        summary_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpusagelineitem_daily_summary.sql'
        )
        summary_sql = summary_sql.decode('utf-8')
        summary_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'cluster_id': cluster_id,
            'schema': self.schema
        }
        summary_sql, summary_sql_params = self.jinja_sql.prepare_query(
            summary_sql, summary_sql_params
        )
        self._execute_raw_sql_query(
            table_name, summary_sql, start_date, end_date, bind_params=list(summary_sql_params)) 
Example 76
Project: pyblish-win   Author: pyblish   File: __init__.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def bootstrap(root=None, upgrade=False, user=False,
              altinstall=False, default_pip=True,
              verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory).

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _require_ssl_for_pip()
    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    tmpdir = tempfile.mkdtemp()
    try:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True) 
Example 77
Project: NiujiaoDebugger   Author: MrSrc   File: __init__.py    GNU General Public License v3.0 4 votes vote down vote up
def _bootstrap(*, root=None, upgrade=False, user=False,
              altinstall=False, default_pip=False,
              verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory). Returns pip command status code.

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    with tempfile.TemporaryDirectory() as tmpdir:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) 
Example 78
Project: klondike   Author: planetlabs   File: get-pip.py    Apache License 2.0 4 votes vote down vote up
def bootstrap(tmpdir=None):
    # Import pip so we can use it to install pip and maybe setuptools too
    import pip

    # We always want to install pip
    packages = ["pip"]

    # Check if the user has requested us not to install setuptools
    if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
        args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
    else:
        args = sys.argv[1:]

        # We want to see if setuptools is available before attempting to
        # install it
        try:
            import setuptools
        except ImportError:
            packages += ["setuptools"]

    delete_tmpdir = False
    try:
        # Create a temporary directory to act as a working directory if we were
        # not given one.
        if tmpdir is None:
            tmpdir = tempfile.mkdtemp()
            delete_tmpdir = True

        # We need to extract the SSL certificates from requests so that they
        # can be passed to --cert
        cert_path = os.path.join(tmpdir, "cacert.pem")
        with open(cert_path, "wb") as cert:
            cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))

        # Use an environment variable here so that users can still pass
        # --cert via sys.argv
        os.environ.setdefault("PIP_CERT", cert_path)

        # Execute the included pip and use it to install the latest pip and
        # setuptools from PyPI
        sys.exit(pip.main(["install", "--upgrade"] + packages + args))
    finally:
        # Remove our temporary directory
        if delete_tmpdir and tmpdir:
            shutil.rmtree(tmpdir, ignore_errors=True) 
Example 79
Project: ironpython2   Author: IronLanguages   File: __init__.py    Apache License 2.0 4 votes vote down vote up
def _bootstrap(root=None, upgrade=False, user=False,
               altinstall=False, default_pip=True,
               verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory). Returns pip command status code.

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    tmpdir = tempfile.mkdtemp()
    try:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if sys.platform == 'cli': # IronPython doesn't do bytecode compilation
            args += ["--no-compile"]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True) 
Example 80
Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    GNU Affero General Public License v3.0 4 votes vote down vote up
def get_ocp_infrastructure_map(self, start_date, end_date, **kwargs):
        """Get the OCP on infrastructure map.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.

        Returns
            (None)

        """
        # kwargs here allows us to optionally pass in a provider UUID based on
        # the provider type this is run for
        ocp_provider_uuid = kwargs.get('ocp_provider_uuid')
        aws_provider_uuid = kwargs.get('aws_provider_uuid')
        azure_provider_uuid = kwargs.get('azure_provider_uuid')
        # In case someone passes this function a string instead of the date object like we asked...
        # Cast the string into a date object, end_date into date object instead of string
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        infra_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpinfrastructure_provider_map.sql'
        )
        infra_sql = infra_sql.decode('utf-8')
        infra_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'schema': self.schema,
            'aws_provider_uuid': aws_provider_uuid,
            'ocp_provider_uuid': ocp_provider_uuid,
            'azure_provider_uuid': azure_provider_uuid
        }
        infra_sql, infra_sql_params = self.jinja_sql.prepare_query(
            infra_sql, infra_sql_params)
        with connection.cursor() as cursor:
            cursor.db.set_schema(self.schema)
            cursor.execute(infra_sql, list(infra_sql_params))
            results = cursor.fetchall()

        db_results = {}
        for entry in results:
            # This dictionary is keyed on an OpenShift provider UUID
            # and the tuple contains
            # (Infrastructure Provider UUID, Infrastructure Provider Type)
            db_results[entry[0]] = (entry[1], entry[2])

        return db_results