Python pkgutil.get_data() Examples

def _get_data(package, resource):
    loader = pkgutil.get_loader(package)
    if loader is None or not hasattr(loader, 'get_data'):
        return None
    mod = sys.modules.get(package) or loader.load_module(package)
    if mod is None or not hasattr(mod, '__file__'):
        return None

    # Modify the resource name to be compatible with the loader.get_data
    # signature - an os.path format "filename" starting with the dirname of
    # the package's __file__
    parts = resource.split('/')
    parts.insert(0, os.path.dirname(mod.__file__))
    resource_name = os.path.join(*parts)
    return loader.get_data(resource_name)

# pkgutil.get_data() not available in python 2.5
# see http://docs.python.org/release/2.5/lib/module-pkgutil.html 

def filename(self, filename=None):
        """Set the XML file name.

        If no filename is specified an library-internal XML file is read.

        :param filename: name of XML parameter file
        :type filename: str or None
        """

        self.__filename = filename

        if filename is not None:
            self.__xml_tree = xmlET.parse(self.__filename)
        else:
            # Use the package parameters.xml by default
            xml_fh = io.StringIO(pkgutil.get_data('pyPRMS', 'xml/parameters.xml').decode('utf-8'))
            self.__xml_tree = xmlET.parse(xml_fh)

        self.__isloaded = False
        self._read()
        self.__isloaded = True 

def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 

def load_zoo_agent_params(tag, env_name, index):
    """Loads parameters for the gym_compete zoo agent, but does not restore them.
    :param tag: (str) version of the zoo agent (e.g. '1', '2', '3').
    :param env_name: (str) Gym environment ID
    :param index: (int) the player ID of the agent we want to load ('0' or '1')
    :return a NumPy array of policy weights."""
    # Load parameters
    canonical_env = env_name_to_canonical(env_name)
    agent_dir = os.path.join("agent_zoo", canonical_env)

    if is_symmetric(env_name):  # asymmetric version, parameters tagged with agent id
        symmetric_fname = f"agent_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, symmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)
    else:  # symmetric version, parameters not associated with a specific agent
        asymmetric_fname = f"agent{index + 1}_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, asymmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)

    pylog.info(f"Loaded zoo parameters from '{path}'")

    return pickle.loads(params_pkl) 

def get_source(self, environment, template_name):
        final_path = template_name
        if not template_name.startswith("/"):
            for tdir in self.template_dirs:
                full_path = os.path.join(tdir, template_name)
                if os.path.isfile(full_path):
                    final_path = full_path
                    break
                else:
                    full_path = os.path.join(tdir, template_name + self.template_extension)
                    if os.path.isfile(full_path):
                        final_path = full_path
                        break
            else:
                # See if parent can return it
                if self.parent_loader:
                    return self.parent_loader.get_source(environment, template_name)
                else:
                    source = pkgutil.get_data("onering", "data/templates/" + template_name).decode('utf-8')
                    return source, final_path, lambda: True

        with file(final_path) as f:
            source = f.read().decode('utf-8')
        return source, final_path, lambda: mtime == getmtime(final_path) 

def get_source(self, environment, template_name):
        final_path = template_name
        if not template_name.startswith("/"):
            for tdir in self.template_dirs:
                full_path = os.path.join(tdir, template_name)
                if os.path.isfile(full_path):
                    final_path = full_path
                    break
                else:
                    full_path = os.path.join(tdir, template_name + self.template_extension)
                    if os.path.isfile(full_path):
                        final_path = full_path
                        break
            else:
                # See if parent can return it
                if self.parent_loader:
                    return self.parent_loader.get_source(environment, template_name)
                else:
                    source = pkgutil.get_data("onering", "data/templates/" + template_name).decode('utf-8')
                    return source, final_path, lambda: True

        with file(final_path) as f:
            source = f.read().decode('utf-8')
        return source, final_path, lambda: mtime == getmtime(final_path) 

def get_data():
		'''
		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                    plot
		filename                 subjectivity_html/obj/2002/Abandon.html
		text           A senior at an elite college (Katie Holmes), a...
		movie_name                                               abandon
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

def get_full_data():
		'''
		Returns all plots and reviews, not just the ones that appear in movies with both plot descriptions and reviews.

		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                             plot
		text                    Vijay Singh Rajput (Amitabh Bachchan) is a qui...
		movie_name                                                        aankhen
		has_plot_and_reviews                                                False
		Name: 0, dtype: object
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus_full.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

def get_data():
        '''
        Returns
        -------
        pd.DataFrame

        I.e.,
        >>> print(WhissellsLexicon.get_data())
                     activation  imagery  pleasantness
        word
        a                1.3846      1.0        2.0000
        abandon          2.3750      2.4        1.0000
        abandoned        2.1000      3.0        1.1429
        abandonment      2.0000      1.4        1.0000
        abated           1.3333      1.2        1.6667
        '''
        data_stream = pkgutil.get_data('scattertext', 'data/whissells_df.csv')
        return pd.read_csv(
            io.StringIO(data_stream.decode('utf8'))
        ).set_index('word').astype(np.float64) 

def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 

def _cache_tlds(self, tlds):
        '''Logs a diff of the new TLDs and caches them on disk, according to
        settings passed to __init__.'''
        if LOG.isEnabledFor(logging.DEBUG):
            import difflib
            snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
            snapshot = sorted(json.loads(snapshot_data.decode('utf-8')))
            new = sorted(tlds)
            LOG.debug('computed TLD diff:\n%s', '\n'.join(difflib.unified_diff(
                snapshot,
                new,
                fromfile=".tld_set_snapshot",
                tofile=self.cache_file
            )))

        if self.cache_file:
            try:
                with open(self.cache_file, 'w') as cache_file:
                    json.dump(tlds, cache_file)
            except IOError as ioe:
                LOG.warning("unable to cache TLDs in file %s: %s", self.cache_file, ioe) 

def _cache_tlds(self, tlds):
        '''Logs a diff of the new TLDs and caches them on disk, according to
        settings passed to __init__.'''
        if LOG.isEnabledFor(logging.DEBUG):
            import difflib
            snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
            snapshot = sorted(json.loads(snapshot_data.decode('utf-8')))
            new = sorted(tlds)
            LOG.debug('computed TLD diff:\n%s', '\n'.join(difflib.unified_diff(
                snapshot,
                new,
                fromfile=".tld_set_snapshot",
                tofile=self.cache_file
            )))

        if self.cache_file:
            try:
                with open(self.cache_file, 'w') as cache_file:
                    json.dump(tlds, cache_file)
            except IOError as ioe:
                LOG.warning("unable to cache TLDs in file %s: %s", self.cache_file, ioe) 

def load_report_map_data(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')

    data = pkgutil.get_data('reporting_common',
                            'data/aws_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0003_auto_20180928_1732 

def load_ocp_report_map_data(apps, schema_editor):
    """Load OCP Usage report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0005_auto_20181127_2046 

def reload_ocp_map_0005(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ocp_items = ReportColumnMap.objects.filter(provider_type='OCP')
    ocp_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        del entry['report_type']
        if entry['database_table'] == "reporting_ocpstoragelineitem":
            continue
        map = ReportColumnMap(**entry)
        map.save()

# reporting_common.migrations.0007_auto_20190208_0316 

def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 

def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)
        del sys.path[0]

        del sys.modules[pkg] 

def get_data(self, path):
            return "Hello, world!" 

def test_getdata_pep302(self):
        # Use a dummy importer/loader
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        del sys.modules['foo'] 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, _ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def test_bigfile(self):
        data = json.loads(pkgutil.get_data(self.__module__, 'contacts_big.json'))
        result = self.registry('res.partner').load(
            self.cr, openerp.SUPERUSER_ID,
            ['name', 'mobile', 'email', 'image'],
            data)
        self.assertFalse(result['messages'])
        self.assertEqual(len(result['ids']), len(data)) 

def test_backlink(self):
        data = json.loads(pkgutil.get_data(self.__module__, 'contacts.json'))
        result = self.registry('res.partner').load(
            self.cr, openerp.SUPERUSER_ID,
            ["name", "type", "street", "city", "country_id", "category_id",
             "supplier", "customer", "is_company", "parent_id"],
            data)
        self.assertFalse(result['messages'])
        self.assertEqual(len(result['ids']), len(data)) 

def __init__(self, filename=None):
        """Create ValidParams object.

        Read an XML file of parameters to use as a master of valid PRMS
        parameters. If no filename is specified an internal library XML file
        is read.

        :param filename: name of XML parameter file
        :type filename: str or None
        """

        super(ValidParams, self).__init__()

        self.__filename = filename

        if filename:
            self.__xml_tree = xmlET.parse(self.__filename)
        else:
            # Use the package file, parameters.xml, by default
            xml_fh = io.StringIO(pkgutil.get_data('pyPRMS', 'xml/parameters.xml').decode('utf-8'))
            self.__xml_tree = xmlET.parse(xml_fh)

        # TODO: need more robust logic here; currently no way to handle failures
        self.__isloaded = False
        self._read()
        self.__isloaded = True 

def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 

def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 

def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 

def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = b'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        names = []
        for moduleinfo in pkgutil.iter_modules([zip_file]):
            self.assertIsInstance(moduleinfo, pkgutil.ModuleInfo)
            names.append(moduleinfo.name)
        self.assertEqual(names, ['test_getdata_zipfile'])

        del sys.path[0]

        del sys.modules[pkg] 

def get_data(self, path):
            return "Hello, world!" 

def test_alreadyloaded(self):
        # Ensure that get_data works without reloading - the "loads" module
        # variable in the example loader should count how many times a reload
        # occurs.
        import foo
        self.assertEqual(foo.loads, 1)
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        self.assertEqual(foo.loads, 1)
        del sys.modules['foo']


# These tests, especially the setup and cleanup, are hideous. They
# need to be cleaned up once issue 14715 is addressed. 

def _extract_from_package(resource):
    data = pkgutil.get_data('tempocnn', resource)
    with tempfile.NamedTemporaryFile(prefix='model', suffix='.h5', delete=False) as f:
        f.write(data)
        name = f.name
    return name 

def get_agent_version():
    data = pkgutil.get_data('cloudify_agent', 'VERSION')
    return json.loads(data)['version'] 

def test_version(self):
        mock_logger = mock.Mock()
        with mock.patch('cloudify_agent.shell.main.get_logger',
                        return_value=mock_logger):
            self._run('cfy-agent --version')
        self.assertEqual(1, len(mock_logger.mock_calls))
        version = json.loads(
            pkgutil.get_data('cloudify_agent', 'VERSION'))['version']
        log_args = mock_logger.mock_calls[0][1]
        logged_output = log_args[0]
        self.assertIn(version, logged_output) 

def get_resource(self, request, filename):
        """Return a static resource from the shared folder."""
        filename = join("shared", basename(filename))
        try:
            data = pkgutil.get_data(__package__, filename)
        except OSError:
            data = None
        if data is not None:
            mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
            return Response(data, mimetype=mimetype)
        return Response("Not Found", status=404) 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def load_default(env_name, config_dir):
    default_config = DEFAULT_CONFIGS.get(env_name, "default.json")
    fname = os.path.join("configs", config_dir, default_config)
    config = pkgutil.get_data("aprl", fname)
    return json.loads(config) 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def load_core_schemas(self):
        from onering import dsl
        source = pkgutil.get_data("onering", "data/schemas/core.schema").decode('utf-8')
        parser = dsl.parser.Parser(source, self)
        parser.parse()
        set_trace() 

def load_core_schemas(self):
        from onering import dsl
        source = pkgutil.get_data("onering", "data/schemas/core.schema").decode('utf-8')
        parser = dsl.parser.Parser(source, self)
        parser.parse()
        set_trace() 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def _convention_speech_iter():
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/political_data.json').decode('utf-8')
		except:
			url = POLITICAL_DATA_URL
			data_stream = urlopen(url).read().decode('utf-8')
		return json.loads(data_stream) 

def get_data():
		clean = ConventionData2012._clean_function_factory()
		get_speaker_name = ConventionData2012._speaker_name_factory()
		data = []
		for party, speech in ConventionData2012._iter_party_speech_pairs():
			cleaned_speech = clean(speech)
			speaker_name = get_speaker_name(speech)
			if cleaned_speech and cleaned_speech != '' and speaker_name != '':
				data.append({'party': party,
				             'text': cleaned_speech,
				             'speaker': speaker_name})
		return pd.DataFrame(data) 

def _get_template(self):
		return pkgutil.get_data('scattertext', SEMIOTIC_SQUARE_HTML_PATH).decode('utf-8') 

def get_packaged_html_template_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/' + file_name).decode('utf-8') 

def get_background_frequency_df(frequency_path=None):
		if frequency_path:
			unigram_freq_table_buf = open(frequency_path)
		else:
			unigram_freq_table_buf = StringIO(pkgutil.get_data('scattertext', 'data/count_1w.txt')
			                                  .decode('utf-8'))
		to_ret = (pd.read_csv(unigram_freq_table_buf,
                              sep='\t',
		                      names=['word', 'background'])
		          .sort_values(ascending=False, by='background')
		          .drop_duplicates(['word'])
		          .set_index('word'))
		return to_ret 

def get_hamlet_docs():
    try:
        cwd = os.path.dirname(os.path.abspath(__file__))
        path = os.path.join(cwd, '..', 'data', 'hamlet.txt')
        buf = open(path).read()
    except:
        buf = pkgutil.get_data('scattertext', os.path.join('data', 'hamlet.txt'))
    hamlet_docs = buf.split('\n\n')
    return hamlet_docs 

def convention_speech_iter():
	relative_path = os.path.join('../scattertext/data', 'political_data.json')
	try:
		cwd = os.path.dirname(os.path.abspath(__file__))
		path = os.path.join(cwd, relative_path)
		return json.load(open(path))
	except:
		return json.loads(pkgutil.get_data('scattertext', relative_path).decode('utf-8')) 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def get_content(file_name, pkg_name='onedrive_d', is_text=True):
    """
    Read a resource file in data/.
    :param str file_name:
    :param str pkg_name:
    :param True | False is_text: True to indicate the text is UTF-8 encoded.
    :return str | bytes: Content of the file.
    """
    content = pkgutil.get_data(pkg_name, 'data/' + file_name)
    if is_text:
        content = content.decode('utf-8')
    return content 

def ReadFileLines(file_path):
  """Read a text file and return the lines.

  If the file cannot be found at the given path, attempt to load it from the
  Lingvo package (useful for data dependencies in par files).

  Args:
    file_path: path to file, either absolute or relative to the bazel workspace.

  Returns:
    A list of lines from the file.
  """
  if not tf.io.gfile.exists(file_path):
    try:
      lines = pkgutil.get_data(
          'lingvo', file_path.replace('lingvo/', '',
                                      1)).splitlines(True)
    except IOError:
      # If pkgutil can't find the file, continue and let GFile raise the error.
      lines = None
  else:
    lines = None

  if not lines:
    with tf.io.gfile.GFile(file_path, 'r') as f:
      lines = f.readlines()

  return lines


# Partially borrowed from
# https://github.com/tensorflow/tensor2tensor/blob/32929305e1a4ec926eff24123758b794df35492b/tensor2tensor/layers/common_layers.py#L349 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def combine_rules(content, user_rule = None):
	builtin_rules = pkgutil.get_data('gfwlist2dnsmasq', 
							'resources/builtin.txt').decode().splitlines(False)
	gfwlist = content.decode().splitlines(False)
	gfwlist.extend(builtin_rules)

	if user_rule:
		gfwlist.extend(user_rule.decode().splitlines(False))

	return gfwlist 

def reduce_domains(domains):
	# reduce 'www.google.com' to 'google.com'
	# remove invalid domains
	tld_content = pkgutil.get_data('gfwlist2dnsmasq', 'resources/tld.txt')
	try:
		tld_content = tld_content.decode()
	except:
		tld_content = tld_content
	tlds = set(tld_content.splitlines(False))
	new_domains = set()
	for domain in domains:
		domain_parts = domain.split('.')
		last_root_domain = None
		for i in xrange(0, len(domain_parts)):
			root_domain = '.'.join(domain_parts[len(domain_parts) - i - 1:])
			if i == 0:
				if not tlds.__contains__(root_domain):
					# root_domain is not a valid tld
					break
			last_root_domain = root_domain
			if tlds.__contains__(root_domain):
				continue
			else:
				break
		if last_root_domain is not None:
			new_domains.add(last_root_domain)
	return sorted(new_domains) 

def get_resource(resource):
    """Get the given resource from the list of known resources."""
    return pkgutil.get_data('argus.resources', resource) 

def get_spectra():
    """Reads the spectral information and stores is for future use."""

    # PROSPECT-D
    prospect_d_spectraf = pkgutil.get_data('pyPro4Sail', 'prospect_d_spectra.txt')
    wl, nr, kab, kcar, kant, kbrown, kw, km= np.loadtxt(
        BytesIO(prospect_d_spectraf), unpack=True)
    #prospect_d_spectra = ProspectDSpectra(wl, nr, kab, kcar, kbrown, kw, km, kant)

    return wl, nr, kab, kcar, kbrown, kw, km, kant 

def test_getdata_filesys(self):
        pkg = 'test_getdata_filesys'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        package_dir = os.path.join(self.dirname, pkg)
        os.mkdir(package_dir)
        # Empty init.py
        f = open(os.path.join(package_dir, '__init__.py'), "wb")
        f.close()
        # Resource files, res.txt, sub/res.txt
        f = open(os.path.join(package_dir, 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()
        os.mkdir(os.path.join(package_dir, 'sub'))
        f = open(os.path.join(package_dir, 'sub', 'res.txt'), "wb")
        f.write(RESOURCE_DATA)
        f.close()

        # Check we can read the resources
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)

        del sys.modules[pkg] 

def test_getdata_zipfile(self):
        zip = 'test_getdata_zipfile.zip'
        pkg = 'test_getdata_zipfile'

        # Include a LF and a CRLF, to test that binary data is read back
        RESOURCE_DATA = 'Hello, world!\nSecond line\r\nThird line'

        # Make a package with some resources
        zip_file = os.path.join(self.dirname, zip)
        z = zipfile.ZipFile(zip_file, 'w')

        # Empty init.py
        z.writestr(pkg + '/__init__.py', "")
        # Resource files, res.txt, sub/res.txt
        z.writestr(pkg + '/res.txt', RESOURCE_DATA)
        z.writestr(pkg + '/sub/res.txt', RESOURCE_DATA)
        z.close()

        # Check we can read the resources
        sys.path.insert(0, zip_file)
        res1 = pkgutil.get_data(pkg, 'res.txt')
        self.assertEqual(res1, RESOURCE_DATA)
        res2 = pkgutil.get_data(pkg, 'sub/res.txt')
        self.assertEqual(res2, RESOURCE_DATA)
        del sys.path[0]

        del sys.modules[pkg] 

def get_data(self, path):
            return "Hello, world!" 

def test_alreadyloaded(self):
        # Ensure that get_data works without reloading - the "loads" module
        # variable in the example loader should count how many times a reload
        # occurs.
        import foo
        self.assertEqual(foo.loads, 1)
        self.assertEqual(pkgutil.get_data('foo', 'dummy'), "Hello, world!")
        self.assertEqual(foo.loads, 1)
        del sys.modules['foo'] 

def _get_snapshot_tld_extractor():
        snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
        return json.loads(snapshot_data.decode('utf-8')) 

def _get_snapshot_tld_extractor():
        snapshot_data = pkgutil.get_data(__name__, '.tld_set_snapshot')
        return json.loads(snapshot_data.decode('utf-8')) 

def resource(resource_name):
    return pkgutil.get_data(RESOURCE_PACKAGE, resource_name).decode('utf-8') 

def resource(resource_name):
    return pkgutil.get_data(RESOURCE_PACKAGE, resource_name).decode('utf-8') 

def reload_ocp_map_0007(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ocp_items = ReportColumnMap.objects.filter(provider_type='OCP')
    ocp_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/ocp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 

def reload_aws_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    aws_items = ReportColumnMap.objects.filter(provider_type='AWS')
    aws_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/aws_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 

def reload_azure_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    azure_items = ReportColumnMap.objects.filter(provider_type='AZURE')
    azure_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/azure_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 

def reload_azure_map(apps, schema_editor):
    """Update report to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    azure_items = ReportColumnMap.objects.filter(provider_type='AZURE')
    azure_items.delete()

    data = pkgutil.get_data('reporting_common',
                            'data/azure_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 

def load_gcp_column_map(apps, schema_editor):
    """Load GCP column map to database mapping."""
    ReportColumnMap = apps.get_model('reporting_common', 'ReportColumnMap')
    ReportColumnMap.objects.filter(provider_type='GCP').delete()

    data = pkgutil.get_data('reporting_common',
                            'data/gcp_report_column_map.json')

    data = json.loads(data)

    for entry in data:
        map = ReportColumnMap(**entry)
        map.save() 

def load_openshift_metric_map(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    CostModelMetricsMap = apps.get_model('api', 'CostModelMetricsMap')
    CostModelMetricsMap.objects.all().delete()

    data = pkgutil.get_data('api',
                            'metrics/data/cost_models_metric_map.json')

    data = json.loads(data)

    for entry in data:
        map = CostModelMetricsMap(**entry)
        map.save() 

def load_openshift_metric_map(apps, schema_editor):
    """Load AWS Cost Usage report to database mapping."""
    CostModelMetricsMap = apps.get_model('api', 'CostModelMetricsMap')

    data = pkgutil.get_data('api',
                            'metrics/data/cost_models_metric_map.json')

    data = json.loads(data)

    for entry in data:
        map = CostModelMetricsMap(**entry)
        map.save() 

def populate_storage_line_item_daily_table(self, start_date, end_date, cluster_id):
        """Populate the daily storage aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            cluster_id (String) Cluster Identifier

        Returns
            (None)

        """
        # Cast string to date object
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
            end_date = end_date.date()
        table_name = OCP_REPORT_TABLE_MAP['storage_line_item_daily']

        daily_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpstoragelineitem_daily.sql'
        )
        daily_sql = daily_sql.decode('utf-8')
        daily_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'cluster_id': cluster_id,
            'schema': self.schema
        }
        daily_sql, daily_sql_params = self.jinja_sql.prepare_query(
            daily_sql, daily_sql_params
        )
        self._execute_raw_sql_query(
            table_name, daily_sql, start_date, end_date, bind_params=list(daily_sql_params)) 

def populate_pod_charge(self, cpu_temp_table, mem_temp_table):
        """Populate the memory and cpu charge on daily summary table.

        Args:
            cpu_temp_table (String) Name of cpu charge temp table
            mem_temp_table (String) Name of mem charge temp table

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        daily_charge_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpusagelineitem_daily_pod_charge.sql'
        )
        charge_line_sql = daily_charge_sql.decode('utf-8')
        charge_line_sql_params = {
            'cpu_temp': cpu_temp_table,
            'mem_temp': mem_temp_table,
            'schema': self.schema
        }
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(
            charge_line_sql, charge_line_sql_params
        )
        self._execute_raw_sql_query(
            table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 

def populate_storage_charge(self, temp_table_name):
        """Populate the storage charge into the daily summary table.

        Args:
            storage_charge (Float) Storage charge.

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        daily_charge_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocp_storage_charge.sql'
        )
        charge_line_sql = daily_charge_sql.decode('utf-8')
        charge_line_sql_params = {
            'temp_table': temp_table_name,
            'schema': self.schema
        }
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(
            charge_line_sql, charge_line_sql_params
        )
        self._execute_raw_sql_query(
            table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 

def populate_line_item_daily_summary_table(self, start_date, end_date, cluster_id):
        """Populate the daily aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            cluster_id (String) Cluster Identifier

        Returns
            (None)

        """
        # Cast start_date to date
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        if isinstance(start_date, datetime.datetime):
            start_date = start_date.date()
            end_date = end_date.date()
        table_name = OCP_REPORT_TABLE_MAP['line_item_daily_summary']

        summary_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpusagelineitem_daily_summary.sql'
        )
        summary_sql = summary_sql.decode('utf-8')
        summary_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'cluster_id': cluster_id,
            'schema': self.schema
        }
        summary_sql, summary_sql_params = self.jinja_sql.prepare_query(
            summary_sql, summary_sql_params
        )
        self._execute_raw_sql_query(
            table_name, summary_sql, start_date, end_date, bind_params=list(summary_sql_params)) 

def bootstrap(root=None, upgrade=False, user=False,
              altinstall=False, default_pip=True,
              verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory).

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _require_ssl_for_pip()
    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    tmpdir = tempfile.mkdtemp()
    try:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True) 

def _bootstrap(*, root=None, upgrade=False, user=False,
              altinstall=False, default_pip=False,
              verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory). Returns pip command status code.

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    with tempfile.TemporaryDirectory() as tmpdir:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths) 

def bootstrap(tmpdir=None):
    # Import pip so we can use it to install pip and maybe setuptools too
    import pip

    # We always want to install pip
    packages = ["pip"]

    # Check if the user has requested us not to install setuptools
    if "--no-setuptools" in sys.argv or os.environ.get("PIP_NO_SETUPTOOLS"):
        args = [x for x in sys.argv[1:] if x != "--no-setuptools"]
    else:
        args = sys.argv[1:]

        # We want to see if setuptools is available before attempting to
        # install it
        try:
            import setuptools
        except ImportError:
            packages += ["setuptools"]

    delete_tmpdir = False
    try:
        # Create a temporary directory to act as a working directory if we were
        # not given one.
        if tmpdir is None:
            tmpdir = tempfile.mkdtemp()
            delete_tmpdir = True

        # We need to extract the SSL certificates from requests so that they
        # can be passed to --cert
        cert_path = os.path.join(tmpdir, "cacert.pem")
        with open(cert_path, "wb") as cert:
            cert.write(pkgutil.get_data("pip._vendor.requests", "cacert.pem"))

        # Use an environment variable here so that users can still pass
        # --cert via sys.argv
        os.environ.setdefault("PIP_CERT", cert_path)

        # Execute the included pip and use it to install the latest pip and
        # setuptools from PyPI
        sys.exit(pip.main(["install", "--upgrade"] + packages + args))
    finally:
        # Remove our temporary directory
        if delete_tmpdir and tmpdir:
            shutil.rmtree(tmpdir, ignore_errors=True) 

def _bootstrap(root=None, upgrade=False, user=False,
               altinstall=False, default_pip=True,
               verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory). Returns pip command status code.

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    tmpdir = tempfile.mkdtemp()
    try:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if sys.platform == 'cli': # IronPython doesn't do bytecode compilation
            args += ["--no-compile"]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        return _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True) 

def get_ocp_infrastructure_map(self, start_date, end_date, **kwargs):
        """Get the OCP on infrastructure map.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.

        Returns
            (None)

        """
        # kwargs here allows us to optionally pass in a provider UUID based on
        # the provider type this is run for
        ocp_provider_uuid = kwargs.get('ocp_provider_uuid')
        aws_provider_uuid = kwargs.get('aws_provider_uuid')
        azure_provider_uuid = kwargs.get('azure_provider_uuid')
        # In case someone passes this function a string instead of the date object like we asked...
        # Cast the string into a date object, end_date into date object instead of string
        if isinstance(start_date, str):
            start_date = datetime.datetime.strptime(start_date, '%Y-%m-%d').date()
            end_date = datetime.datetime.strptime(end_date, '%Y-%m-%d').date()
        infra_sql = pkgutil.get_data(
            'masu.database',
            'sql/reporting_ocpinfrastructure_provider_map.sql'
        )
        infra_sql = infra_sql.decode('utf-8')
        infra_sql_params = {
            'uuid': str(uuid.uuid4()).replace('-', '_'),
            'start_date': start_date,
            'end_date': end_date,
            'schema': self.schema,
            'aws_provider_uuid': aws_provider_uuid,
            'ocp_provider_uuid': ocp_provider_uuid,
            'azure_provider_uuid': azure_provider_uuid
        }
        infra_sql, infra_sql_params = self.jinja_sql.prepare_query(
            infra_sql, infra_sql_params)
        with connection.cursor() as cursor:
            cursor.db.set_schema(self.schema)
            cursor.execute(infra_sql, list(infra_sql_params))
            results = cursor.fetchall()

        db_results = {}
        for entry in results:
            # This dictionary is keyed on an OpenShift provider UUID
            # and the tuple contains
            # (Infrastructure Provider UUID, Infrastructure Provider Type)
            db_results[entry[0]] = (entry[1], entry[2])

        return db_results