Python pkgutil.get_data() Examples

def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 

def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 

def load_wordlist(name):
    """Iterate over lines of a wordlist data file.

    `name` should be the name of a package data file within the data/
    directory.

    Whitespace and #-prefixed comments are stripped from each line.

    """
    text = pkgutil.get_data('pydocstyle', 'data/' + name).decode('utf8')
    for line in text.splitlines():
        line = COMMENT_RE.sub('', line).strip()
        if line:
            yield line


#: A dict mapping stemmed verbs to the imperative form 

def init_jsengine():
    global js_ctx
    if js_ctx is None:
        from ykdl.util.jsengine import JSEngine
        assert JSEngine, "No JS Interpreter found, can't use cmd5x!"
        js_ctx = JSEngine()

        from pkgutil import get_data
        # code from https://zsaim.github.io/2019/08/23/Iqiyi-cmd5x-Analysis/
        try:
            # try load local .js file first
            js = get_data(__name__, 'cmd5x.js')
        except IOError:
            # origin https://raw.githubusercontent.com/ZSAIm/ZSAIm.github.io/master/misc/2019-08-23/iqiyi_cmd5x.js
            js = get_content('https://raw.githubusercontent.com/zhangn1985/ykdl/master/ykdl/extractors/iqiyi/cmd5x.js')
        js_ctx.append(js)

        # code from https://github.com/lldy/js
        try:
            # try load local .js file first
            js = get_data(__name__, 'cmd5x_iqiyi3.js')
        except IOError:
            js = get_content('https://raw.githubusercontent.com/zhangn1985/ykdl/master/ykdl/extractors/iqiyi/cmd5x_iqiyi3.js')
        js_ctx.append(js) 

def get_data():
		'''
		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                    plot
		filename                 subjectivity_html/obj/2002/Abandon.html
		text           A senior at an elite college (Katie Holmes), a...
		movie_name                                               abandon
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

def get_full_data():
		'''
		Returns all plots and reviews, not just the ones that appear in movies with both plot descriptions and reviews.

		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                             plot
		text                    Vijay Singh Rajput (Amitabh Bachchan) is a qui...
		movie_name                                                        aankhen
		has_plot_and_reviews                                                False
		Name: 0, dtype: object
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus_full.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 

def chars(self, category=None):
        """
        This module returns a list of characters from  the Perl Unicode Properties.
        They are very useful when porting Perl tokenizers to Python.

            >>> from sacremoses.corpus import Perluniprops
            >>> pup = Perluniprops()
            >>> list(pup.chars('Open_Punctuation'))[:5] == [u'(', u'[', u'{', u'\u0f3a', u'\u0f3c']
            True
            >>> list(pup.chars('Currency_Symbol'))[:5] == [u'$', u'\xa2', u'\xa3', u'\xa4', u'\xa5']
            True
            >>> pup.available_categories[:5]
            ['Close_Punctuation', 'Currency_Symbol', 'IsAlnum', 'IsAlpha', 'IsLower']

        :return: a generator of characters given the specific unicode character category
        """
        relative_path = os.path.join("data", "perluniprops", category + ".txt")
        binary_data = pkgutil.get_data("sacremoses", relative_path)
        for ch in binary_data.decode("utf-8"):
            yield ch 

def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 

def populate_pod_charge(self, cpu_temp_table, mem_temp_table):
        """Populate the memory and cpu charge on daily summary table.

        Args:
            cpu_temp_table (String) Name of cpu charge temp table
            mem_temp_table (String) Name of mem charge temp table

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]

        daily_charge_sql = pkgutil.get_data("masu.database", "sql/reporting_ocpusagelineitem_daily_pod_charge.sql")
        charge_line_sql = daily_charge_sql.decode("utf-8")
        charge_line_sql_params = {"cpu_temp": cpu_temp_table, "mem_temp": mem_temp_table, "schema": self.schema}
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(charge_line_sql, charge_line_sql_params)
        self._execute_raw_sql_query(table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 

def populate_line_item_daily_table(self, start_date, end_date, bill_ids):
        """Populate the daily aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            bill_ids (list)

        Returns
            (None)

        """
        table_name = AWS_CUR_TABLE_MAP["line_item_daily"]

        daily_sql = pkgutil.get_data("masu.database", "sql/reporting_awscostentrylineitem_daily.sql")
        daily_sql = daily_sql.decode("utf-8")
        daily_sql_params = {
            "uuid": str(uuid.uuid4()).replace("-", "_"),
            "start_date": start_date,
            "end_date": end_date,
            "bill_ids": bill_ids,
            "schema": self.schema,
        }
        daily_sql, daily_sql_params = self.jinja_sql.prepare_query(daily_sql, daily_sql_params)
        self._execute_raw_sql_query(table_name, daily_sql, start_date, end_date, bind_params=list(daily_sql_params)) 

def __init__(self, root, engine, ctx=None, debug=False):
        self.root = root
        self.engine = engine
        self.ctx = ctx
        self.debug = debug
        self._console_html = string.Template(_decode(
            pkgutil.get_data('hiku.console', 'assets/console.html')
        ))
        self._docs_content = dumps_typedef(root) 

def _static_get(self, environ, start_response):
        content = pkgutil.get_data('hiku.console', 'assets/console.js')
        start_response('200 OK', [
            ('Content-Type', 'text/javascript; charset=UTF-8'),
            ('Content-Length', str(len(content))),
        ])
        return [content] 

def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 

def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 

def loadFromPackage(path, withNumbers=True):
    """
    功能：从模块包中加载 YAML 数据库
    输入：路径 path
    输出：yaml 解析器加载后的数据
    """
    return yaml.load(pkgutil.get_data(__package__, path).decode(), Loader=yaml.SafeLoader if withNumbers else yaml.BaseLoader) 

def loadFromPackage(path, withNumbers=True):
    """
    功能：从模块包中加载 YAML 数据库
    输入：路径 path
    输出：yaml 解析器加载后的数据
    """
    return yaml.load(pkgutil.get_data(__package__, path).decode(), Loader=yaml.SafeLoader if withNumbers else yaml.BaseLoader) 

def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 

def _extract_from_package(resource):
    data = pkgutil.get_data('tempocnn', resource)
    with tempfile.NamedTemporaryFile(prefix='model', suffix='.h5', delete=False) as f:
        f.write(data)
        name = f.name
    return name 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def get_resource(self, request, filename):
        """Return a static resource from the shared folder."""
        filename = join("shared", basename(filename))
        try:
            data = pkgutil.get_data(__package__, filename)
        except OSError:
            data = None
        if data is not None:
            mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
            return Response(data, mimetype=mimetype)
        return Response("Not Found", status=404) 

def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 

def load_zoo_agent_params(tag, env_name, index):
    """Loads parameters for the gym_compete zoo agent, but does not restore them.
    :param tag: (str) version of the zoo agent (e.g. '1', '2', '3').
    :param env_name: (str) Gym environment ID
    :param index: (int) the player ID of the agent we want to load ('0' or '1')
    :return a NumPy array of policy weights."""
    # Load parameters
    canonical_env = env_name_to_canonical(env_name)
    agent_dir = os.path.join("agent_zoo", canonical_env)

    if is_symmetric(env_name):  # asymmetric version, parameters tagged with agent id
        symmetric_fname = f"agent_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, symmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)
    else:  # symmetric version, parameters not associated with a specific agent
        asymmetric_fname = f"agent{index + 1}_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, asymmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)

    pylog.info(f"Loaded zoo parameters from '{path}'")

    return pickle.loads(params_pkl) 

def load_default(env_name, config_dir):
    default_config = DEFAULT_CONFIGS.get(env_name, "default.json")
    fname = os.path.join("configs", config_dir, default_config)
    config = pkgutil.get_data("aprl", fname)
    return json.loads(config) 

def read_table(filename):
    datastream = pkgutil.get_data('burnman', 'data/' + filename)
    datalines = [line.strip()
                 for line in datastream.decode('ascii').split('\n') if line.strip()]
    table = []

    for line in datalines:
        if (line[0] != '#'):
            numbers = np.fromstring(line, sep=' ')
            table.append(numbers)
    return np.array(table) 

def read_masses():
    """
    A simple function to read a file with a two column list of
    elements and their masses into a dictionary
    """
    datastream = pkgutil.get_data(
        'burnman', 'data/input_masses/atomic_masses.dat')
    datalines = [line.strip()
                 for line in datastream.decode('ascii').split('\n') if line.strip()]
    lookup = dict()
    for line in datalines:
        data = "%".join(line.split("%")[:1]).split()
        if data != []:
            lookup[data[0]] = float(data[1])
    return lookup 

def _convention_speech_iter():
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/political_data.json').decode('utf-8')
		except:
			url = POLITICAL_DATA_URL
			data_stream = urlopen(url).read().decode('utf-8')
		return json.loads(data_stream) 

def get_data():
		clean = ConventionData2012._clean_function_factory()
		get_speaker_name = ConventionData2012._speaker_name_factory()
		data = []
		for party, speech in ConventionData2012._iter_party_speech_pairs():
			cleaned_speech = clean(speech)
			speaker_name = get_speaker_name(speech)
			if cleaned_speech and cleaned_speech != '' and speaker_name != '':
				data.append({'party': party,
				             'text': cleaned_speech,
				             'speaker': speaker_name})
		return pd.DataFrame(data) 

def _load_mfd(self):
        return pd.read_csv(
            io.StringIO(pkgutil.get_data('scattertext', 'data/mfd2.0.csv').decode('utf-8'))
        ).set_index('term') 

def get_packaged_script_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/scripts/' + file_name).decode('utf-8') 

def get_packaged_html_template_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/' + file_name).decode('utf-8')