Python pkgutil.get_data() Examples

The following are 30 code examples of pkgutil.get_data(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module pkgutil , or try the search function .
Example #1
Source Project: misp42splunk   Author: remg427   File: driver.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 
Example #2
Source Project: misp42splunk   Author: remg427   File: driver.py    License: GNU Lesser General Public License v3.0 6 votes vote down vote up
def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 
Example #3
Source Project: linter-pylama   Author: AtomLinter   File: wordlists.py    License: MIT License 6 votes vote down vote up
def load_wordlist(name):
    """Iterate over lines of a wordlist data file.

    `name` should be the name of a package data file within the data/
    directory.

    Whitespace and #-prefixed comments are stripped from each line.

    """
    text = pkgutil.get_data('pydocstyle', 'data/' + name).decode('utf8')
    for line in text.splitlines():
        line = COMMENT_RE.sub('', line).strip()
        if line:
            yield line


#: A dict mapping stemmed verbs to the imperative form 
Example #4
Source Project: bilibiliupload   Author: ForgQi   File: util.py    License: MIT License 6 votes vote down vote up
def init_jsengine():
    global js_ctx
    if js_ctx is None:
        from ykdl.util.jsengine import JSEngine
        assert JSEngine, "No JS Interpreter found, can't use cmd5x!"
        js_ctx = JSEngine()

        from pkgutil import get_data
        # code from https://zsaim.github.io/2019/08/23/Iqiyi-cmd5x-Analysis/
        try:
            # try load local .js file first
            js = get_data(__name__, 'cmd5x.js')
        except IOError:
            # origin https://raw.githubusercontent.com/ZSAIm/ZSAIm.github.io/master/misc/2019-08-23/iqiyi_cmd5x.js
            js = get_content('https://raw.githubusercontent.com/zhangn1985/ykdl/master/ykdl/extractors/iqiyi/cmd5x.js')
        js_ctx.append(js)

        # code from https://github.com/lldy/js
        try:
            # try load local .js file first
            js = get_data(__name__, 'cmd5x_iqiyi3.js')
        except IOError:
            js = get_content('https://raw.githubusercontent.com/zhangn1985/ykdl/master/ykdl/extractors/iqiyi/cmd5x_iqiyi3.js')
        js_ctx.append(js) 
Example #5
Source Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    License: Apache License 2.0 6 votes vote down vote up
def get_data():
		'''
		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                    plot
		filename                 subjectivity_html/obj/2002/Abandon.html
		text           A senior at an elite college (Katie Holmes), a...
		movie_name                                               abandon
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 
Example #6
Source Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    License: Apache License 2.0 6 votes vote down vote up
def get_full_data():
		'''
		Returns all plots and reviews, not just the ones that appear in movies with both plot descriptions and reviews.

		Returns
		-------
		pd.DataFrame

		I.e.,
		>>> convention_df.iloc[0]
		category                                                             plot
		text                    Vijay Singh Rajput (Amitabh Bachchan) is a qui...
		movie_name                                                        aankhen
		has_plot_and_reviews                                                False
		Name: 0, dtype: object
		'''
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/rotten_tomatoes_corpus_full.csv.bz2')
		except:
			url = ROTTEN_TOMATOES_DATA_URL
			data_stream = urlopen(url).read()
		return pd.read_csv(io.BytesIO(bz2.decompress(data_stream))) 
Example #7
Source Project: sacremoses   Author: alvations   File: corpus.py    License: MIT License 6 votes vote down vote up
def chars(self, category=None):
        """
        This module returns a list of characters from  the Perl Unicode Properties.
        They are very useful when porting Perl tokenizers to Python.

            >>> from sacremoses.corpus import Perluniprops
            >>> pup = Perluniprops()
            >>> list(pup.chars('Open_Punctuation'))[:5] == [u'(', u'[', u'{', u'\u0f3a', u'\u0f3c']
            True
            >>> list(pup.chars('Currency_Symbol'))[:5] == [u'$', u'\xa2', u'\xa3', u'\xa4', u'\xa5']
            True
            >>> pup.available_categories[:5]
            ['Close_Punctuation', 'Currency_Symbol', 'IsAlnum', 'IsAlpha', 'IsLower']

        :return: a generator of characters given the specific unicode character category
        """
        relative_path = os.path.join("data", "perluniprops", category + ".txt")
        binary_data = pkgutil.get_data("sacremoses", relative_path)
        for ch in binary_data.decode("utf-8"):
            yield ch 
Example #8
Source Project: ironpython2   Author: IronLanguages   File: driver.py    License: Apache License 2.0 6 votes vote down vote up
def load_packaged_grammar(package, grammar_source):
    """Normally, loads a pickled grammar by doing
        pkgutil.get_data(package, pickled_grammar)
    where *pickled_grammar* is computed from *grammar_source* by adding the
    Python version and using a ``.pickle`` extension.

    However, if *grammar_source* is an extant file, load_grammar(grammar_source)
    is called instead. This facilitates using a packaged grammar file when needed
    but preserves load_grammar's automatic regeneration behavior when possible.

    """
    if os.path.isfile(grammar_source):
        return load_grammar(grammar_source)
    pickled_name = _generate_pickle_name(os.path.basename(grammar_source))
    data = pkgutil.get_data(package, pickled_name)
    g = grammar.Grammar()
    g.loads(data)
    return g 
Example #9
Source Project: koku   Author: project-koku   File: ocp_report_db_accessor.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def populate_pod_charge(self, cpu_temp_table, mem_temp_table):
        """Populate the memory and cpu charge on daily summary table.

        Args:
            cpu_temp_table (String) Name of cpu charge temp table
            mem_temp_table (String) Name of mem charge temp table

        Returns
            (None)

        """
        table_name = OCP_REPORT_TABLE_MAP["line_item_daily_summary"]

        daily_charge_sql = pkgutil.get_data("masu.database", "sql/reporting_ocpusagelineitem_daily_pod_charge.sql")
        charge_line_sql = daily_charge_sql.decode("utf-8")
        charge_line_sql_params = {"cpu_temp": cpu_temp_table, "mem_temp": mem_temp_table, "schema": self.schema}
        charge_line_sql, charge_line_sql_params = self.jinja_sql.prepare_query(charge_line_sql, charge_line_sql_params)
        self._execute_raw_sql_query(table_name, charge_line_sql, bind_params=list(charge_line_sql_params)) 
Example #10
Source Project: koku   Author: project-koku   File: aws_report_db_accessor.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def populate_line_item_daily_table(self, start_date, end_date, bill_ids):
        """Populate the daily aggregate of line items table.

        Args:
            start_date (datetime.date) The date to start populating the table.
            end_date (datetime.date) The date to end on.
            bill_ids (list)

        Returns
            (None)

        """
        table_name = AWS_CUR_TABLE_MAP["line_item_daily"]

        daily_sql = pkgutil.get_data("masu.database", "sql/reporting_awscostentrylineitem_daily.sql")
        daily_sql = daily_sql.decode("utf-8")
        daily_sql_params = {
            "uuid": str(uuid.uuid4()).replace("-", "_"),
            "start_date": start_date,
            "end_date": end_date,
            "bill_ids": bill_ids,
            "schema": self.schema,
        }
        daily_sql, daily_sql_params = self.jinja_sql.prepare_query(daily_sql, daily_sql_params)
        self._execute_raw_sql_query(table_name, daily_sql, start_date, end_date, bind_params=list(daily_sql_params)) 
Example #11
Source Project: hiku   Author: vmagamedov   File: ui.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self, root, engine, ctx=None, debug=False):
        self.root = root
        self.engine = engine
        self.ctx = ctx
        self.debug = debug
        self._console_html = string.Template(_decode(
            pkgutil.get_data('hiku.console', 'assets/console.html')
        ))
        self._docs_content = dumps_typedef(root) 
Example #12
Source Project: hiku   Author: vmagamedov   File: ui.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _static_get(self, environ, start_response):
        content = pkgutil.get_data('hiku.console', 'assets/console.js')
        start_response('200 OK', [
            ('Content-Type', 'text/javascript; charset=UTF-8'),
            ('Content-Length', str(len(content))),
        ])
        return [content] 
Example #13
Source Project: misp42splunk   Author: remg427   File: _utils.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 
Example #14
Source Project: misp42splunk   Author: remg427   File: _utils.py    License: GNU Lesser General Public License v3.0 5 votes vote down vote up
def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data('jsonschema', "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 
Example #15
Source Project: Chai   Author: lanluoxiao   File: tools.py    License: GNU General Public License v3.0 5 votes vote down vote up
def loadFromPackage(path, withNumbers=True):
    """
    功能:从模块包中加载 YAML 数据库
    输入:路径 path
    输出:yaml 解析器加载后的数据
    """
    return yaml.load(pkgutil.get_data(__package__, path).decode(), Loader=yaml.SafeLoader if withNumbers else yaml.BaseLoader) 
Example #16
Source Project: Chai   Author: lanluoxiao   File: tools.py    License: GNU General Public License v3.0 5 votes vote down vote up
def loadFromPackage(path, withNumbers=True):
    """
    功能:从模块包中加载 YAML 数据库
    输入:路径 path
    输出:yaml 解析器加载后的数据
    """
    return yaml.load(pkgutil.get_data(__package__, path).decode(), Loader=yaml.SafeLoader if withNumbers else yaml.BaseLoader) 
Example #17
Source Project: core   Author: getavalon   File: _utils.py    License: MIT License 5 votes vote down vote up
def load_schema(name):
    """
    Load a schema from ./schemas/``name``.json and return it.

    """

    data = pkgutil.get_data(__package__, "schemas/{0}.json".format(name))
    return json.loads(data.decode("utf-8")) 
Example #18
Source Project: tempo-cnn   Author: hendriks73   File: classifier.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def _extract_from_package(resource):
    data = pkgutil.get_data('tempocnn', resource)
    with tempfile.NamedTemporaryFile(prefix='model', suffix='.h5', delete=False) as f:
        f.write(data)
        name = f.name
    return name 
Example #19
Source Project: plugin.video.emby   Author: MediaBrowser   File: __init__.py    License: GNU General Public License v3.0 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example #20
Source Project: recruit   Author: Frank-qlu   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def get_resource(self, request, filename):
        """Return a static resource from the shared folder."""
        filename = join("shared", basename(filename))
        try:
            data = pkgutil.get_data(__package__, filename)
        except OSError:
            data = None
        if data is not None:
            mimetype = mimetypes.guess_type(filename)[0] or "application/octet-stream"
            return Response(data, mimetype=mimetype)
        return Response("Not Found", status=404) 
Example #21
Source Project: recruit   Author: Frank-qlu   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def getzoneinfofile_stream():
    try:
        return BytesIO(get_data(__name__, ZONEFILENAME))
    except IOError as e:  # TODO  switch to FileNotFoundError?
        warnings.warn("I/O error({0}): {1}".format(e.errno, e.strerror))
        return None 
Example #22
Source Project: adversarial-policies   Author: HumanCompatibleAI   File: gym_compete.py    License: MIT License 5 votes vote down vote up
def load_zoo_agent_params(tag, env_name, index):
    """Loads parameters for the gym_compete zoo agent, but does not restore them.
    :param tag: (str) version of the zoo agent (e.g. '1', '2', '3').
    :param env_name: (str) Gym environment ID
    :param index: (int) the player ID of the agent we want to load ('0' or '1')
    :return a NumPy array of policy weights."""
    # Load parameters
    canonical_env = env_name_to_canonical(env_name)
    agent_dir = os.path.join("agent_zoo", canonical_env)

    if is_symmetric(env_name):  # asymmetric version, parameters tagged with agent id
        symmetric_fname = f"agent_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, symmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)
    else:  # symmetric version, parameters not associated with a specific agent
        asymmetric_fname = f"agent{index + 1}_parameters-v{tag}.pkl"
        path = os.path.join(agent_dir, asymmetric_fname)
        params_pkl = pkgutil.get_data("gym_compete", path)

    pylog.info(f"Loaded zoo parameters from '{path}'")

    return pickle.loads(params_pkl) 
Example #23
Source Project: adversarial-policies   Author: HumanCompatibleAI   File: train.py    License: MIT License 5 votes vote down vote up
def load_default(env_name, config_dir):
    default_config = DEFAULT_CONFIGS.get(env_name, "default.json")
    fname = os.path.join("configs", config_dir, default_config)
    config = pkgutil.get_data("aprl", fname)
    return json.loads(config) 
Example #24
Source Project: burnman   Author: geodynamics   File: tools.py    License: GNU General Public License v2.0 5 votes vote down vote up
def read_table(filename):
    datastream = pkgutil.get_data('burnman', 'data/' + filename)
    datalines = [line.strip()
                 for line in datastream.decode('ascii').split('\n') if line.strip()]
    table = []

    for line in datalines:
        if (line[0] != '#'):
            numbers = np.fromstring(line, sep=' ')
            table.append(numbers)
    return np.array(table) 
Example #25
Source Project: burnman   Author: geodynamics   File: processchemistry.py    License: GNU General Public License v2.0 5 votes vote down vote up
def read_masses():
    """
    A simple function to read a file with a two column list of
    elements and their masses into a dictionary
    """
    datastream = pkgutil.get_data(
        'burnman', 'data/input_masses/atomic_masses.dat')
    datalines = [line.strip()
                 for line in datastream.decode('ascii').split('\n') if line.strip()]
    lookup = dict()
    for line in datalines:
        data = "%".join(line.split("%")[:1]).split()
        if data != []:
            lookup[data[0]] = float(data[1])
    return lookup 
Example #26
Source Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    License: Apache License 2.0 5 votes vote down vote up
def _convention_speech_iter():
		try:
			data_stream = pkgutil.get_data('scattertext', 'data/political_data.json').decode('utf-8')
		except:
			url = POLITICAL_DATA_URL
			data_stream = urlopen(url).read().decode('utf-8')
		return json.loads(data_stream) 
Example #27
Source Project: scattertext   Author: JasonKessler   File: SampleCorpora.py    License: Apache License 2.0 5 votes vote down vote up
def get_data():
		clean = ConventionData2012._clean_function_factory()
		get_speaker_name = ConventionData2012._speaker_name_factory()
		data = []
		for party, speech in ConventionData2012._iter_party_speech_pairs():
			cleaned_speech = clean(speech)
			speaker_name = get_speaker_name(speech)
			if cleaned_speech and cleaned_speech != '' and speaker_name != '':
				data.append({'party': party,
				             'text': cleaned_speech,
				             'speaker': speaker_name})
		return pd.DataFrame(data) 
Example #28
Source Project: scattertext   Author: JasonKessler   File: FeatsFromMoralFoundationsDictionary.py    License: Apache License 2.0 5 votes vote down vote up
def _load_mfd(self):
        return pd.read_csv(
            io.StringIO(pkgutil.get_data('scattertext', 'data/mfd2.0.csv').decode('utf-8'))
        ).set_index('term') 
Example #29
Source Project: scattertext   Author: JasonKessler   File: BasicHTMLFromScatterplotStructure.py    License: Apache License 2.0 5 votes vote down vote up
def get_packaged_script_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/scripts/' + file_name).decode('utf-8') 
Example #30
Source Project: scattertext   Author: JasonKessler   File: BasicHTMLFromScatterplotStructure.py    License: Apache License 2.0 5 votes vote down vote up
def get_packaged_html_template_content(file_name):
        return pkgutil.get_data('scattertext',
                                'data/viz/' + file_name).decode('utf-8')