Python yaml.org() Examples

def yaml_load(source, loader=yaml.Loader):
    """
    Wrap PyYaml's loader so we can extend it to suit our needs.

    Load all strings as Unicode: http://stackoverflow.com/a/2967461/3609487.
    """

    def construct_yaml_str(self, node):
        """Override the default string handling function to always return Unicode objects."""
        return self.construct_scalar(node)

    class Loader(loader):
        """Define a custom loader to leave the global loader unaltered."""

    # Attach our Unicode constructor to our custom loader ensuring all strings
    # will be Unicode on translation.
    Loader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str)

    return yaml.load(source, Loader) 

def get_dict_from_yaml(config_file: Union[str, io.StringIO]) -> dict:
    """
    Read a config file or file like object of YAML into a dict
    """
    # We must override the default constructor for timestamp to ensure the result
    # has tzinfo. Yaml loader never adds tzinfo, but converts to UTC.
    yaml.FullLoader.add_constructor(
        tag="tag:yaml.org,2002:timestamp", constructor=_timestamp_constructor
    )
    if hasattr(config_file, "read"):
        yaml_content = yaml.load(config_file, Loader=yaml.FullLoader)
    else:
        try:
            path_to_config_file = os.path.abspath(config_file)  # type: ignore
            with open(path_to_config_file, "r") as yamlfile:  # type: ignore
                yaml_content = yaml.load(yamlfile, Loader=yaml.FullLoader)
        except FileNotFoundError:
            raise FileNotFoundError(
                f"Unable to find config file <{path_to_config_file}>"
            )
    # Handle multiple versions of workflow config structure
    if "spec" in yaml_content:
        yaml_content = yaml_content["spec"]["config"]

    return yaml_content 

def load_config(file):
	"""
	Loads a config from the file

	:param file: the file from which to load the config
	:return: the loaded config represented as a dictionary, might be empty if config file was not found or empty
	"""
	import yaml
	import os

	def datetime_constructor(loader, node):
		return dateutil.parser.parse(node.value)
	yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:timestamp', datetime_constructor)

	config = None
	if file is not None and os.path.exists(file) and os.path.isfile(file):
		with open(file, "r") as f:
			config = yaml.safe_load(f)

	if config is None:
		config = {}

	return config 

def make_loader(*tags_to_remove: str) -> Type[yaml.SafeLoader]:
    """Create a YAML loader, that doesn't parse specific tokens into Python objects."""
    cls: Type[yaml.SafeLoader] = type("YAMLLoader", (SafeLoader,), {})
    cls.yaml_implicit_resolvers = {
        key: [(tag, regexp) for tag, regexp in mapping if tag not in tags_to_remove]
        for key, mapping in cls.yaml_implicit_resolvers.copy().items()
    }

    # Fix pyyaml scientific notation parse bug
    # See PR: https://github.com/yaml/pyyaml/pull/174 for upstream fix
    cls.add_implicit_resolver(  # type: ignore
        "tag:yaml.org,2002:float",
        re.compile(
            r"""^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+]?[0-9]+)?
                       |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
                       |\.[0-9_]+(?:[eE][-+]?[0-9]+)?
                       |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]*
                       |[-+]?\.(?:inf|Inf|INF)
                       |\.(?:nan|NaN|NAN))$""",
            re.X,
        ),
        list("-+0123456789."),
    )

    return cls 

def extract_substructure_for_test(test_case, substructure, config):
    """
    Extract the keys from the config in substructure, which may be a nested
    dictionary.

    Raises a ``unittest.SkipTest`` if the substructure is not found in the
    configuration.

    This can be used to load credentials all at once for testing purposes.
    """
    try:
        return extract_substructure(config, substructure)
    except MissingConfigError as e:
        yaml.add_representer(
            Optional,
            lambda d, x: d.represent_scalar(u'tag:yaml.org,2002:str', repr(x)))
        test_case.skip(
            'Skipping test: could not get configuration: {}\n\n'
            'In order to run this test, add ensure file at $ACCEPTANCE_YAML '
            'has structure like:\n\n{}'.format(
                e.message,
                yaml.dump(substructure, default_flow_style=False))
        ) 

def _get_entrypoint(data):
    # N.B. We can't use data.get() here, because that might return
    # None, leading to ambiguity between entrypoint being absent or set
    # to a null value.
    #
    # "Note that a null is different from an empty string and that a
    # mapping entry with some key and a null value is valid and
    # different from not having that key in the mapping."
    #   - http://yaml.org/type/null.html
    key = 'entrypoint'

    if not key in data:
        return None

    ep = data[key]

    # We represent a null value as an empty string.
    if ep is None:
        ep = ''

    if not isinstance(ep, str):
        raise ConfigError("'{}' must be a string, not {}".format(
                key, type(ep).__name__))
    return ep 

def test_compactdb(setup):
    bdb = BukuDb()

    # adding bookmarks
    for bookmark in TEST_BOOKMARKS:
        bdb.add_rec(*bookmark)

    # manually deleting 2nd index from db, calling compactdb
    bdb.cur.execute('DELETE FROM bookmarks WHERE id = ?', (2,))
    bdb.compactdb(2)

    # asserting bookmarks have correct indices
    assert bdb.get_rec_by_id(1) == (
        1, 'http://slashdot.org', 'SLASHDOT', ',news,old,', "News for old nerds, stuff that doesn't matter", 0)
    assert bdb.get_rec_by_id(2) == (
        2, 'http://example.com/', 'test', ',es,est,tes,test,', 'a case for replace_tag test', 0)
    assert bdb.get_rec_by_id(3) is None 

def unicode_representer(_, data):
    has_wide_lines = False
    for line in data.splitlines():
        if len(line) > 80:
            has_wide_lines = True
            break

    if has_wide_lines:
        return yaml.ScalarNode(
            u'tag:yaml.org,2002:str', data, style='>')

    if "\n" in data:
        return yaml.ScalarNode(
            u'tag:yaml.org,2002:str', data, style='|')

    return yaml.ScalarNode(
        u'tag:yaml.org,2002:str', data, style='') 

def get_category_info_string():
    header = '# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n' \
        + '#\n' \
        + '# Licensed under the Apache License, Version 2.0 (the "License");\n' \
        + '# you may not use this file except in compliance with the License.\n' \
        + '# You may obtain a copy of the License at\n' \
        + '#\n' \
        + '#     http://www.apache.org/licenses/LICENSE-2.0\n' \
        + '#\n' \
        + '# Unless required by applicable law or agreed to in writing, software\n' \
        + '# distributed under the License is distributed on an "AS IS" BASIS,\n' \
        + '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' \
        + '# See the License for the specific language governing permissions and\n' \
        + '# limitations under the License.\n' \
        + '#\n' \
        + '# DO NOT EDIT THIS FILE BY HAND\n' \
        + '# THIS FILE IS GENERATED FROM NNABLA.\n' \
        + '#\n' \
        + '\n'

    return header + yaml.dump(_nnabla_func_info, default_flow_style=False) 

def parse_args(parser):
    args = parser.parse_args()
    if args.config_file:
        loader = yaml.SafeLoader
        loader.add_implicit_resolver(
            u'tag:yaml.org,2002:float',
            re.compile(u'''^(?:
        [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
        |[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
        |\\.[0-9_]+(?:[eE][-+][0-9]+)?
        |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
        |[-+]?\\.(?:inf|Inf|INF)
        |\\.(?:nan|NaN|NAN))$''', re.X),
            list(u'-+0123456789.'))
        data = yaml.load(args.config_file, Loader=loader)
        delattr(args, 'config_file')
        arg_dict = args.__dict__
        # print(len(list(arg_dict.keys())))
        # print(len(list(data.keys())))
        for key, value in arg_dict.items():
            default_arg = parser.get_default(key)
            if arg_dict[key] == default_arg and key in data:
                arg_dict[key] = data[key]
    return args 

def represent_unicode(_, data):
    """
    Represent unicode as regular string

    Source:
        https://stackoverflow.com/questions/1950306/pyyaml-dumping-without-tags

    """

    return yaml.ScalarNode(
        tag="tag:yaml.org,2002:str", value=data.encode("utf-8")
    ) 

def save_params_dict_to_yaml(params, file_path):
  """Saves the input ParamsDict to a YAML file."""
  with tf.gfile.Open(file_path, 'w') as f:
    def _my_list_rep(dumper, data):
      # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence.
      return dumper.represent_sequence(
          u'tag:yaml.org,2002:seq', data, flow_style=True)
    yaml.add_representer(list, _my_list_rep)
    yaml.dump(params.as_dict(), f, default_flow_style=False) 

def represent_ordereddict(dumper, data):
    value = []
    for item_key, item_value in data.items():
        node_key = dumper.represent_data(item_key)
        node_value = dumper.represent_data(item_value)
        value.append((node_key, node_value))
    return yaml.nodes.MappingNode(u'tag:yaml.org,2002:map', value) 

def __init__(self, *args, **kwargs):
        yaml.Loader.__init__(self, *args, **kwargs)
        self.add_constructor(u'tag:yaml.org,2002:omap', type(self).construct_odict) 

def func_set_export_yaml(func_dict, yaml_file):
    for cat, cat_info in _nnabla_func_info.items():
        for func, func_info in cat_info.items():
            if func in func_dict.keys():
                func_info['func_type'] = func_dict[func]
            else:
                func_info['func_type'] = ['None']

    header = '# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n' \
        + '#\n' \
        + '# Licensed under the Apache License, Version 2.0 (the "License");\n' \
        + '# you may not use this file except in compliance with the License.\n' \
        + '# You may obtain a copy of the License at\n' \
        + '#\n' \
        + '#     http://www.apache.org/licenses/LICENSE-2.0\n' \
        + '#\n' \
        + '# Unless required by applicable law or agreed to in writing, software\n' \
        + '# distributed under the License is distributed on an "AS IS" BASIS,\n' \
        + '# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' \
        + '# See the License for the specific language governing permissions and\n' \
        + '# limitations under the License.\n' \
        + '#\n' \
        + '# DO NOT EDIT THIS FILE BY HAND\n' \
        + '# THIS FILE IS GENERATED FROM NNABLA.\n' \
        + '#\n' \
        + '\n'

    with open(yaml_file, 'w') as f:
        f.write(header + yaml.dump(_nnabla_func_info, default_flow_style=False)) 

def _quoted_string_representer(dumper, data):
    return dumper.represent_scalar("tag:yaml.org,2002:str", str(data)) 

def represent_odict(dumper, instance):
    return dumper.represent_mapping('tag:yaml.org,2002:map', instance.items()) 

def __init__(self, o, depth):
        self.tag = 'tag:yaml.org,2002:map'
        self.object = o
        self.depth = depth
        self.fields = FieldGroups(o.__class__) 

def represent_odict(dumper, instance):
    return dumper.represent_mapping('tag:yaml.org,2002:map', instance.items()) 

def represent_dict(dumper, data):
    """
    Represent dictionary with insert order
    """
    value = []

    for item_key, item_value in data.items():
        node_key = dumper.represent_data(item_key)
        node_value = dumper.represent_data(item_value)
        value.append((node_key, node_value))

    return yaml.nodes.MappingNode("tag:yaml.org,2002:map", value) 

def represent_dict(self, data):
    def key_function(elem):
        key = elem[0]
        # Prioritizes certain keys when sorting.
        prio = {"version": 0, "projects": 1, "repo": 2, "hash": 3}.get(key, 99)
        return (prio, key)
    items = data.items()
    items.sort(key=key_function)
    return self.represent_mapping(u'tag:yaml.org,2002:map', items) 

def represent_dict(self, data):
    def key_function(elem):
        key = elem[0]
        # Prioritizes certain keys when sorting.
        prio = {"model": 0, "pk": 1, "fields": 2}.get(key, 99)
        return (prio, key)
    items = data.items()
    items.sort(key=key_function)
    return self.represent_mapping(u'tag:yaml.org,2002:map', items) 

def save_params_dict_to_yaml(params, file_path):
  """Saves the input ParamsDict to a YAML file."""
  with tf.gfile.Open(file_path, 'w') as f:
    def _my_list_rep(dumper, data):
      # u'tag:yaml.org,2002:seq' is the YAML internal tag for sequence.
      return dumper.represent_sequence(
          u'tag:yaml.org,2002:seq', data, flow_style=True)
    yaml.add_representer(list, _my_list_rep)
    yaml.dump(params.as_dict(), f, default_flow_style=False) 

def represent_dictionary_order(self, dict_data):
    return self.represent_mapping('tag:yaml.org,2002:map', dict_data.items()) 

def __init__(self, *args, **kwargs):
        yaml.Loader.__init__(self, *args, **kwargs)

        self.add_constructor(u'tag:yaml.org,2002:map', type(self).construct_yaml_map)
        self.add_constructor(u'tag:yaml.org,2002:omap', type(self).construct_yaml_map) 

def test_search_by_tags_exclusion(self):
        # adding bookmarks
        for bookmark in self.bookmarks:
            self.bdb.add_rec(*bookmark)

        new_bookmark = ['https://newbookmark.com',
                        'New Bookmark',
                        parse_tags(['test,old,new']),
                        'additional bookmark to test multiple tag search']

        self.bdb.add_rec(*new_bookmark)

        with mock.patch('buku.prompt'):
            # search for bookmarks matching ANY of the supplied tags
            # while excluding bookmarks from results that match a given tag
            results = self.bdb.search_by_tag('test, old - est')
            # Expect a list of five-element tuples containing all bookmark data
            # db index, URL, title, tags, description
            expected = [
                (4, 'https://newbookmark.com', 'New Bookmark',
                 parse_tags([',test,old,new,']),
                 'additional bookmark to test multiple tag search', 0),
                (1, 'http://slashdot.org', 'SLASHDOT',
                 parse_tags([',news,old,']),
                 "News for old nerds, stuff that doesn't matter", 0),
            ]
            self.assertEqual(results, expected) 

def test_search_by_multiple_tags_search_any(self):
        # adding bookmarks
        for bookmark in self.bookmarks:
            self.bdb.add_rec(*bookmark)

        new_bookmark = ['https://newbookmark.com',
                        'New Bookmark',
                        parse_tags(['test,old,new']),
                        'additional bookmark to test multiple tag search', 0]

        self.bdb.add_rec(*new_bookmark)

        with mock.patch('buku.prompt'):
            # search for bookmarks matching ANY of the supplied tags
            results = self.bdb.search_by_tag('test, old')
            # Expect a list of five-element tuples containing all bookmark data
            # db index, URL, title, tags, description, ordered by records with
            # the most number of matches.
            expected = [
                (4, 'https://newbookmark.com', 'New Bookmark',
                 parse_tags([',test,old,new,']),
                 'additional bookmark to test multiple tag search', 0),
                (1, 'http://slashdot.org', 'SLASHDOT',
                 parse_tags([',news,old,']),
                 "News for old nerds, stuff that doesn't matter", 0),
                (3, 'http://example.com/', 'test', ',es,est,tes,test,', 'a case for replace_tag test', 0)
            ]
            self.assertEqual(results, expected) 

def test_get_rec_by_id(self):
        for bookmark in self.bookmarks:
            # adding bookmark from self.bookmarks
            self.bdb.add_rec(*bookmark)

        # the expected bookmark
        expected = (1, 'http://slashdot.org', 'SLASHDOT', ',news,old,',
                    "News for old nerds, stuff that doesn't matter", 0)
        bookmark_from_db = self.bdb.get_rec_by_id(1)
        # asserting bookmark matches expected
        self.assertEqual(expected, bookmark_from_db)
        # asserting None returned if index out of range
        self.assertIsNone(self.bdb.get_rec_by_id(len(self.bookmarks[0]) + 1)) 

def no_duplicate_yaml():
    yaml.SafeLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        no_duplicates_constructor)
    yaml.SafeLoader.add_constructor('tag:yaml.org,2002:python/tuple',
                                    tuple_constructor)
    try:
        yield
    finally:
        yaml.SafeLoader.add_constructor(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            yaml.constructor.SafeConstructor.construct_yaml_map
        ) 

def represent_str(dumper, data):

    # override to remove tag '!!python/str'
    tag = u'tag:yaml.org,2002:str'
    try:
        data = unicode(data, 'utf-8')
        style = None
    except UnicodeDecodeError:
        data = data.encode('base64')
        tag = u'tag:yaml.org,2002:binary'
        style = '|'

    return dumper.represent_scalar(tag, data, style=style)