Python yaml.add_constructor() Examples

The following are 24 code examples for showing how to use yaml.add_constructor(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module yaml , or try the search function .

Example 1
Project: TextDetector   Author: zchengquan   File: yaml_parse.py    License: GNU General Public License v3.0 7 votes vote down vote up
def initialize():
    """
    Initialize the configuration system by installing YAML handlers.
    Automatically done on first call to load() specified in this file.
    """
    global is_initialized

    # Add the custom multi-constructor
    yaml.add_multi_constructor('!obj:', multi_constructor_obj)
    yaml.add_multi_constructor('!pkl:', multi_constructor_pkl)
    yaml.add_multi_constructor('!import:', multi_constructor_import)

    yaml.add_constructor('!import', constructor_import)
    yaml.add_constructor("!float", constructor_float)

    pattern = re.compile(SCIENTIFIC_NOTATION_REGEXP)
    yaml.add_implicit_resolver('!float', pattern)

    is_initialized = True


###############################################################################
# Callbacks used by PyYAML 
Example 2
Project: custodia   Author: latchset   File: yaml_ext.py    License: GNU General Public License v3.0 6 votes vote down vote up
def demo():
    constructor = CustodiaSimpleConstructor(
        'http+unix://%2E%2Fserver_socket/secrets'
    )
    constructor.client.headers['REMOTE_USER'] = 'user'

    # create entries
    try:
        c = constructor.client.list_container('test')
    except HTTPError:
        constructor.client.create_container('test')
        c = []
    if 'key' not in c:
        constructor.client.set_secret('test/key', 'secret password')

    yaml.add_constructor(CustodiaSimpleConstructor.yaml_tag,
                         constructor)
    yaml_str = 'password: !custodia/simple test/key'
    print(yaml_str)
    result = yaml.load(yaml_str)
    print(result) 
Example 3
Project: HGNN   Author: iMoonLab   File: config.py    License: MIT License 6 votes vote down vote up
def get_config(dir='config/config.yaml'):
    # add direction join function when parse the yaml file
    def join(loader, node):
        seq = loader.construct_sequence(node)
        return os.path.sep.join(seq)

    # add string concatenation function when parse the yaml file
    def concat(loader, node):
        seq = loader.construct_sequence(node)
        seq = [str(tmp) for tmp in seq]
        return ''.join(seq)

    yaml.add_constructor('!join', join)
    yaml.add_constructor('!concat', concat)
    with open(dir, 'r') as f:
        cfg = yaml.load(f)

    check_dirs(cfg)

    return cfg 
Example 4
Project: DHGNN   Author: iMoonLab   File: config.py    License: MIT License 6 votes vote down vote up
def get_config(dir):
    # add direction join function when parse the yaml file
    def join(loader, node):
        seq = loader.construct_sequence(node)
        return os.path.sep.join(seq)

    # add string concatenation function when parse the yaml file
    def concat(loader, node):
        seq = loader.construct_sequence(node)
        return ''.join(seq)

    yaml.add_constructor('!join', join)
    yaml.add_constructor('!concat', concat)
    with open(dir, 'r') as f:
        cfg = yaml.load(f)

    return cfg 
Example 5
Project: agents-aea   Author: fetchai   File: loader.py    License: Apache License 2.0 6 votes vote down vote up
def _config_loader():
    envvar_matcher = re.compile(r"\${([^}^{]+)\}")

    def envvar_constructor(_loader, node):  # pragma: no cover
        """Extract the matched value, expand env variable, and replace the match."""
        node_value = node.value
        match = envvar_matcher.match(node_value)
        env_var = match.group()[2:-1]

        # check for defaults
        var_name, default_value = env_var.split(":")
        var_name = var_name.strip()
        default_value = default_value.strip()
        var_value = os.getenv(var_name, default_value)
        return var_value + node_value[match.end() :]

    yaml.add_implicit_resolver("!envvar", envvar_matcher, None, SafeLoader)
    yaml.add_constructor("!envvar", envvar_constructor, SafeLoader)


# TODO: instead of this, create custom loader and use it
#       by wrapping yaml.safe_load to use it 
Example 6
Project: dipper   Author: monarch-initiative   File: test_trtable.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def testIfTableIsAMap(self):

        from yaml.constructor import ConstructorError

        try:
            from yaml import CLoader as Loader
        except ImportError:
            from yaml import Loader

        # Credit https://gist.github.com/pypt/94d747fe5180851196eb
        def no_duplicates_constructor(loader, node, deep=False):
            """Check for duplicate keys."""

            mapping = {}
            for key_node, value_node in node.value:
                key = loader.construct_object(key_node, deep=deep)
                value = loader.construct_object(value_node, deep=deep)
                if key in mapping:
                    raise ConstructorError(
                        "while constructing a mapping", node.start_mark,
                        "found duplicate key (%s)" % key, key_node.start_mark)
                mapping[key] = value

            return loader.construct_mapping(node, deep)

        yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)

        file_path = '../translationtable/GLOBAL_TERMS.yaml'
        if os.path.exists(os.path.join(os.path.dirname(__file__), file_path)):
            tt_file = open(os.path.join(os.path.dirname(__file__), file_path), 'r')
            try:
                translation_table = yaml.safe_load(tt_file)
            except yaml.constructor.ConstructorError as e:
                tt_file.close()
                self.assertTrue(False)
                print(e)
            tt_file.close() 
Example 7
Project: forge   Author: datawire   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def setup_yaml():
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
    yaml.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_representer(os._Environ, dict_representer)
    yaml.add_representer(unicode, unicode_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor) 
Example 8
Project: pykit   Author: bsc-s2   File: utfyaml.py    License: MIT License 5 votes vote down vote up
def load(s, encoding='utf-8'):

    # override to set output encoding
    def construct_yaml_str(loader, node):
        value = loader.construct_scalar(node)

        if encoding is not None:
            return value.encode(encoding)

        return value

    yaml.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str)

    return yaml.load(s) 
Example 9
Project: yolo_v2   Author: rky0930   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 
Example 10
Project: Gun-Detector   Author: itsamitgoel   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 
Example 11
Project: prometheus-pgbouncer-exporter   Author: spreaker   File: config.py    License: MIT License 5 votes vote down vote up
def read(self, filepath):
        stream = False

        # Setup environment variables replacement
        def env_var_single_replace(match):
            return os.environ[match.group(1)] if match.group(1) in os.environ else match.group()

        def env_var_multi_replacer(loader, node):
            value = loader.construct_scalar(node)

            return re.sub(ENV_VAR_REPLACER_PATTERN, env_var_single_replace, value)

        yaml.add_implicit_resolver("!envvarreplacer", ENV_VAR_MATCHER_PATTERN)
        yaml.add_constructor('!envvarreplacer', env_var_multi_replacer)

        # Read file
        try:
            stream = open(filepath, "r")
            self.config = yaml.load(stream)

            # Handle an empty configuration file
            if not self.config:
                self.config = {}
        finally:
            if stream:
                stream.close() 
Example 12
Project: luscan-devel   Author: blackye   File: yamltags.py    License: GNU General Public License v2.0 5 votes vote down vote up
def register_tag(tag, classpath):
    yaml.add_constructor(u'!'+tag, metaloader(classpath))
    yaml.add_constructor(u'tag:nltk.org,2011:'+tag,
                         metaloader(classpath)) 
Example 13
Project: mayo   Author: deep-fry   File: parse.py    License: MIT License 5 votes vote down vote up
def register(cls):
        yaml.add_constructor(cls.tag, cls.constructor)
        yaml.add_representer(cls, cls.representer) 
Example 14
Project: bellybutton   Author: hchasestevens   File: parsing.py    License: MIT License 5 votes vote down vote up
def constructor(tag=None, pattern=None):
    """Register custom constructor with pyyaml."""
    def decorator(f):
        if tag is None or f is tag:
            tag_ = '!{}'.format(f.__name__)
        else:
            tag_ = tag
        yaml.add_constructor(tag_, f)
        if pattern is not None:
            yaml.add_implicit_resolver(tag_, re.compile(pattern))
        return f
    if callable(tag):  # little convenience hack to avoid empty arg list
        return decorator(tag)
    return decorator 
Example 15
Project: knowledge-repo   Author: airbnb   File: post.py    License: Apache License 2.0 5 votes vote down vote up
def setup_yaml():
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG

    def dict_representer(dumper, data):
        return dumper.represent_dict(data.items())

    def dict_constructor(loader, node):
        return collections.OrderedDict(loader.construct_pairs(node))

    yaml.SafeDumper.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor) 
Example 16
Project: review-rot   Author: redhat-aqe   File: __init__.py    License: GNU General Public License v3.0 5 votes vote down vote up
def load_ordered_config(config_path):
    """
      Loads the configuration in the same order as it's defined in yaml file,
      so that, while saving it in new format, order is maintained
      Args:
            config_path (str): Path to the configuration file
      Returns:
            config(dict): Returns the configurations in the defined ordered
    """

    #  To load data from yaml in ordered dict format
    _mapping_tag = yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG

    def dict_representer(dumper, data):
        return dumper.represent_mapping(_mapping_tag, iteritems(data))

    def dict_constructor(loader, node):
        return collections.OrderedDict(loader.construct_pairs(node))

    yaml.add_representer(collections.OrderedDict, dict_representer)
    yaml.add_constructor(_mapping_tag, dict_constructor)

    #  format the output to print a blank scalar rather than null
    def represent_none(self, _):
        return self.represent_scalar('tag:yaml.org,2002:null', u'')

    yaml.add_representer(type(None), represent_none)

    # read input from home directory for pull requests
    with open(config_path, 'r') as f:
        config = yaml.safe_load(f)
    return config 
Example 17
Project: MultiQC   Author: ewels   File: minionqc.py    License: GNU General Public License v3.0 5 votes vote down vote up
def parse_minionqc_report(self, s_name, f):
        '''
        Parses minionqc's 'summary.yaml' report file for results.
        Uses only the "All reads" stats. Ignores "Q>=x" part.
        '''
        try:
            # Parsing as OrderedDict is slightly messier with YAML
            # http://stackoverflow.com/a/21048064/713980
            def dict_constructor(loader, node):
                return OrderedDict(loader.construct_pairs(node))
            yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_constructor)
            summary_dict = yaml.safe_load(f)
        except Exception as e:
            log.error("Error parsing MinIONQC input file: {}".format(f))
            return

        # Do a deep copy as dicts are immutable
        self.minionqc_raw_data[s_name] = copy.deepcopy(summary_dict)

        # get q value threshold used for reads
        q_threshold = None
        for k in summary_dict.keys():
            if k.startswith('Q>='):
                q_threshold = k

        data_dict = {}
        data_dict['all'] = summary_dict['All reads']        # all reads
        data_dict['q_filt'] = summary_dict[q_threshold]     # quality filtered reads

        for q_key in ['all', 'q_filt']:
            for key_1 in ['reads', 'gigabases']:
                for key_2 in data_dict[q_key][key_1]:
                    new_key = '{} {}'.format(key_1, key_2)
                    data_dict[q_key][new_key] = data_dict[q_key][key_1][key_2]
                data_dict[q_key].pop(key_1)    # removes key after flattening

        self.minionqc_data[s_name] = data_dict['all']     # stats for all reads
        self.qfilt_data[s_name] = data_dict['q_filt']     # stats for q-filtered reads
        self.q_threshold_list.add(q_threshold)            # quality threshold used in this file 
Example 18
Project: object_detection_with_tensorflow   Author: scotthuang1989   File: util.py    License: MIT License 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 
Example 19
Project: monopsr   Author: kujason   File: config_utils.py    License: MIT License 5 votes vote down vote up
def parse_yaml_config(yaml_path):
    """Parses a yaml config

    Args:
        yaml_path: path to yaml config

    Returns:
        config_obj: config converted to object
    """

    # Add check for duplicate keys in yaml
    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, no_duplicates_constructor)

    with open(yaml_path, 'r') as yaml_file:
        config_dict = yaml.load(yaml_file)

    config_obj = config_dict_to_object(config_dict)
    config_obj.config_name = os.path.splitext(os.path.basename(yaml_path))[0]
    config_obj.exp_output_dir = monopsr.data_dir() + '/outputs/' + config_obj.config_name

    # Prepend data folder to paths
    paths_config = config_obj.train_config.paths_config
    if paths_config.checkpoint_dir is None:
        checkpoint_dir = config_obj.exp_output_dir + '/checkpoints'

        if not os.path.exists(checkpoint_dir):
            os.makedirs(checkpoint_dir)

        paths_config.checkpoint_dir = checkpoint_dir
    else:
        paths_config.checkpoint_dir = os.path.expanduser(paths_config.checkpoint_dir)

    paths_config.logdir = config_obj.exp_output_dir + '/logs'
    paths_config.pred_dir = config_obj.exp_output_dir + '/predictions'

    return config_obj 
Example 20
Project: nova   Author: saksdirect   File: nova_service_loader.py    License: MIT License 5 votes vote down vote up
def __init__(self, environment_name, nova_descriptor_file=None):
        self._nova_descriptor_file = nova_descriptor_file or 'nova.yml'
        self._environment_name = environment_name
        self._environment = None
        self._codedeploy_app = None

        self.templates_used = dict()
        yaml.add_constructor("!include", yaml_include)

        with open(os.path.join(spec.__path__[0], 'nova_service_schema.yml'), 'r') as schemaYaml:
            schema = yaml.load(schemaYaml)

        v = Validator(schema)
        try:
            with open(self._nova_descriptor_file, 'r') as novaYaml:
                self.service_spec = yaml.safe_load(novaYaml)

            # Validate loaded dictionary
            valid = v.validate(self.service_spec)
            if not valid:
                raise NovaError("Invalid nova service descriptor file '%s': %s" % (self._nova_descriptor_file, v.errors))
            else:
                self.service = Service.load(self.service_spec)
                self.service_name = self.service.name
                self.service_port = self.service.port
                self.service_healthcheck_url = self.service.healthcheck_url
        except IOError:
            raise NovaError("No nova service descriptor found at '%s'" % self._nova_descriptor_file) 
Example 21
Project: g-tensorflow-models   Author: generalized-iou   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 
Example 22
Project: models   Author: tensorflow   File: util.py    License: Apache License 2.0 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config 
Example 23
Project: vel   Author: MillionIntegrals   File: parser.py    License: MIT License 5 votes vote down vote up
def register(cls):
        """ Register variable handling in YAML """
        if not cls.IS_LOADED:
            cls.IS_LOADED = True

            yaml.add_constructor('!param', Parameter.parameter_constructor, Loader=yaml.SafeLoader)
            yaml.add_constructor('!env', EnvironmentVariable.parameter_constructor, Loader=yaml.SafeLoader) 
Example 24
Project: multilabel-image-classification-tensorflow   Author: isobar-us   File: util.py    License: MIT License 5 votes vote down vote up
def LoadConfigDict(config_paths, model_params):
  """Loads config dictionary from specified yaml files or command line yaml."""

  # Ensure that no duplicate keys can be loaded (causing pain).
  yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
                       NoDuplicatesConstructor)

  # Handle either ',' or '#' separated config lists, since borg will only
  # accept '#'.
  sep = ',' if ',' in config_paths else '#'

  # Load flags from config file.
  final_config = {}
  if config_paths:
    for config_path in config_paths.split(sep):
      config_path = config_path.strip()
      if not config_path:
        continue
      config_path = os.path.abspath(config_path)
      tf.logging.info('Loading config from %s', config_path)
      with tf.gfile.GFile(config_path.strip()) as config_file:
        config_flags = yaml.load(config_file)
        final_config = DeepMergeDict(final_config, config_flags)
  if model_params:
    model_params = MaybeLoadYaml(model_params)
    final_config = DeepMergeDict(final_config, model_params)
  tf.logging.info('Final Config:\n%s', yaml.dump(final_config))
  return final_config