Python yaml.Loader() Examples

def initialize_logger(self):

        if os.path.exists('config/logging.yml'):
            with open('config/logging.yml') as f:
                config = yaml.load(f, Loader=yaml.Loader)
                logging.config.dictConfig(config)
        else:
            log_format = '%(levelname)s - %(name)s - %(message)s'
            logging.basicConfig(format=log_format)

        self.logger = logging.getLogger("hemppa")

        if self.debug:
            logging.root.setLevel(logging.DEBUG)
            self.logger.info("enabled debugging")

        self.logger.debug("Logger initialized") 

def load(self, content):
        """Loads Limit Definitions from the given YAML content into this
        Telemetry Dictionary.  Content may be either a filename
        containing YAML content or a YAML string.

        Load has no effect if this Limits Dictionary was already
        instantiated with a filename or YAML content.
        """
        if self.filename is None:
            if os.path.isfile(content):
                self.filename = content
                stream        = open(self.filename, 'rb')
            else:
                stream        = content
            
            limits = yaml.load(stream, Loader=yaml.Loader)

            for lmt in limits:
                self.add(lmt)

            if isinstance(stream, IOBase):
                stream.close() 

def load(self, content):
        """Loads Command Definitions from the given YAML content into
        into this Command Dictionary.  Content may be either a
        filename containing YAML content or a YAML string.

        Load has no effect if this Command Dictionary was already
        instantiated with a filename or YAML content.
        """
        if self.filename is None:
            if os.path.isfile(content):
                self.filename = content
                stream        = open(self.filename, 'rb')
            else:
                stream        = content

            cmds = yaml.load(stream, Loader=yaml.Loader)
            cmds = handle_includes(cmds)
            for cmd in cmds:
                self.add(cmd)

            if isinstance(stream, IOBase):
                stream.close() 

def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    """load mappings and ordered mappings

    loader to load mappings and ordered mappings into the Python 2.7+ OrderedDict type,
    instead of the vanilla dict and the list of pairs it currently uses.
    """

    class OrderedLoader(Loader):
        pass

    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))

    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping
    )

    return yaml.load(stream, OrderedLoader) 

def read_yaml_env(fname):
    # type: (str) -> Dict[str, Any]
    """Parse YAML file with environment variable substitution.

    Parameters
    ----------
    fname : str
        yaml file name.

    Returns
    -------
    table : Dict[str, Any]
        the yaml file as a dictionary.
    """
    content = read_file(fname)
    # substitute environment variables
    content = string.Template(content).substitute(os.environ)
    return yaml.load(content, Loader=yaml.Loader) 

def read_yaml(fname):
    """Read the given file using YAML.

    Parameters
    ----------
    fname : string
        the file name.

    Returns
    -------
    content : Any
        the object returned by YAML.
    """
    with open_file(fname, 'r') as f:
        content = yaml.load(f, Loader=yaml.Loader)

    return content 

def load_yaml(config_file):
    """Load meta.yaml that is output when the model is converted.

    Args:
        config_file (str): Path of the configuration file.

    Returns:
        EasyDict: Dictionary object of loaded configuration file.

    Examples:
        >>> config = load_yaml("/path/of/meta.yaml")
    """
    with open(config_file) as config_file_stream:
        config = yaml.load(config_file_stream, Loader=yaml.Loader)
    # use only upper key.
    return EasyDict({k: v for k, v in config.items() if k.isupper()}) 

def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    """
    Получение сортированного словаря
    :param stream:
    :param Loader:
    :param object_pairs_hook:
    :return:
    """
    class OrderedLoader(Loader):
        pass
    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))
    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)
    return yaml.load(stream, OrderedLoader) 

def from_yaml(stream, cls=None, loader_cls=yaml.Loader,
              object_pairs_hook=OrderedDict, **extras):
    """
    Convert a YAML stream into a class via the OrderedLoader class.
    """

    class OrderedLoader(loader_cls):
        pass

    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))

    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)

    yaml_dict = yaml.load(stream, OrderedLoader) or {}
    yaml_dict.update(extras)
    return cls(**yaml_dict) if cls else yaml_dict 

def train(config_file, experiment_id=None, recreate=False, profile_step=-1):
    if not experiment_id:
        # Default model_name will be taken from config file: {model_name}.yml.
        model_name = os.path.splitext(os.path.basename(config_file))[0]
        experiment_id = '{}_{:%Y%m%d%H%M%S}'.format(model_name, datetime.now())

    run(config_file, experiment_id, recreate, profile_step)

    output_dir = os.environ.get('OUTPUT_DIR', 'saved')
    experiment_dir = os.path.join(output_dir, experiment_id)
    checkpoint = os.path.join(experiment_dir, 'checkpoints', 'checkpoint')

    if not tf.io.gfile.exists(checkpoint):
        raise Exception('Checkpoints are not created in {}'.format(experiment_dir))

    with tf.io.gfile.GFile(checkpoint) as stream:
        data = yaml.load(stream, Loader=yaml.Loader)
    checkpoint_name = os.path.basename(data['model_checkpoint_path'])

    return experiment_id, checkpoint_name 

def load(self, content):
        """Loads Packet Definitions from the given YAML content into this
        Telemetry Dictionary.  Content may be either a filename
        containing YAML content or a YAML string.

        Load has no effect if this Command Dictionary was already
        instantiated with a filename or YAML content.
        """
        if self.filename is None:
            if os.path.isfile(content):
                self.filename = content
                stream        = open(self.filename, 'rb')
            else:
                stream        = content

            pkts = yaml.load(stream, Loader=yaml.Loader)
            pkts = handle_includes(pkts)
            for pkt in pkts:
                self.add(pkt)

            if isinstance(stream, IOBase):
                stream.close() 

def get_config(infile):
    """
    read in the contents of the inputs yaml file
    """

    with open(infile, "r") as fp:
        config = load(fp, Loader=Loader)
    try:
        v_pen = tuple(config["v_pen"])
    except TypeError:
        v_pen = (config["v_pen"],)

    try:
        w_pen = tuple(config["w_pen"])
    except TypeError:
        w_pen = (config["w_pen"],)

    return v_pen, w_pen, config 

def load_yaml_file(filename, label='config file', loader=yaml.Loader):
    """Load a yaml config file.
    """
    try:
        with open(filename, "r") as file:
            res = yaml.load(file.read(), Loader=loader) or {}
            if not isinstance(res, dict):
                msg = f"Please ensure that {label} consists of key value pairs."
                raise VergeMLError(f"Invalid {label}: {filename}", msg)
            return res
    except yaml.YAMLError as err:
        if hasattr(err, 'problem_mark'):
            mark = getattr(err, 'problem_mark')
            problem = getattr(err, 'problem')
            message = f"Could not read {label} {filename}:"
            message += "\n" + display_err_in_file(filename, mark.line, mark.column, problem)
        elif hasattr(err, 'problem'):
            problem = getattr(err, 'problem')
            message = f"Could not read {label} {filename}: {problem}"
        else:
            message = f"Could not read {label} {filename}: YAML Error"

        suggestion = f"There is a syntax error in your {label} - please fix it and try again."

        raise VergeMLError(message, suggestion)

    except OSError as err:
        msg = "Please ensure the file exists and you have the required access privileges."
        raise VergeMLError(f"Could not open {label} {filename}: {err.strerror}", msg) 

def yaml_ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
    """
    Function to load YAML file using an OrderedDict
    See: https://stackoverflow.com/questions/5121931/in-python-how-can-you-load-yaml-mappings-as-ordereddicts
    """
    class OrderedLoader(Loader):
        pass

    def construct_mapping(loader, node):
        loader.flatten_mapping(node)
        return object_pairs_hook(loader.construct_pairs(node))

    OrderedLoader.add_constructor(
        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
        construct_mapping)

    return yaml.load(stream, OrderedLoader) 

def load_module(self, module_path, element_key, element_value):
        path_elements = module_path.split("::")
        dir_pattern = re.compile(fnmatch.translate(path_elements.pop(0)), re.IGNORECASE)
        matched_dirs = [dir for dir in os.listdir(self.path) if dir_pattern.match(dir)]
        matched_dir = module_path
        if matched_dirs:
            matched_dir = matched_dirs[0]

        module_path = self.path + "/" + "/".join([matched_dir] + path_elements)

        file_name = "*"

        if not os.path.isdir(module_path):
            file_name = module_path.split("/")[-1]
            module_path = "/".join(module_path.split("/")[:-1])

        properties = element_value.get("Properties", {})
        properties["module_name"] = element_key
        vars = self.merge_variables(properties)

        loader = Loader(module_path, file_name, vars)
        loader.load()
        self.merge(loader.template_dictionary(), file=file_name) 

def _download_results(config, _blob_client, out_path, count, ptrn=FOLD_FILE_PATTERN):

    pathlib.Path(config.BATCH_DIRECTORY).mkdir(parents=True, exist_ok=True)
    blob_names = [b.name for b in _blob_client.list_blobs(config.CONTAINER_NAME)]

    results = []
    for i in range(count):
        blob_name = ptrn.format(i)
        if not blob_name in blob_names:
            raise RuntimeError("incomplete blob set: missing blob {}".format(blob_name))
        out_path = os.path.join(config.BATCH_DIRECTORY, blob_name)
        with _blob_client.get_blob_to_stream(
            config.CONTAINER_NAME, blob_name, out_path
        ) as blob:
            results[i] = load(blob, Loader=Loader)
    return results 

def parse(self, parser):
        def parse_arguments():
            args = [parser.parse_expression()]
            # append task filters if any
            if parser.stream.skip_if('comma'):
                args.append(parser.parse_expression())
            return args

        lineno = next(parser.stream).lineno
        tag_args = parse_arguments()
        call = self.call_method(self._include_tasks.__name__, tag_args)
        return Output([call], lineno=lineno)


# By default, `yaml` package does not preserve field order, and
# playbook tasks do not look the same as in playbooks and in docs.
# These functions create custom Loader and Dumper to preserve field order.
# Source: https://stackoverflow.com/a/21912744 

def safe_yaml_loader(yamlfile, loaders=[yaml.FullLoader, yaml.Loader, None]):
    """
    Try different YAML loaders
    """
    args = None
    for loader in loaders:
        with open(yamlfile) as fp:
            try:
                args = yaml.load(fp, Loader=loader)
                break
                print(loader)
            except:
                pass

    if args is None:
        raise IOError('Failed to load {0} with {1}'.format(file, loaders))

    return args 

def load(self, ymlfile=None):
        """Load and process the YAML file"""
        if ymlfile is not None:
            self.ymlfile = ymlfile

        try:
            # If yaml should be 'cleaned' of document references
            if self._clean:
                self.data = self.process(self.ymlfile)
            else:
                with open(self.ymlfile, 'rb') as stream:
                    for data in yaml.load_all(stream, Loader=yaml.Loader):
                        self.data.append(data)

            self.loaded = True
        except ScannerError as e:
            msg = "YAML formattting error - '" + self.ymlfile + ": '" + str(e) + "'"
            raise util.YAMLError(msg) 

def yaml_load(source, loader=yaml.Loader):
    """
    Wrap PyYaml's loader so we can extend it to suit our needs.

    Load all strings as Unicode: http://stackoverflow.com/a/2967461/3609487.
    """

    def construct_yaml_str(self, node):
        """Override the default string handling function to always return Unicode objects."""
        return self.construct_scalar(node)

    class Loader(loader):
        """Define a custom loader to leave the global loader unaltered."""

    # Attach our Unicode constructor to our custom loader ensuring all strings
    # will be Unicode on translation.
    Loader.add_constructor('tag:yaml.org,2002:str', construct_yaml_str)

    return yaml.load(source, Loader) 

def _parse_filebeatyaml(self):

        def set_instance_var_from_token(variable_name, data):
            """
            :param variable_name: The name of the instance variable to update
            :param data: The parsed yaml object
            :return: True if successfully located
            """
            if variable_name not in self.tokens.keys():
                return False
            key_path = self.tokens[variable_name]
            value = data
            try:
                for k in key_path:
                    value = value[k]
                setattr(self, var_name, value)
            except KeyError:
                pass
            return True

        filebeatyaml_path = os.path.join(self.install_directory, 'filebeat.yml')
        try:
            with open(filebeatyaml_path, 'r') as configyaml:
                self.config_data = load(configyaml, Loader=Loader)
        except IOError:
            raise filebeat_exceptions.ReadFilebeatConfigError("Could not locate config at {}".format(filebeatyaml_path))
        except Exception as e:
            raise filebeat_exceptions.ReadFilebeatConfigError(
                "General exception when opening/parsing config at {}; {}".format(filebeatyaml_path, e))

        for var_name in vars(self).keys():
            set_instance_var_from_token(variable_name=var_name, data=self.config_data) 

def _test(self, artman_yaml, artifact_name, expected_legacy_config):
        artifact_config = loader.load_artifact_config(os.path.join(
            self.TESTDATA, artman_yaml), artifact_name)
        actual_legacy_config_dict = converter.convert_to_legacy_config_dict(
            artifact_config, '/tmp/input', '/tmp/output')
        with io.open(os.path.join(
                self.TESTDATA, expected_legacy_config), 'r') as yaml_file:
            expected_legacy_config_dict = yaml.load(yaml_file, Loader=yaml.Loader)
            self.assertDictEqual(
                expected_legacy_config_dict, actual_legacy_config_dict,
                'Actual yaml is:\n{}\nExpected yaml is:\n{}\n'.format(
                    yaml.dump(actual_legacy_config_dict, default_flow_style=False),
                    yaml.dump(expected_legacy_config_dict, default_flow_style=False))) 

def _parse_elasticyaml(self):

        def set_instance_var_from_token(variable_name, data):
            """
            :param variable_name: The name of the instance variable to update
            :param data: The parsed yaml object
            :return: True if successfully located
            """

            if variable_name not in self.tokens.keys():
                return False
            key_path = self.tokens[variable_name]
            value = data
            try:
                for k in key_path:
                    value = value[k]
                setattr(self, var_name, value)
            except KeyError:
                pass
            return True

        elasticyaml_path = os.path.join(self.configuration_directory, 'elasticsearch.yml')
        try:
            with open(elasticyaml_path, 'r') as configyaml:
                self.config_data = load(configyaml, Loader=Loader)
        except IOError:
            raise elastic_exceptions.ReadElasticConfigError("Could not locate config at {}".format(elasticyaml_path))
        except Exception as e:
            raise elastic_exceptions.ReadElasticConfigError(
                "General exception when opening/parsing config at {}; {}".format(elasticyaml_path, e))

        for var_name in vars(self).keys():
            set_instance_var_from_token(variable_name=var_name, data=self.config_data) 

def _parse_suricatayaml(self):

        def set_instance_var_from_token(variable_name, data):
            """
            :param variable_name: The name of the instance variable to update
            :param data: The parsed yaml object
            :return: True if successfully located
            """
            if variable_name not in self.tokens.keys():
                return False
            key_path = self.tokens[variable_name]
            value = data
            for k in key_path:
                value = value[k]
            setattr(self, var_name, value)
            return True

        suricatayaml_path = os.path.join(self.configuration_directory, 'suricata.yaml')
        try:
            with open(suricatayaml_path, 'r') as configyaml:
                self.config_data = load(configyaml, Loader=Loader)
        except IOError:
            raise suricata_exceptions.ReadsSuricataConfigError(
                "Could not locate config at {}".format(suricatayaml_path))
        except Exception as e:
            raise suricata_exceptions.ReadsSuricataConfigError(
                "General exception when opening/parsing config at {}; {}".format(suricatayaml_path, e))

        for var_name in vars(self).keys():
            set_instance_var_from_token(variable_name=var_name, data=self.config_data) 

def _parse_logstashyaml(self):

        def set_instance_var_from_token(variable_name, data):
            """
            :param variable_name: The name of the instance variable to update
            :param data: The parsed yaml object
            :return: True if successfully located
            """
            if variable_name not in self.tokens.keys():
                return False
            key_path = self.tokens[variable_name]
            value = data
            try:
                for k in key_path:
                    value = value[k]
                setattr(self, var_name, value)
            except KeyError:
                pass
            return True

        logstashyaml_path = os.path.join(self.configuration_directory, 'logstash.yml')
        try:
            with open(logstashyaml_path, 'r') as configyaml:
                self.config_data = load(configyaml, Loader=Loader)
        except IOError:
            raise logstash_exceptions.ReadLogstashConfigError("Could not locate config at {}".format(logstashyaml_path))
        except Exception as e:
            raise logstash_exceptions.ReadLogstashConfigError(
                "General exception when opening/parsing config at {}; {}".format(logstashyaml_path, e))

        for var_name in vars(self).keys():
            set_instance_var_from_token(variable_name=var_name, data=self.config_data) 

def _nc_dict_decoder(nc_variable):
        decoded_string = nc_string_decoder(nc_variable)
        # Handle array type
        try:
            output = yaml.load(decoded_string, Loader=_DictYamlLoader)
        except (AttributeError, TypeError):  # Appended data
            n_entries = decoded_string.shape[0]
            output = np.empty(n_entries, dtype=dict)
            for n in range(n_entries):
                output[n] = yaml.load(str(decoded_string[n, 0]), Loader=_DictYamlLoader)
        return output 

def ordered_load(self, text, Loader=yaml.Loader, object_pairs_hook=OrderedDict):
        """Allows you to use `pyyaml` to load as OrderedDict.

        Taken from https://stackoverflow.com/a/21912744/1927102
        """
        class OrderedLoader(Loader):
            pass

        def construct_mapping(loader, node):
            loader.flatten_mapping(node)
            return object_pairs_hook(loader.construct_pairs(node))
        OrderedLoader.add_constructor(
            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
            construct_mapping)
        try:
            text = text.replace('\\(', '\\\\(')
            text = text.replace('\\)', '\\\\)')
            try:
                result = yaml.load(text, OrderedLoader)
            except yaml.scanner.ScannerError:
                # will it load as plain json?
                result = json.loads(text, object_pairs_hook=object_pairs_hook)
        except Exception as e:
            print("exception: ",e)
            self.error(e)
            result = {}
        return result 

def _parse_kibanayaml(self):

        def set_instance_var_from_token(variable_name, data):
            """
            :param variable_name: The name of the instance variable to update
            :param data: The parsed yaml object
            :return: True if successfully located
            """
            if variable_name not in self.tokens.keys():
                return False
            key_path = self.tokens[variable_name]
            value = data
            try:
                for k in key_path:
                    value = value[k]
                setattr(self, var_name, value)
            except KeyError:
                pass
            return True

        kibanayaml_path = os.path.join(self.configuration_directory, 'kibana.yml')
        try:
            with open(kibanayaml_path, 'r') as configyaml:
                self.config_data = load(configyaml, Loader=Loader)
        except IOError:
            raise kibana_exceptions.ReadKibanaConfigError("Could not locate config at {}".format(kibanayaml_path))
        except Exception as e:
            raise kibana_exceptions.ReadKibanaConfigError(
                "General exception when opening/parsing config at {}; {}".format(kibanayaml_path, e))

        for var_name in vars(self).keys():
            set_instance_var_from_token(variable_name=var_name, data=self.config_data) 

def __init__(self, *args, **kwargs):
        yaml.Loader.__init__(self, *args, **kwargs)
        self.add_constructor(u'tag:yaml.org,2002:omap', type(self).construct_odict) 

def __init__(self, stream):
        """Initialise Loader."""

        try:
            self._root = os.path.split(stream.name)[0]
        except AttributeError:
            self._root = os.path.curdir

        super().__init__(stream)