Python ruamel.yaml.YAML Examples

def fix_date_and_remove_null(yaml_file, date, input_type='ruamel'):
    """
    Remove the single quotes around the date key-value pair in the provided yaml_file and remove any 'null' values
    :param yaml_file: ruamel.yaml instance or location of YAML file
    :param date: string date value (e.g. 2019-01-01)
    :param input_type: input type can be a ruamel.yaml instance or list
    :return: YAML file lines in a list
    """
    _yaml = init_yaml()
    if input_type == 'ruamel':
        # ruamel does not support output to a variable. Therefore we make use of StringIO.
        file = StringIO()
        _yaml.dump(yaml_file, file)
        file.seek(0)
        new_lines = file.readlines()
    elif input_type == 'list':
        new_lines = yaml_file
    elif input_type == 'file':
        new_lines = yaml_file.readlines()

    fixed_lines = [l.replace('\'' + str(date) + '\'', str(date)).replace('null', '')
                   if REGEX_YAML_DATE.match(l) else
                   l.replace('null', '') for l in new_lines]

    return fixed_lines 

def write_case_settings_file(settings, folder, file_name):
    """Write a finalized dictionary to YAML file.

    Parameters
    ----------
    settings : dict
        A dictionary with settings
    folder : Path-like
        A Path object representing the folder for a single case/scenario
    file_name : str
        Name of the file.
    """
    folder.mkdir(exist_ok=True, parents=True)
    path_out = folder / file_name

    # yaml = YAML(typ="unsafe")
    _settings = deepcopy(settings)
    # for key, value in _settings.items():
    #     if isinstance(value, Path):
    #         _settings[key] = str(value)
    # yaml.register_class(Path)
    # stream = file(path_out, 'w')
    with open(path_out, "w") as f:
        yaml.dump(_settings, f) 

def main():
    input_file_path, output_file_path = getRuntimeArgs()
    yaml_parser = YAML()

    #parse the cli command and the parameters
    cli_cmd, params = parseCLISpec(input_file_path)
    print(cli_cmd)
    print(params)

    # load yml base template
    with open(YML_TEMPLATE_PATH, 'r') as yml_template_stream:
        yml = yaml_parser.load(yml_template_stream)

    # Update the base yaml with new cli spec
    updateYMLWithCLISpec(yaml_parser, cli_cmd, params, yml)

    # write it out
    yaml_parser.dump(yml, sys.stdout)
    with open(output_file_path, 'w') as output_stream:
        yaml_parser.dump(yml, output_stream) 

def _load_aws_credentials_from_yaml(yaml_file_path) -> Tuple[str, str]:
    with open(yaml_file_path, "r") as yaml_file:
        try:
            credentials_yaml = YAML().load(yaml_file.read())
        except Exception as e:
            print(
                PaastaColors.red(
                    "Encountered %s when trying to parse AWS credentials yaml %s. "
                    "Suppressing further output to avoid leaking credentials."
                    % (type(e), yaml_file_path)
                )
            )
            sys.exit(1)

        return (
            credentials_yaml["aws_access_key_id"],
            credentials_yaml["aws_secret_access_key"],
        ) 

def get_api_definition_from_file(src_file):
    try:
        with open(src_file, mode='rb') as f:
            api_definition = f.read()
        try:
            return json.loads(api_definition.decode('utf-8'))
        except ValueError as e:
            print('Failed to load input as JSON, maybe YAML?')
        try:
            yaml = YAML(typ='safe')
            return yaml.load(api_definition)
        except (TypeError, ScannerError) as e:
            print('Failed to load input as YAML:{}'.format(e))
            raise e
    except (Exception, FileNotFoundError):
        print('Failed to parse input file, exit')
        raise FailedToParseFileException 

def yaml_dump(cls, stream=sys.stdout):
        class MyRepresenter(SafeRepresenter):
            def ignore_aliases(self, data):
                return True

        yaml = YAML(typ="safe")
        yaml.default_flow_style = False
        yaml.Representer = MyRepresenter

        types = EntityTypeBase.get_entity_types()

        for _, t in types.items():
            yaml.register_class(t)

        yaml.indent(mapping=2, sequence=4, offset=2)
        yaml.dump(cls, stream=stream) 

def read_meta(path: str) -> Optional[PluginMeta]:
    try:
        with open(os.path.join(path, "maubot.yaml")) as meta_file:
            try:
                meta_dict = yaml.load(meta_file)
            except YAMLError as e:
                print(Fore.RED + "Failed to build plugin: Metadata file is not YAML")
                print(Fore.RED + str(e) + Fore.RESET)
                return None
    except FileNotFoundError:
        print(Fore.RED + "Failed to build plugin: Metadata file not found" + Fore.RESET)
        return None
    try:
        meta = PluginMeta.deserialize(meta_dict)
    except SerializerError as e:
        print(Fore.RED + "Failed to build plugin: Metadata file is not valid")
        print(Fore.RED + str(e) + Fore.RESET)
        return None
    return meta 

def _read_meta(source) -> Tuple[ZipFile, PluginMeta]:
        try:
            file = ZipFile(source)
            data = file.read("maubot.yaml")
        except FileNotFoundError as e:
            raise MaubotZipMetaError("Maubot plugin not found") from e
        except BadZipFile as e:
            raise MaubotZipMetaError("File is not a maubot plugin") from e
        except KeyError as e:
            raise MaubotZipMetaError("File does not contain a maubot plugin definition") from e
        try:
            meta_dict = yaml.load(data)
        except (YAMLError, KeyError, IndexError, ValueError) as e:
            raise MaubotZipMetaError("Maubot plugin definition file is not valid YAML") from e
        try:
            meta = PluginMeta.deserialize(meta_dict)
        except SerializerError as e:
            raise MaubotZipMetaError("Maubot plugin definition in file is invalid") from e
        return file, meta 

def to_hour(num) -> str:
    """
    Convert YAML input to hours

    Args:
        num: number in YMAL file, e.g., 900, 1700, etc.

    Returns:
        str

    Examples:
        >>> to_hour(900)
        '09:00'
        >>> to_hour(1700)
        '17:00'
    """
    to_str = str(int(num))
    return pd.Timestamp(f'{to_str[:-2]}:{to_str[-2:]}').strftime('%H:%M') 

def write_configuration(config_filename, rendered_config, mode=0o600):
    '''
    Given a target config filename and rendered config YAML, write it out to file. Create any
    containing directories as needed.
    '''
    if os.path.exists(config_filename):
        raise FileExistsError('{} already exists. Aborting.'.format(config_filename))

    try:
        os.makedirs(os.path.dirname(config_filename), mode=0o700)
    except (FileExistsError, FileNotFoundError):
        pass

    with open(config_filename, 'w') as config_file:
        config_file.write(rendered_config)

    os.chmod(config_filename, mode) 

def generate_sample_configuration(source_filename, destination_filename, schema_filename):
    '''
    Given an optional source configuration filename, and a required destination configuration
    filename, and the path to a schema filename in pykwalify YAML schema format, write out a
    sample configuration file based on that schema. If a source filename is provided, merge the
    parsed contents of that configuration into the generated configuration.
    '''
    schema = yaml.round_trip_load(open(schema_filename))
    source_config = None

    if source_filename:
        source_config = load.load_configuration(source_filename)

    destination_config = merge_source_configuration_into_destination(
        _schema_to_sample_configuration(schema), source_config
    )

    write_configuration(
        destination_filename,
        _comment_out_optional_configuration(_render_configuration(destination_config)),
    ) 

def get_platform_from_yaml(yaml_content):
    """
    Read the platform field from the YAML file supporting both string and list values.
    :param yaml_content: the content of the YAML file containing the platform field
    :return: the platform value
    """
    platform = yaml_content.get('platform', None)
    if platform is None:
        return []
    if isinstance(platform, str):
        platform = [platform]
    platform = [p.lower() for p in platform if p is not None]

    if platform == ['all']:
        platform = 'all'
    else:
        valid_platform_list = []
        for p in platform:
            if p in PLATFORMS.keys():
                valid_platform_list.append(PLATFORMS[p])
        platform = valid_platform_list
    return platform 

def __init__(self, bento_service=None, kind="BentoService"):
        self.kind = kind
        self._yaml = YAML()
        self._yaml.default_flow_style = False
        self.config = self._yaml.load(
            BENTOML_CONFIG_YAML_TEPMLATE.format(
                kind=self.kind,
                bentoml_version=get_bentoml_deploy_version(),
                created_at=str(datetime.utcnow()),
            )
        )

        if bento_service is not None:
            self.config["metadata"].update(
                {
                    "service_name": bento_service.name,
                    "service_version": bento_service.version,
                }
            )
            self.config["env"] = bento_service.env.to_dict()
            self.config['apis'] = _get_apis_list(bento_service)
            self.config['artifacts'] = _get_artifacts_list(bento_service) 

def get_devices_conn_params() -> Dict[str, Dict[str, str]]:
    """Creates a dictionary of connection parameters for SSH"""
    result: Dict[str, Dict[str, str]] = {}
    yaml = YAML()
    with open(HOSTS_FILE, 'r') as f:
        hosts = yaml.load(f)
    for device, device_details in hosts["devices"]["routers"].items():
        device_params = {
            "host": device_details["host"],
            "username": DEVICE_USERNAME,
            "password": DEVICE_PASSWORD,
            "device_type": DEVICE_TYPE,
            "timeout": CONNECTION_TIMEOUT,
            "global_delay_factor": constants.NETMIKO_GLOBAL_DELAY_FACTOR,
        }
        result[device] = device_params
    return result 

def set_yaml_dv_comments(yaml_object):
    """
    Set all comments in the detection or visibility YAML object when the 'comment' key-value pair is missing or is None.
    This gives the user the flexibility to have YAML files with missing 'comment' key-value pairs.
    :param yaml_object: detection or visibility object
    :return: detection or visibility object for which empty comments are filled with an empty string
    """
    yaml_object['comment'] = yaml_object.get('comment', '')
    if yaml_object['comment'] is None:
        yaml_object['comment'] = ''
    if 'score_logbook' in yaml_object:
        for score_obj in yaml_object['score_logbook']:
            score_obj['comment'] = score_obj.get('comment', '')
            if score_obj['comment'] is None:
                score_obj['comment'] = ''

    return yaml_object 

def replace_environment_variables() -> None:
    """Enable yaml loader to process the environment variables in the yaml."""
    import re
    import os

    # noinspection RegExpRedundantEscape
    env_var_pattern = re.compile(r"^(.*)\$\{(.*)\}(.*)$")
    yaml.add_implicit_resolver('!env_var', env_var_pattern)

    def env_var_constructor(loader, node):
        """Process environment variables found in the YAML."""
        value = loader.construct_scalar(node)
        prefix, env_var, postfix = env_var_pattern.match(value).groups()
        return prefix + os.environ[env_var] + postfix

    yaml.SafeConstructor.add_constructor(u'!env_var', env_var_constructor) 

def main():
    yaml = YAML()
    opts = parse_args()

    source_yaml = yaml.load(Path(opts.source_yaml))

    nav_entries = build_api_toc(Path(opts.api_docs_path), Path(opts.docs_root))

    # Add version to name.
    source_yaml["site_name"] = f"AllenNLP {opts.docs_version}"

    # Find the yaml sub-object corresponding to the API table of contents.
    site_nav = source_yaml["nav"]
    for nav_obj in site_nav:
        if API_TOC_KEY in nav_obj:
            break
    nav_obj[API_TOC_KEY] = nav_entries

    with open(opts.target_yaml, "w") as f:
        yaml.dump(source_yaml, f)

    print(f"{opts.target_yaml} created") 

def _readYaml(self, stream):
        """
        Read geometry from yaml.

        Notes
        -----
        This is intended to replace the XML format as we converge on
        consistent inputs.
        """
        yaml = YAML()
        tree = yaml.load(stream)
        tree = INPUT_SCHEMA(tree)
        self.assemTypeByIndices.clear()
        for _systemName, system in tree[INP_SYSTEMS].items():
            # no need to check for valid since the schema handled that.
            self.geomType = system[INP_GEOM]
            self.symmetry = system[INP_SYMMETRY]
            if INP_DISCRETES in system:
                self._read_yaml_discretes(system)
            elif INP_LATTICE in system:
                self._read_yaml_lattice(system) 

def _readYaml(self, stream, handleInvalids=True):
        """
        Read settings from a YAML stream.

        Notes
        -----
        This is intended to replace the XML stuff as we converge on consistent input formats.
        """
        from armi.physics.thermalHydraulics import const  # avoid circular import

        yaml = YAML()
        tree = yaml.load(stream)
        if "settings" not in tree:
            raise exceptions.InvalidSettingsFileError(
                self.inputPath,
                "Missing the `settings:` header required in YAML settings",
            )
        if const.ORIFICE_SETTING_ZONE_MAP in tree:
            raise exceptions.InvalidSettingsFileError(
                self.inputPath, "Appears to be an orifice_settings file"
            )
        caseSettings = tree[Roots.CUSTOM]
        self.inputVersion = tree["metadata"][Roots.VERSION]
        for settingName, settingVal in caseSettings.items():
            self._applySettings(settingName, settingVal) 

def get_aws_region_for_paasta_cluster(paasta_cluster: str) -> str:
    with open(CLUSTERMAN_YAML_FILE_PATH, "r") as clusterman_yaml_file:
        clusterman_yaml = YAML().load(clusterman_yaml_file.read())
        return clusterman_yaml["clusters"][paasta_cluster]["aws_region"] 

def save(self, filename):
        """ Save a YamlConfig to disk. """
        y = yaml.YAML()
        y.dump(self.config, open(filename, 'w')) 

def save(self, path):
        '''Serialize self to a file using YAML format
        '''
        yaml = YAML(typ='safe')
        yaml.default_flow_style = False
        with path.open('w') as fp:
            fp.write("#\n"
                     "# This file was generated using mkCTF utility.\n"
                     "# Do not edit it manually unless you know exactly what you're doing.\n"
                     "# Keep #PEBCAK in mind.\n"
                     "#\n")
            yaml.dump(self.raw, fp) 

def parse_config_file(config_fn, sort_names):
    yaml = YAML()
    with open(config_fn) as f:
        d = yaml.load(f)
    datasets = d['Datasets'].keys()
    if sort_names:
        datasets.sort()
    datasets_labels = {}
    datasets_titles = {}
    for v in datasets:
        datasets_labels[v] = d['Datasets'][v]['label']
        if 'title' in d['Datasets'][v]:
            datasets_titles[v] = d['Datasets'][v]['title']

    algorithms = d['Algorithms'].keys()
    if sort_names:
        algorithms.sort()
    alg_labels = {}
    alg_fn = {}
    for v in algorithms:
        alg_labels[v] = d['Algorithms'][v]['label']
        alg_fn[v] = d['Algorithms'][v]['fn']

    boxplot_distances = []
    if 'RelDistances' in d:
        boxplot_distances = d['RelDistances']
    boxplot_percentages = []
    if 'RelDistancePercentages' in d:
        boxplot_percentages = d['RelDistancePercentages']

    if boxplot_distances and boxplot_percentages:
        print(Fore.RED + "Found both both distances and percentages for boxplot distances")
        print(Fore.RED + "Will use the distances instead of percentages.")
        boxplot_percentages = []

    return datasets, datasets_labels, datasets_titles, algorithms, alg_labels, alg_fn,\
        boxplot_distances, boxplot_percentages 

def load(cls, path):
        '''Load and build a class from a YAML configuration
        '''
        conf = cls()
        if path.is_file():
            try:
                app_log.debug(f"loading {cls.TYPE} configuration from {path}")
                conf = YAML(typ='safe').load(path)
                conf = cls(conf)
            except:
                app_log.exception(f"failed to load {cls.TYPE} configuration from {path}")
                raise MKCTFAPIException("configuration load failed.")
        return conf 

def __init__(self, name, python_version):
        self._yaml = YAML()
        self._yaml.default_flow_style = False

        self._conda_env = self._yaml.load(
            CONDA_ENV_BASE_YAML.format(name=name, python_version=python_version)
        ) 

def get_default_event_log_dir(**kwargs) -> str:
    if "access_key" not in kwargs or "secret_key" not in kwargs:
        access_key, secret_key = get_aws_credentials(**kwargs)
    else:
        access_key, secret_key = kwargs["access_key"], kwargs["secret_key"]
    if access_key is None:
        log.warning(
            "Since no AWS credentials were provided, spark event logging "
            "will be disabled"
        )
        return None

    try:
        with open(DEFAULT_SPARK_RUN_CONFIG) as fp:
            spark_run_conf = YAML().load(fp.read())
    except Exception as e:
        log.warning(f"Failed to load {DEFAULT_SPARK_RUN_CONFIG}: {e}")
        log.warning("Returning empty default configuration")
        spark_run_conf = {}

    try:
        account_id = (
            boto3.client(
                "sts", aws_access_key_id=access_key, aws_secret_access_key=secret_key
            )
            .get_caller_identity()
            .get("Account")
        )
    except Exception as e:
        log.warning("Failed to identify account ID, error: {}".format(str(e)))
        return None

    for conf in spark_run_conf.get("environments", {}).values():
        if account_id == conf["account_id"]:
            default_event_log_dir = conf["default_event_log_dir"]
            print(f"default event logging at: {default_event_log_dir}")
            return default_event_log_dir
    return None 

def read_yaml(content: Text) -> Any:
    """Parses yaml from a text.

     Args:
        content: A text containing yaml content.
    """
    fix_yaml_loader()
    replace_environment_variables()

    yaml_parser = yaml.YAML(typ="safe")
    yaml_parser.version = "1.2"
    yaml_parser.unicode_supplementary = True

    # noinspection PyUnresolvedReferences
    try:
        return yaml_parser.load(content)
    except yaml.scanner.ScannerError as _:
        # A `ruamel.yaml.scanner.ScannerError` might happen due to escaped
        # unicode sequences that form surrogate pairs. Try converting the input
        # to a parsable format based on
        # https://stackoverflow.com/a/52187065/3429596.
        content = (content.encode('utf-8')
                   .decode('raw_unicode_escape')
                   .encode("utf-16", 'surrogatepass')
                   .decode('utf-16'))
        return yaml_parser.load(content) 

def load_spec_dict(spec_path):
    with open(spec_path, 'r') as spec_file:
        return YAML().load(spec_file.read()) 

def test_yamlIO(self):
        """Ensure we can read/write this custom setting object to yaml"""
        yaml = YAML()
        inp = yaml.load(io.StringIO(XS_EXAMPLE))
        xs = XSSettingDef("TestSetting", XSSettings())
        xs._load(inp)  # pylint: disable=protected-access
        self.assertEqual(xs.value["BA"].geometry, "1D slab")
        outBuf = io.StringIO()
        output = xs.dump()
        yaml.dump(output, outBuf)
        outBuf.seek(0)
        inp2 = yaml.load(outBuf)
        self.assertEqual(inp.keys(), inp2.keys()) 

def __convert_key(expression):
        """Converts keys in YAML that reference other keys.
        """
        if type(expression) is str and len(expression) > 2 and expression[1] == '!':
            expression = eval(expression[2:-1])
        return expression