Python ruamel.yaml.safe_load() Examples

def _init(cls):

        if cls.package_name is None:
            raise NotImplementedError("Package name not given")

        if cls.spec_template_file is None:
            raise NotImplementedError("Spec file not given")

        loader = PackageLoader(cls.package_name, "")
        env = Environment(loader=loader)
        template = env.get_template(cls.spec_template_file)
        tdict = yaml.safe_load(StringIO(template.render()))
        tdict = jsonref.loads(json.dumps(tdict))

        # TODO - Check if keys are present
        cls.provider_spec = tdict["components"]["schemas"]["provider_spec"]
        cls.Validator = StrictDraft7Validator(cls.provider_spec) 

def load_yaml(filename):
    """Return object in yaml file."""
    with open(filename) as myfile:
        content = myfile.read()
        if "win" in sys.platform:
            content = content.replace("\\", "/")

        try:
            obj = yaml.safe_load(content)

            # packed workflow, will be ignored later
            if obj.get('$graph'):
                obj = {}
        # packed workflow, will be ignored later
        # (it seems in some cases a packed workflow gives an ParserError, while
        # in other cases it is loaded correctly)
        except yaml.parser.ParserError:
            obj = {}
        return obj 

def compile(self):
        '''build an internal mapping of all nwo meta'''
        migration_repo = self.g.get_repo('ansible-community/collection_migration')
        ansible_repo = self.g.get_repo('ansible/ansible')

        self._nwo = {}
        self._flatmaps = set()

        # Read in migration scenarios
        for f in migration_repo.get_contents('scenarios/nwo'):
            data = yaml.safe_load(base64.b64decode(f.content))
            namespace, ext = os.path.splitext(f.name)
            if ext != '.yml':
                continue
            for collection, content in data.items():
                name = '%s.%s' % (namespace, collection)
                if content.get('_options', {}).get('flatmap'):
                    self._flatmaps.add(name)
                for ptype, paths in content.items():
                    for relpath in paths:
                        if ptype in ('modules', 'module_utils'):
                            path = 'lib/ansible/%s/%s' % (ptype, relpath)
                        else:
                            path = 'lib/ansible/plugins/%s/%s' % (ptype, relpath)
                        self._nwo[path] = name 

def run():
    options.logging = 'info'
    parse_command_line(args=[])
    enable_pretty_logging()

    with open(args.config, 'r') as f:
        conf = yaml.safe_load(f)

    parameters = conf['parameters']

    AsyncIOMainLoop().install()

    app = Application([
        (r"/", MainHandler, dict(flag=conf['flag']))
    ], debug=args.debug)

    app.listen(parameters['port'])

    asyncio.get_event_loop().run_forever()
#-------------------------------------------------------------------------------
# SCRIPT
#------------------------------------------------------------------------------- 

def _update_from_file(self, filename):
        """ Helper method to update an existing configuration with the values from a file.

        Loads a configuration file and replaces all values in the existing configuration
        dictionary with the values from the file.

        Args:
            filename (str): The path and name to the configuration file.
        """
        if os.path.exists(filename):
            try:
                with open(filename, 'r') as config_file:
                    yaml_dict = yaml.safe_load(config_file.read())
                    if yaml_dict is not None:
                        self._update_dict(self._config, yaml_dict)
            except IsADirectoryError:
                raise ConfigLoadError(
                    'The specified configuration file is a directory not a file')
        else:
            raise ConfigLoadError('The config file {} does not exist'.format(filename)) 

def update_component_meta(self, component_name, component_module_name, model_alias, model_proto_index):
        """
        update meta info yaml
        TODO: with lock
        :param component_name:
        :param component_module_name:
        :param model_alias:
        :param model_proto_index:
        :return:
        """
        with open(self.define_meta_path, "r", encoding="utf-8") as fr:
            define_index = yaml.safe_load(fr)
        with open(self.define_meta_path, "w", encoding="utf-8") as fw:
            define_index["component_define"] = define_index.get("component_define", {})
            define_index["component_define"][component_name] = define_index["component_define"].get(component_name, {})
            define_index["component_define"][component_name].update({"module_name": component_module_name})
            define_index["model_proto"] = define_index.get("model_proto", {})
            define_index["model_proto"][component_name] = define_index["model_proto"].get(component_name, {})
            define_index["model_proto"][component_name][model_alias] = define_index["model_proto"][component_name].get(model_alias, {})
            define_index["model_proto"][component_name][model_alias].update(model_proto_index)
            yaml.dump(define_index, fw, Dumper=yaml.RoundTripDumper) 

def collect_models(self, in_bytes=False, b64encode=True):
        model_buffers = {}
        with open(self.define_meta_path, "r", encoding="utf-8") as fr:
            define_index = yaml.safe_load(fr)
            for component_name in define_index.get("model_proto", {}).keys():
                for model_alias, model_proto_index in define_index["model_proto"][component_name].items():
                    component_model_storage_path = os.path.join(self.variables_data_path, component_name, model_alias)
                    for model_name, buffer_name in model_proto_index.items():
                        with open(os.path.join(component_model_storage_path, model_name), "rb") as fr:
                            buffer_object_serialized_string = fr.read()
                            if not in_bytes:
                                model_buffers[model_name] = self.parse_proto_object(buffer_name=buffer_name,
                                                                                    buffer_object_serialized_string=buffer_object_serialized_string)
                            else:
                                if b64encode:
                                    buffer_object_serialized_string = base64.b64encode(buffer_object_serialized_string).decode()
                                model_buffers["{}.{}:{}".format(component_name, model_alias, model_name)] = buffer_object_serialized_string
        return model_buffers 

def get_resource_file(sfile, sfile_type='yaml'):
        ''' return the service file '''
        contents = None
        with open(sfile) as sfd:
            contents = sfd.read()

        if sfile_type == 'yaml':
            # AUDIT:no-member makes sense here due to ruamel.YAML/PyYAML usage
            # pylint: disable=no-member
            if hasattr(yaml, 'RoundTripLoader'):
                contents = yaml.load(contents, yaml.RoundTripLoader)
            else:
                contents = yaml.safe_load(contents)
        elif sfile_type == 'json':
            contents = json.loads(contents)

        return contents 

def test_get_account_with_creds(self):
        os.environ["MY_USERNAME"] = "foo"
        os.environ["MY_APIKEY"] = "bar"
        config = yaml.safe_load(
            "environments:\n"
            "  - name: {}\n"
            '    account: "{}"\n'
            "    rs_username_var: MY_USERNAME\n"
            "    rs_apikey_var: MY_APIKEY".format(self.environment, self.account)
        )
        account, role, username, apikey = run.get_account(config, self.environment)
        del os.environ["MY_USERNAME"]
        del os.environ["MY_APIKEY"]
        self.assertEqual(account, self.account)
        self.assertIsNone(role)
        self.assertEqual(username, "foo")
        self.assertEqual(apikey, "bar") 

def get_resource_file(sfile, sfile_type='yaml'):
        ''' return the service file '''
        contents = None
        with open(sfile) as sfd:
            contents = sfd.read()

        if sfile_type == 'yaml':
            # AUDIT:no-member makes sense here due to ruamel.YAML/PyYAML usage
            # pylint: disable=no-member
            if hasattr(yaml, 'RoundTripLoader'):
                contents = yaml.load(contents, yaml.RoundTripLoader)
            else:
                contents = yaml.safe_load(contents)
        elif sfile_type == 'json':
            contents = json.loads(contents)

        return contents 

def test_get_account_with_stage_creds(self):
        os.environ["MY_USERNAME"] = "foo"
        os.environ["MY_APIKEY"] = "bar"
        config = yaml.safe_load(
            "stages:\n"
            "  sandwhich:\n"
            "    environment: {env_name}\n"
            "environments:\n"
            "  - name: {env_name}\n"
            '    account: "{account}"\n'
            "    rs_username_var: MY_USERNAME\n"
            "    rs_apikey_var: MY_APIKEY".format(
                env_name=self.environment, account=self.account
            )
        )
        account, role, username, apikey = run.get_account(config, None, "sandwhich")
        del os.environ["MY_USERNAME"]
        del os.environ["MY_APIKEY"]
        self.assertEqual(account, self.account)
        self.assertIsNone(role)
        self.assertEqual(username, "foo")
        self.assertEqual(apikey, "bar") 

def test_get_account_with_stage_creds_2(self):
        os.environ["MY_USERNAME"] = "foo"
        os.environ["MY_APIKEY"] = "bar"
        config = yaml.safe_load(
            "stages:\n"
            "  /.*/:\n"
            "    environment: {env_name}\n"
            "environments:\n"
            "  - name: {env_name}\n"
            '    account: "{account}"\n'
            "    rs_username_var: MY_USERNAME\n"
            "    rs_apikey_var: MY_APIKEY".format(
                env_name=self.environment, account=self.account
            )
        )
        account, role, username, apikey = run.get_account(
            config, None, "made-up-nonsense"
        )
        del os.environ["MY_USERNAME"]
        del os.environ["MY_APIKEY"]
        self.assertEqual(account, self.account)
        self.assertIsNone(role)
        self.assertEqual(username, "foo")
        self.assertEqual(apikey, "bar") 

def test_import_json_config(self, *args):
        stdin = sys.stdin
        sys.stdin = StringIO(test_json_config)
        config.main(["-c", TEST_CONFIG, "import"])
        sys.stdin = stdin

        with open(TEST_CONFIG, "rt") as f:
            data = yaml.safe_load(f.read())
        self.assertEqual(
            data,
            {
                "stages": {
                    "/.*/": {"environment": "dev", "key": "dev-key"},
                    "prod": {"environment": "prod", "key": "prod-key"},
                },
                "config": {
                    "foo": "bar",
                    "password": {
                        "+dev": ":decrypt:ZGV2OmRldi1wYXNzd29yZA==",
                        "+prod": ":decrypt:cHJvZDpwcm9kLXBhc3N3b3Jk",
                    },
                    "nest": {"bird": "pigeon", "tree": "birch"},
                },
            },
        ) 

def test_render_yaml_config(self, *args):
        stdout = sys.stdout
        sys.stdout = StringIO()
        with open(TEST_CONFIG, "wt") as f:
            f.write(test_config_file)
        config.main(["-c", TEST_CONFIG, "render", "dev"])
        sys.stdout.seek(0)
        data = sys.stdout.read()
        sys.stdout = stdout
        self.assertEqual(
            yaml.safe_load(data),
            {
                "foo": "bar",
                "password": "dev-password",
                "nest": {"bird": "pigeon", "tree": "birch"},
            },
        ) 

def setup_logging(
        default_path='./parameter/logger.yml',
        default_level=logging.INFO,
        env_key='LOG_CFG'
):
    """Setup logging configuration

    """
    path = os.path.abspath(default_path)
    value = os.getenv(env_key, None)
    if value:
        path = value
    if os.path.exists(os.path.abspath(path)):
        with open(path, 'rt') as f:
            config = yaml.safe_load(f.read())
        logging.config.dictConfig(config)
    else:
        logging.basicConfig(level=default_level) 

def test_render_yaml_config_custom(self, *args):
        stdout = sys.stdout
        sys.stdout = StringIO()
        with open(TEST_CONFIG, "wt") as f:
            f.write(test_config_file)
        config.main(["-c", TEST_CONFIG, "render", "foo"])
        sys.stdout.seek(0)
        data = sys.stdout.read()
        sys.stdout = stdout
        self.assertEqual(
            yaml.safe_load(data),
            {
                "foo": "bar",
                "password": "foo-password",
                "nest": {"bird": "pigeon", "tree": "birch"},
            },
        ) 

def get_config(fname):
    config=None
    with open(fname) as f:
        config=yaml.safe_load(f)
    return config 

def generate_name(event, physical_resource_id):
    manifest = event['ResourceProperties']['Manifest']
    if type(manifest) == str:
        manifest = yaml.safe_load(manifest)
    stack_name = event['StackId'].split('/')[1]
    if "metadata" in manifest.keys():
        if 'name' not in manifest["metadata"].keys() and 'generateName' not in manifest["metadata"].keys():
            if physical_resource_id:
                manifest["metadata"]["name"] = physical_resource_id.split('/')[-1]
            else:
                manifest["metadata"]["generateName"] = "cfn-%s-" % stack_name.lower()
    return manifest 

def load_yaml(filename):
    """Return object in yaml file."""
    with open(filename) as myfile:
        content = myfile.read()
        if "win" in sys.platform:
            content = content.replace("\\", "/")
        return yaml.safe_load(content) 

def handler_init(event):
    logger.debug('Received event: %s' % json.dumps(event, default=json_serial))

    physical_resource_id = None
    manifest_file = None
    create_kubeconfig(event['ResourceProperties']['ClusterName'])
    if 'HttpProxy' in event['ResourceProperties'].keys() and event['RequestType'] != 'Delete':
        enable_proxy(event['ResourceProperties']['HttpProxy'], event['ResourceProperties']['VpcId'])
    if 'Manifest' in event['ResourceProperties'].keys():
        manifest_file = '/tmp/manifest.json'
        if "PhysicalResourceId" in event.keys():
            physical_resource_id = event["PhysicalResourceId"]
        if type(event['ResourceProperties']['Manifest']) == str:
            manifest = generate_name(event, physical_resource_id)
        else:
            manifest = fix_types(generate_name(event, physical_resource_id))
        write_manifest(manifest, manifest_file)
        logger.debug("Applying manifest: %s" % json.dumps(manifest, default=json_serial))
    elif 'Url' in event['ResourceProperties'].keys():
        manifest_file = '/tmp/manifest.json'
        url = event['ResourceProperties']["Url"]
        if re.match(s3_scheme, url):
            response = s3_get(url)
        else:
            response = http_get(url)
        manifest = yaml.safe_load(response)
        write_manifest(manifest, manifest_file)
    return physical_resource_id, manifest_file 

def get_flag(config):
    with open(config, 'r') as f:
        conf = yaml.safe_load(f)
    return conf['flag']
##
## @brief      { function_description }
## 

def _read_config_file(args):
    """Decrypt config file, returns a tuple with stages and config."""
    stage = args.stage
    with open(args.config, "rt") as f:
        config = yaml.safe_load(f.read())
    STATE["stages"] = config["stages"]
    config["config"] = _decrypt_item(config["config"], stage=stage, key="", render=True)
    return config["stages"], config["config"] 

def read_yaml_cases(config, open):
    for case in yaml.safe_load(config)['cases']:
        if 'judge' not in case:
            yield DefaultCase(partial(open, case['input']),
                              partial(open, case['output']),
                              parse_time_ns(case['time']),
                              parse_memory_bytes(case['memory']),
                              int(case['score']))
        else:
            yield CustomJudgeCase(partial(open, case['input']),
                                  parse_time_ns(case['time']),
                                  parse_memory_bytes(case['memory']),
                                  partial(open, case['judge']),
                                  path.splitext(case['judge'])[1][1:]) 

def set_to_default(self):
        """ Overwrite the configuration with the default configuration. """
        self._config = yaml.safe_load(self.default()) 

def import_from_yaml(text: str=None, filepath: str=None, *, ignore_schema: bool=False, ignore_validation: bool=False) -> Statechart:
    """
    Import a statechart from a YAML representation (first argument) or a YAML file (filepath argument).

    Unless specified, the structure contained in the YAML is validated against a predefined
    schema (see *sismic.io.SCHEMA*), and the resulting statechart is validated using its *validate()* method.

    :param text: A YAML text. If not provided, filepath argument has to be provided.
    :param filepath: A path to a YAML file.
    :param ignore_schema: set to *True* to disable yaml validation.
    :param ignore_validation: set to *True* to disable statechart validation.
    :return: a *Statechart* instance
    """
    if not text and not filepath:
        raise TypeError('A YAML must be provided, either using first argument or filepath argument.')
    elif text and filepath:
        raise TypeError('Either provide first argument or filepath argument, not both.')
    elif filepath:
        with open(filepath, 'r') as f:
            text = f.read()

    if yaml.version_info < (0, 15):
        data = yaml.safe_load(text)  # type: dict
    else:
        yml = yaml.YAML(typ='safe', pure=True)
        data = yml.load(text)

    if not ignore_schema:
        try:
            data = schema.Schema(SCHEMA.statechart).validate(data)
        except schema.SchemaError as e:
            raise StatechartError('YAML validation failed') from e

    sc = import_from_dict(data)

    if not ignore_validation:
        sc.validate()
    return sc 

def test_get_config(self):
        mock_open = mock.mock_open(read_data=self.config)
        with mock.patch("fleece.cli.run.run.open", mock_open, create=True):
            config = run.get_config("./wat")

        self.assertDictEqual(yaml.safe_load(self.config), config) 

def test_export_yaml_config(self, *args):
        stdout = sys.stdout
        sys.stdout = StringIO()
        with open(TEST_CONFIG, "wt") as f:
            f.write(test_config_file)
        config.main(["-c", TEST_CONFIG, "export"])
        sys.stdout.seek(0)
        data = sys.stdout.read()
        sys.stdout = stdout
        self.assertEqual(
            yaml.safe_load(data),
            {
                "stages": {
                    "/.*/": {"environment": "dev", "key": "dev-key"},
                    "prod": {"environment": "prod", "key": "prod-key"},
                },
                "config": {
                    "foo": "bar",
                    "password": {
                        "+dev": ":encrypt:dev-password",
                        "+prod": ":encrypt:prod-password",
                        "+foo": ":encrypt:foo-password",
                        "+/ba.*/": ":encrypt:bar-password",
                    },
                    "nest": {"bird": "pigeon", "tree": "birch"},
                },
            },
        ) 

def test_get_account(self):
        config = yaml.safe_load(self.config)
        account, role, username, apikey = run.get_account(config, self.environment)
        self.assertEqual(account, self.account)
        self.assertIsNone(role)
        self.assertIsNone(username)
        self.assertIsNone(apikey) 

def test_import_yaml_config(self, *args):
        stdin = sys.stdin
        sys.stdin = StringIO(test_yaml_config)
        config.main(["-c", TEST_CONFIG, "import"])
        sys.stdin = stdin

        with open(TEST_CONFIG, "rt") as f:
            data = yaml.safe_load(f.read())
        self.assertEqual(
            data,
            {
                "stages": {
                    "/.*/": {"environment": "dev", "key": "dev-key"},
                    "prod": {"environment": "prod", "key": "prod-key"},
                },
                "config": {
                    "foo": "bar",
                    "password": {
                        "+dev": ":decrypt:ZGV2OmRldi1wYXNzd29yZA==",
                        "+prod": ":decrypt:cHJvZDpwcm9kLXBhc3N3b3Jk",
                        "+foo": ":decrypt:Zm9vOmZvby1wYXNzd29yZA==",
                        "+/ba.*/": ":decrypt:L2JhLiovOmJhci1wYXNzd29yZA==",
                    },
                    "nest": {"bird": "pigeon", "tree": "birch"},
                },
            },
        ) 

def render(self, **variables):
        # Render variables into the yolo file.
        template = jinja2.Template(
            yaml.dump(self._raw_content, Dumper=yaml.RoundTripDumper)
        )
        rendered_content = template.render(**variables)
        new_content = yaml.safe_load(rendered_content)
        return YoloFile(new_content)