Python yaml.load_all() Examples

def read_yaml_file(path):
        """Open the yaml file and load it to the variable.
        Return created list"""
        if path == 'config.yml':
            wrn = "Use 'load_config' or 'ATCConfig' instead for config"
            # Warning will not show, 
            # unless captured by logging facility or python called with -Wd
            warnings.warn(message=wrn,
                          category=DeprecationWarning)
            return ATCConfig(path).config

        with open(path) as f:
            yaml_fields = yaml.load_all(f.read(), Loader=yaml.FullLoader)

        buff_results = [x for x in yaml_fields]
        if len(buff_results) > 1:
            result = buff_results[0]
            result['additions'] = buff_results[1:]
        else:
            result = buff_results[0]
        return result 

def parse_yaml(ioc_filename):
    print(banner2)
    print("YAML FILE: {}".format(ioc_filename))
    try:
        raw_iocs = yaml.load_all(open(ioc_filename, 'r').read())
        start_log("Adversarial Simulation", "1.0")

        for raw_ioc in raw_iocs:

            scenario = raw_ioc.get('meta').get('scenario')
            purple = raw_ioc.get('meta').get('purple_actions')
            # if we cant find the scenario tag, default to run_uuid
            if not scenario:
                run_uuid(ioc_filename)
            # if the scenario field is found and if it's true run the run_scenario function
            if scenario is True:
                run_scenario(ioc_filename)
        close_log()

    except KeyboardInterrupt:
        print("CTRL-C received, exiting...")

    except Exception as e:
        print(e) 

def load(self, ymlfile=None):
        """Load and process the YAML file"""
        if ymlfile is not None:
            self.ymlfile = ymlfile

        try:
            # If yaml should be 'cleaned' of document references
            if self._clean:
                self.data = self.process(self.ymlfile)
            else:
                with open(self.ymlfile, 'rb') as stream:
                    for data in yaml.load_all(stream, Loader=yaml.Loader):
                        self.data.append(data)

            self.loaded = True
        except ScannerError as e:
            msg = "YAML formattting error - '" + self.ymlfile + ": '" + str(e) + "'"
            raise util.YAMLError(msg) 

def get_deployer_notes(stig_id):
    """Read deployer notes based on the STIG ID."""
    filename = "{0}/rhel7/{1}.rst".format(METADATA_DIR, stig_id)

    # Does this deployer note exist?
    if not os.path.isfile(filename):
        return False

    # Read the note and parse it with YAML
    with open(filename, 'r') as f:
        rst_file = f.read()

    # Split the RST into frontmatter and text
    # NOTE(mhayden): Can't use the standard yaml.load_all() here at it will
    #                have scanner errors in documents that have colons (:).
    yaml_boundary = re.compile(r'^-{3,}$', re.MULTILINE)
    _, metadata, text = yaml_boundary.split(rst_file, 2)

    # Assemble the metadata and the text from the deployer note.
    post = yaml.safe_load(metadata)
    post['content'] = text

    return post 

def run_scenario(ioc_filename):
    try:
        print("### Running the Scenario ###")
        raw_iocs = yaml.load_all(open(ioc_filename, 'r').read())

        timenow = datetime.datetime.utcnow()

        for raw_ioc in raw_iocs:
            scenario = raw_ioc.get('meta').get('scenario_actions')
            rule_name = raw_ioc.get('name')
            print("### {} ###".format(rule_name))

            scenario_actions = []
            # read the steps from purple_actions in yaml and load them into purple_actions
            for x in range(1, len(scenario)+1):
                scenario_actions.append(raw_ioc.get('meta').get('scenario_actions').get(x))

            for uuid_file in scenario_actions:
                run_uuid(uuid_file)

    except Exception as e:
        print(e) 

def lolbas(name: str):
    """Search binaries from LOLBAS within command line

    Arguments:
        name {[type]} -- Name of the exe to get info about

    Keyword Arguments:
        cmd {str} -- get only the code section (default: {False})
    """

    exes = get_exe()
    if name in exes.keys():
        url = RAW_URL + exes[name] + '.md'
        r = requests.get(url).text
        data = list(yaml.load_all(r, Loader=yaml.SafeLoader))[0]
        parse(data)
    else:
        print(colors("[!] Binary not found on LOLBAS", 91))
        #TODO: Match user input and make suggestion for search
        print(colors("[!] Make sure to provide name with proper extension", 91)) 

def gtfobins(bin_name: str):
    """Search binaries from GTFOBins within command line

    Arguments:
        bin_name {[type]} -- Name of the binary to get info about

    """

    bins = get_bins()

    if bin_name in bins:
        r = requests.get(RAW_URL.format(bin_name)).text
        data = list(yaml.load_all(r, Loader=yaml.SafeLoader))[0]

        parse(data)
    else:
        print(colors("[!] Binary not found on GTFObins: ", 91)) 

def load_module(self, fullname):
        mod = super(YamlLoader, self).load_module(fullname)

        with codecs.open(self.cfg_file, 'r', 'utf-8') as f:
            try:
                for doc in yaml.load_all(f, yaml.Loader):
                    if isinstance(doc, dict):
                        mod.__dict__.update(doc)
            except yaml.YAMLError:
                self.e = "YAMLError"
                self.err_msg = sys.exc_info()[1]

        if self.e == "YAMLError":
            err_msg = "\nYaml file not valid: "
            err_msg += self.cfg_file + '\n'
            err_msg += str(self.err_msg)
            raise InvalidYamlError(err_msg)

        return mod 

def load_info(path, qt_parent=None):
    path_base, path_extension = _ospath.splitext(path)
    filename = path_base + ".yaml"
    try:
        with open(filename, "r") as info_file:
            info = list(_yaml.load_all(info_file, Loader=_yaml.FullLoader))
    except FileNotFoundError as e:
        print(
            "\nAn error occured. Could not find metadata file:\n{}".format(
                filename
            )
        )
        if qt_parent is not None:
            _QMessageBox.critical(
                qt_parent,
                "An error occured",
                "Could not find metadata file:\n{}".format(filename),
            )
        raise NoMetadataFileError(e)
    return info 

def parse_kubeconfig():
        if not os.path.exists(os.path.expanduser(kubeconfig_filepath)):
            return ("", "", "")

        with open(os.path.expanduser(kubeconfig_filepath), "r") as fd:
            docs = yaml.load_all(fd)
            for doc in docs:
                current_context = doc.get("current-context", "")
                contexts = doc.get("contexts")
                if contexts:
                    for index, context in enumerate(contexts):
                        if context['name'] == current_context:
                            KubeConfig.current_context_index = index
                            KubeConfig.current_context_name = context['name']
                            if 'cluster' in context['context']:
                                KubeConfig.clustername = context['context']['cluster']
                            if 'namespace' in context['context']:
                                KubeConfig.namespace = context['context']['namespace']
                            if 'user' in context['context']:
                                KubeConfig.user = context['context']['user']
                            return (KubeConfig.clustername, KubeConfig.user, KubeConfig.namespace)
        return ("", "", "") 

def loads(filename):
    """
    Load the given YAML string
    """
    template = None
    template_temp = None
    with open(filename, 'r') as fp:
        content = fp.read()


        content = "[" + content + "]"
        content = content.replace('}{', '},{')
        content = content.replace('}\n{', '},\n{')

        template_temp = list(yaml.load_all(content, Loader=SafeLineLoader))

    # Convert an empty file to an empty dict
    if template_temp is None:
        template = {}
    else:
        template = template_temp[0]

    return template 

def get_chart_templates(
            self, template_name, name, release_name, namespace, chart,
            disable_hooks, values):
        # returns some info

        LOG.info("Template( %s ) : %s ", template_name, name)

        stub = ReleaseServiceStub(self.channel)
        release_request = InstallReleaseRequest(
            chart=chart,
            values=values,
            name=name,
            namespace=namespace,
            wait=False)

        templates = stub.InstallRelease(
            release_request, self.timeout, metadata=self.metadata)

        for template in yaml.load_all(getattr(templates.release, 'manifest',
                                              [])):
            if template_name == template.get('metadata', None).get('name',
                                                                   None):
                LOG.info(template_name)
                return template 

def load_yaml(file_name):
    """Load the contents of a YAML file.

    :param file_name: File name
    :type file_name: str or unicode
    :return: content (list of dictionaries, one per YAML document
    in the file) or None if file is empty
    :rtype: list of dict
    :raises IOError: if the file is not found
    :raises FileContentError: if there is a problem parsing YAML
    """
    with open(file_name, 'r') as f:
        try:
            content = [data for data in yaml.load_all(f)]
        except yaml.YAMLError as e:
            raise FileContentError(file_name, e)
    return content 

def load_camera_from_calibr(f):
    s = open(f).read()

    y = list(yaml.load_all(s))[0]

    K0 = Camera.buildK(y['cam0']['intrinsics'])
    C0 = Camera(K0, np.eye(3), np.zeros((3,)))

    K1 = Camera.buildK(y['cam1']['intrinsics'])
    M = y['cam1']['T_cn_cnm1'][:3]
    R1 = np.asarray([k[:3] for k in M])
    t1 = np.asarray([k[3] for k in M])
    C1 = Camera(K1, R1, t1)

    dist0 = np.asarray(y['cam0']['distortion_coeffs'])
    dist1 = np.array(y['cam1']['distortion_coeffs'])
    return C0, C1, dist0, dist1 

def get_data(self):
        index = {'REQUEST_Headers': {}, 'RESPONSE_Headers': {}}
        doc = {}
        for yaml_file in self.files:
            f = open(yaml_file, encoding='utf-8')
            cont =f.read()
            y = yaml.load_all(cont)
            for api in y:
                if api.get('Name'):
                    doc[api['Name'].upper()] = api
                elif api.get('Title'):
                    index = api

        for name,api in doc.items():
            if not api['REQUEST'].get('Headers'):
                api['REQUEST']['Headers'] = {}
            api['REQUEST']['Headers'] = dict(index['REQUEST_Headers'], **api['REQUEST']['Headers'])
            if api['Method'] == 'GET':
                api['REQUEST']['Headers'].pop('Content-Type')

        return doc 

def get_chart_templates(self, template_name, name, release_name, namespace,
                            chart, disable_hooks, values):
        # returns some info

        LOG.info("Template( %s ) : %s ", template_name, name)

        stub = ReleaseServiceStub(self.channel)
        release_request = InstallReleaseRequest(
            chart=chart,
            dry_run=True,
            values=values,
            name=name,
            namespace=namespace,
            wait=False)

        templates = stub.InstallRelease(
            release_request, self.timeout, metadata=self.metadata)

        for template in yaml.load_all(
                getattr(templates.release, 'manifest', [])):
            if template_name == template.get('metadata', None).get(
                    'name', None):
                LOG.info(template_name)
                return template 

def test_update_dictionary_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '01')
        merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')

        with open(self.base_manifest) as f, open(expected) as e, open(
                merge) as m:
            merging_values = list(yaml.safe_load_all(m.read()))
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['chart', 'blog-1']
            ovr = Override(documents)
            ovr.update_document(merging_values)
            ovr_doc = ovr.find_manifest_document(doc_path)
            expect_doc = list(yaml.load_all(e.read()))[0]

            self.assertEqual(ovr_doc, expect_doc) 

def test_set_list_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '03')

        with open(self.base_manifest) as f, open(expected) as e:
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['manifest', 'simple-armada']
            override = ('manifest:simple-armada:chart_groups=\
                         blog-group3,blog-group4',)
            ovr = Override(documents, override)
            ovr.update_manifests()
            ovr_doc = ovr.find_manifest_document(doc_path)
            target_docs = list(yaml.load_all(e.read()))
            expected_doc = [x
                            for x
                            in target_docs
                            if x.get('schema') == 'armada/Manifest/v1'][0]
            self.assertEqual(expected_doc.get('data'), ovr_doc.get('data')) 

def test_update_dictionary_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '01')
        merge = "{}/templates/override-{}.yaml".format(self.basepath, '01')

        with open(self.base_manifest) as f, open(expected) as e, open(
                merge) as m:
            merging_values = list(yaml.safe_load_all(m.read()))
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['chart', 'blog-1']
            ovr = Override(documents)
            ovr.update_documents(merging_values)
            ovr_doc = ovr.find_manifest_document(doc_path)
            expect_doc = list(yaml.load_all(e.read()))[0]

            self.assertEqual(ovr_doc, expect_doc) 

def test_set_list_valid(self):
        expected = "{}/templates/override-{}-expected.yaml".format(
            self.basepath, '03')

        with open(self.base_manifest) as f, open(expected) as e:
            documents = list(yaml.safe_load_all(f.read()))
            doc_path = ['manifest', 'simple-armada']
            override = (
                'manifest:simple-armada:chart_groups=\
                         blog-group3,blog-group4', )
            ovr = Override(documents, override)
            ovr.update_manifests()
            ovr_doc = ovr.find_manifest_document(doc_path)
            target_docs = list(yaml.load_all(e.read()))
            expected_doc = [
                x for x in target_docs
                if x.get('schema') == 'armada/Manifest/v1'
            ][0]
            self.assertEqual(expected_doc.get('data'), ovr_doc.get('data')) 

def create_new_app_yaml(source_file, app_name):
  yaml_spec = YamlSpec(source_file).to_file()
  with open(yaml_spec, 'r') as stream:
    try:
      yaml_content = list(yaml.load_all(stream))
    except yaml.YAMLError as exc:
      raise HokusaiError("Cannot read source yaml file %s." % source_file)

  for c in yaml_content: update_namespace(c, clean_string(app_name))

  new_namespace = OrderedDict([
      ('apiVersion', 'v1'),
      ('kind', 'Namespace'),
      ('metadata', {
        'name': clean_string(app_name)
      })
    ])
  yaml_content = [new_namespace] + yaml_content

  with open(os.path.join(CWD, HOKUSAI_CONFIG_DIR, "%s.yml" % app_name), 'w') as output:
    output.write(YAML_HEADER)
    yaml.safe_dump_all(yaml_content, output, default_flow_style=False)

  print_green("Created %s/%s.yml" % (HOKUSAI_CONFIG_DIR, app_name)) 

def helm_template(config):
    with tempfile.NamedTemporaryFile() as temp:
        with open(temp.name, "w") as values:
            values.write(config)
        helm_cmd = "helm template -f {0} ./".format(temp.name)
        result = yaml.load_all(check_output(helm_cmd.split()))

        results = {}
        for r in result:
            if r:
                kind = r["kind"].lower()
                if kind not in results:
                    results[kind] = {}
                results[kind][r["metadata"]["name"]] = r

        if os.environ.get("DEBUG"):
            print(json.dumps(results, indent=4, sort_keys=True))
        return results 

def load_file(path: str) -> List[object]:
    """Load an individual Kubernetes manifest YAML file.

    This file may contain multiple YAML documents. It will attempt to auto-detect
    the type of each object to load.

    Args:
        path: The fully qualified path to the file.

    Returns:
        A list of the Kubernetes API objects for this manifest file.
    """
    with open(path, 'r') as f:
        manifests = yaml.load_all(f, Loader=yaml.SafeLoader)

        objs = []
        for manifest in manifests:
            obj_type = get_type(manifest)
            if obj_type is None:
                raise ValueError(
                    f'Unable to determine object type for manifest: {manifest}',
                )
            objs.append(new_object(obj_type, manifest))

    return objs 

def _pyyaml(f):
    import yaml
    try:
        Loader = yaml.CSafeLoader
    except AttributeError:  # System does not have libyaml
        Loader = yaml.SafeLoader
    return list(yaml.load_all(f, Loader=Loader)) 

def _ruamel(f):
    from ruamel.yaml import YAML
    yaml = YAML(typ='safe')
    return list(yaml.load_all(f)) 

def get_task_kwargs_list(config=None):
    """Get a list of dictionaries that are read from the flo.yaml
    file and collapse the global variables into each task.
    """

    # get workflow configuration file
    config_path = find_config_path(config=config)

    # load the data
    with open(config_path) as stream:
        config_yaml = yaml.load_all(stream.read())
    try:
        return config_yaml2task_kwargs_list(config_yaml)
    except yaml.constructor.ConstructorError, error:
        raise exceptions.YamlError(config_path, error) 

def import_calculations_from_file(filename, skip_incomplete_records=False):
    """
    Construct a list of :obj:`Calculation` objects by reading a YAML file.
    Each YAML document should include ``title``, ``stoichiometry``, and ``energy`` fields, e.g.::

        title: my calculation
        stoichiometry:
            - A: 1
            - B: 2
        energy: -0.1234 eV

    Separate calculations should be distinct YAML documents, separated by `---`
    
    Args:
        filename (str): Name of the YAML file to read.
        skip_incomplete_records (bool): Do not parse YAML documents missing one or more of
            the required keys. Default is ``False``.

    Returns:
        (dict(vasppy.Calculation)): A dictionary of :obj:`Calculation` objects. For each :obj:`Calculation` object, the ``title`` field from the YAML input is used as the dictionary key.
    """
    calcs = {}
    with open( filename, 'r' ) as stream:
        docs = yaml.load_all( stream, Loader=yaml.SafeLoader )
        for d in docs:
            if skip_incomplete_records:
                if ('title' not in d) or ('stoichiometry' not in d) or ('energy' not in d):
                    continue
            if 'stoichiometry' in d:
                stoichiometry = Counter()
                for s in d['stoichiometry']:
                    stoichiometry.update( s )
            else:
                raise ValueError('stoichiometry not found for "{d["title"]}"')
            calcs[ d['title'] ] = Calculation( title=d['title'], 
                                               stoichiometry=stoichiometry, 
                                               energy=energy_string_to_float( d['energy'] ) )
    return calcs 

def parse_new_snpsplit_log(self, f):
        data = next(yaml.load_all(f['f'], Loader=yaml.SafeLoader))
        flat_data = {}
        for k in data:
            for sk in data[k]:
                key = sk
                for prefix in ['PE_', 'SE_', 'HiC_']:
                    if sk.startswith(prefix):
                        key = sk[len(prefix):]
                flat_key = '{}_{}'.format(k.lower(), key)
                flat_data[flat_key] = data[k][sk]
        input_fn = data['Meta']['infile']
        return [input_fn, flat_data] 

def load_current_file():
	print("Loading current file: {}".format(current_file))
	global current_docs, current_doc
	try:
		with open(current_file, "r", encoding="UTF-8") as f:
			current_docs = list(yaml.load_all(f))
		current_doc = len(current_docs)
	except FileNotFoundError:
		pass
	except yaml.YAMLError:
		print("Failed to parse edit file") 

def load_hparam(filename):
    stream = open(filename, 'r')
    docs = yaml.load_all(stream, Loader=yaml.Loader)
    hparam_dict = dict()
    for doc in docs:
        for k, v in doc.items():
            hparam_dict[k] = v
    return hparam_dict