Python yaml.CLoader() Examples

def create_people(srcdir, clean=False):
    """Creates page stubs for all authors/editors in the Anthology."""
    log.info("Creating stubs for people...")
    if not check_directory("{}/content/people".format(srcdir), clean=clean):
        return

    for yamlfile in tqdm(glob("{}/data/people/*.yaml".format(srcdir))):
        log.debug("Processing {}".format(yamlfile))
        with open(yamlfile, "r") as f:
            data = yaml.load(f, Loader=Loader)
        # Create a page stub for each person
        for name, entry in data.items():
            person_dir = "{}/content/people/{}".format(srcdir, name[0])
            if not os.path.exists(person_dir):
                os.makedirs(person_dir)
            yaml_data = {"name": name, "title": entry["full"], "lastname": entry["last"]}
            with open("{}/{}.md".format(person_dir, name), "w") as f:
                print("---", file=f)
                # "lastname" is dumped to allow sorting by it in Hugo
                yaml.dump(yaml_data, default_flow_style=False, stream=f)
                print("---", file=f)

    return data 

def load_app_from_config(path):
    """
    Generate app directly from config file, bypassing command line settings (useful for testing in ipython)
    """
    Setting.generate_missing_shorthands()
    defaults = Setting.generate_defaults_dict()
    if osp.isfile(path):
        file_stream = open(path, "r", encoding="utf-8")
        config_defaults = load(file_stream, Loader=Loader)
        file_stream.close()
        for key, value in config_defaults.items():
            defaults[key] = value
    else:
        raise ValueError("Settings file not found at: {0:s}".format(path))
    args = ap.Namespace()
    for key, value in defaults.items():
        args.__dict__[key] = value
    if args.unsynced:
        app = ApplicationUnsynced(args)
    else:
        app = ApplicationSynced(args)
    return app 

def load_app_from_config(path):
    """
    Generate app directly from config file, bypassing command line settings (useful for testing in ipython)
    """
    Setting.generate_missing_shorthands()
    defaults = Setting.generate_defaults_dict()
    if osp.isfile(path):
        file_stream = open(path, "r", encoding="utf-8")
        config_defaults = load(file_stream, Loader=Loader)
        file_stream.close()
        for key, value in config_defaults.items():
            defaults[key] = value
    else:
        raise ValueError("Settings file not found at: {0:s}".format(path))
    args = ap.Namespace()
    for key, value in defaults.items():
        args.__dict__[key] = value
    if args.unsynced:
        app = ApplicationUnsynced(args)
    else:
        app = ApplicationSynced(args)
    return app 

def load_app_from_config(path):
    """
    Generate app directly from config file, bypassing command line settings (useful for testing in ipython)
    """
    Setting.generate_missing_shorthands()
    defaults = Setting.generate_defaults_dict()
    if osp.isfile(path):
        file_stream = open(path, "r", encoding="utf-8")
        config_defaults = load(file_stream, Loader=Loader)
        file_stream.close()
        for key, value in config_defaults.items():
            defaults[key] = value
    else:
        raise ValueError("Settings file not found at: {0:s}".format(path))
    args = ap.Namespace()
    for key, value in defaults.items():
        args.__dict__[key] = value

    app = StereoMatcherApp(args)

    return app 

def load_results_sixd17(path):
    """
    Loads 6D object pose estimates from a file.

    :param path: Path to a file with poses.
    :return: List of the loaded poses.
    """
    with open(path, "r") as f:
        res = yaml.load(f, Loader=yaml.CLoader)
        if not res["ests"] or res["ests"] == [{}]:
            res["ests"] = []
        else:
            for est in res["ests"]:
                est["R"] = np.array(est["R"]).reshape((3, 3))
                est["t"] = np.array(est["t"]).reshape((3, 1))
                if isinstance(est["score"], str):
                    if "nan" in est["score"]:
                        est["score"] = 0.0
                    else:
                        raise ValueError("Bad type of score.")
    return res 

def merge_archive():
    """
    merge all archive yaml files into one file
    """
    print("Creating archive YAML files")
    for name in ['stable_recovery', 'stable_fastboot', 'weekly_recovery', 'weekly_fastboot']:
        yaml_files = [x for x in sorted(glob(f'archive/{name}/*.yml'))
                      if not x.endswith('recovery.yml')
                      and not x.endswith('fastboot.yml')]
        yaml_data = []
        for file in yaml_files:
            with open(file, "r") as yaml_file:
                yaml_data.append(yaml.load(yaml_file, Loader=yaml.CLoader))
        with open(f'archive/{name}/{name}', "w") as output:
            yaml.dump(yaml_data, output, Dumper=yaml.CDumper)
        if path.exists(f'archive/{name}/{name}'):
            rename(f'archive/{name}/{name}', f'archive/{name}/{name}.yml') 

def archive(update: dict):
    """Append new update to the archive"""
    codename = update['codename']
    link = update['download']
    version = update['version']
    branch = get_branch(version)
    rom_type = 'recovery' if update['filename'].endswith('.zip') else 'fastboot'
    try:
        with open(f'archive/{branch}_{rom_type}/{codename}.yml', 'r') as yaml_file:
            data = yaml.load(yaml_file, Loader=yaml.CLoader)
            data[codename].update({version: link})
            data.update({codename: data[codename]})
            with open(f'archive/{branch}_{rom_type}/{codename}.yml', 'w') as output:
                yaml.dump(data, output, Dumper=yaml.CDumper)
    except FileNotFoundError:
        data = {codename: {version: link}}
        with open(f'archive/{branch}_{rom_type}/{codename}.yml', 'w') as output:
            yaml.dump(data, output, Dumper=yaml.CDumper) 

def __init__(self,filename=None):
        self.joint_p = []
        self.joint_v = []
        self.joint_t = []
        self.gripper_cmd = []
        self.gripper_t = []
        self.tform = {}
        self.world_t = []

        if not filename==None:
            stream = file(filename,'r')
            demo = yaml.load(stream,Loader=Loader)
            self.joint_p = demo.joint_p
            self.joint_v = demo.joint_v
            self.joint_t = demo.joint_t
            self.gripper_cmd = demo.gripper_cmd
            self.tform = demo.tform
            self.world_t = demo.world_t 

def get_yaml(self) -> Dict[Text, Any]:
        yaml_path = self.yaml_path
        if not self.yaml_path_exists:
            return {}
        cache = self.get_yaml_subcache()
        cache_data = cache.get() or None
        if cache_data:
            return cache_data
        try:
            cache.clear()
            with open(yaml_path, 'r') as fd:
                data = load(fd.read(), Loader=Loader)
            cache.set(data)
            return data
        except:  # nocv
            logger.debug(traceback.format_exc())
            return cache_data 

def parse_yaml_file(file: TextIOWrapper) -> Optional[dict]:
    """Parses a YAML file and returns a nested dictionary containing its contents.

    :param file: Handle of file to parse
    :return: Parsed file contents"""
    try:
        # Parses the YAML file into a dict
        return load(file, Loader=Loader)
    except YAMLError as exc:
        logging.critical("Could not parse YAML file %s", file.name)
        if hasattr(exc, 'problem_mark'):
            # Tell user exactly where the syntax error is
            mark = exc.problem_mark
            logging.error("Error position: (%s:%s)",
                          mark.line + 1, mark.column + 1)
        else:
            logging.error("Error: %s", exc)
            return None


# PyYAML Reference: http://pyyaml.org/wiki/PyYAMLDocumentation 

def is_roll_back(update: dict):
    """
    check if the new update is actually a rollback one
    """
    codename = update['codename']
    version = update['version']
    filename = update['filename']
    branch = get_branch(version)
    rom_type = get_type(filename).lower()
    try:
        with open(f'archive/{branch}_{rom_type}/{codename}.yml', 'r') as yaml_file:
            data = yaml.load(yaml_file, Loader=yaml.CLoader)
            versions = list(data[codename].keys())
        return bool(version in versions)
    except FileNotFoundError:
        return False 

def parse_yaml(filename: str) -> Optional[dict]:
    """Parses a YAML file and returns a nested dictionary containing its contents.

    :param filename: Name of YAML file to parse
    :return: Parsed file contents"""
    try:
        # Enables use of stdin if '-' is specified
        with sys.stdin if filename == '-' else open(filename) as f:
            try:
                # Parses the YAML file into a dict
                return load(f, Loader=Loader)
            except YAMLError as exc:
                logging.critical("Could not parse YAML file %s", filename)
                if hasattr(exc, 'problem_mark'):
                    # Tell user exactly where the syntax error is
                    mark = exc.problem_mark
                    logging.error("Error position: (%s:%s)",
                                  mark.line + 1, mark.column + 1)
                else:
                    logging.error("Error: %s", exc)
                    return None
    except FileNotFoundError:
        logging.critical("Could not find YAML file for parsing: %s", filename)
        return None 

def loads(self, s, force_document_comment_to_first_entry=False, with_comments=False):
        if not with_comments:  # don't care about first comment if we don't care about coments globaly
            force_document_comment_to_first_entry = False
        if force_document_comment_to_first_entry:
            s = '____useless_property: true\n' + s
        yamllib = self.get_yaml_lib(with_comments)
        if with_comments:
            data = yamllib.round_trip_load(s)
        else:
            # The yaml lib do not manage loads, so need to fake it first
            StringIO = libstore.get_StringIO_unicode_compatible()
            fake_file = StringIO(s)  # unicode_to_bytes(s))
            if simple_yaml_loader is not None:
                data = yamllib.load(fake_file, Loader=simple_yaml_loader)
            else:
                data = yamllib.round_trip_load(s)
        if force_document_comment_to_first_entry:
            del data['____useless_property']
        return data 

def initialize():
    """
    Initial loading and preparing
    """
    remove("create_flashable_firmware.py")
    axel("https://raw.githubusercontent.com/XiaomiFirmwareUpdater/xiaomi-flashable-firmware-creator.py/py/xiaomi_flashable_firmware_creator/create_flashable_firmware.py")
    with open('devices/stable_devices.yml', 'r') as stable_json:
        stable_devices = yaml.load(stable_json, Loader=yaml.CLoader)
    open('log', 'w').close()
    all_stable = yaml.load(get(
        "https://raw.githubusercontent.com/XiaomiFirmwareUpdater/miui-updates-tracker/master/" +
        "stable_recovery/stable_recovery.yml").text, Loader=yaml.CLoader)
    names = yaml.load(get(
        "https://raw.githubusercontent.com/XiaomiFirmwareUpdater/miui-updates-tracker/master/" +
        "devices/names.yml").text, Loader=yaml.CLoader)
    return stable_devices, all_stable, names 

def diff(name: str) -> list:
    """
    compare yaml files
    """
    changes = []
    try:
        with open(f'{name}.yml', 'r') as new, \
                open(f'{name}_old.yml', 'r') as old_data:
            latest = yaml.load(new, Loader=yaml.CLoader)
            old = yaml.load(old_data, Loader=yaml.CLoader)
            first_run = False
    except FileNotFoundError:
        print(f"Can't find old {name} files, skipping")
        first_run = True
    if first_run is False:
        if len(latest) == len(old):
            codenames = list(latest.keys())
            for codename in codenames:
                if not latest[codename] == old[codename]:
                    changes.append({codename: latest[codename]})
        else:
            new_codenames = [i for i in list(latest.keys()) if i not in list(old.keys())]
            for codename in new_codenames:
                changes.append({codename: latest[codename]})
    return changes 

def get_config(infile):
    """
    read in the contents of the inputs yaml file
    """

    with open(infile, "r") as fp:
        config = load(fp, Loader=Loader)
    try:
        v_pen = tuple(config["v_pen"])
    except TypeError:
        v_pen = (config["v_pen"],)

    try:
        w_pen = tuple(config["w_pen"])
    except TypeError:
        w_pen = (config["w_pen"],)

    return v_pen, w_pen, config 

def create_sigs(srcdir, clean=False):
    """Creates page stubs for all SIGs in the Anthology."""
    yamlfile = "{}/data/sigs.yaml".format(srcdir)
    log.debug("Processing {}".format(yamlfile))
    with open(yamlfile, "r") as f:
        data = yaml.load(f, Loader=Loader)

    log.info("Creating stubs for SIGs...")
    if not check_directory("{}/content/sigs".format(srcdir), clean=clean):
        return
    # Create a paper stub for each SIGS (e.g. SIGMORPHON)
    for sig, sig_data in data.items():
        sig_str = sig_data["slug"]
        with open("{}/content/sigs/{}.md".format(srcdir, sig_str), "w") as f:
            print("---", file=f)
            yaml.dump(
                {
                    "acronym": sig,
                    "short_acronym": sig[3:] if sig.startswith("SIG") else sig,
                    "title": sig_data["name"],
                },
                default_flow_style=False,
                stream=f,
            )
            print("---", file=f) 

def _load_month_from_disk(path, year_number, month_number):
    """
    Load the month file at path and return a month object

    If an error occurs, return None
    """
    try:
        # Try to read the contents of the file.
        with codecs.open(path, "rb", encoding="utf-8") as month_file:
            logging.debug('Loading file "%s"' % path)
            month_contents = yaml.load(month_file, Loader=Loader)
            month = Month(
                year_number, month_number, month_contents, os.path.getmtime(path)
            )
            return month
    except yaml.YAMLError as exc:
        logging.error("Error in file {}:\n{}".format(path, exc))
    except OSError:
        # If that fails, there is nothing to load, so just display an error message.
        logging.error("Error: The file %s could not be read" % path)
    except Exception:
        logging.error("An error occured while reading %s:" % path)
        raise
    # If we continued here, the possibly corrupted file would be overwritten.
    sys.exit(1) 

def create_papers(srcdir, clean=False):
    """Creates page stubs for all papers in the Anthology."""
    log.info("Creating stubs for papers...")
    if not check_directory("{}/content/papers".format(srcdir), clean=clean):
        return

    # Go through all paper volumes
    for yamlfile in tqdm(glob("{}/data/papers/*.yaml".format(srcdir))):
        log.debug("Processing {}".format(yamlfile))
        with open(yamlfile, "r") as f:
            data = yaml.load(f, Loader=Loader)
        # Create a paper stub for each entry in the volume
        for anthology_id, entry in data.items():
            paper_dir = "{}/content/papers/{}".format(srcdir, anthology_id.split("-")[0])
            if not os.path.exists(paper_dir):
                os.makedirs(paper_dir)
            with open("{}/{}.md".format(paper_dir, anthology_id), "w") as f:
                print("---", file=f)
                yaml.dump(
                    {"anthology_id": anthology_id, "title": entry["title"]},
                    default_flow_style=False,
                    stream=f,
                )
                print("---", file=f) 

def _download_results(config, _blob_client, out_path, count, ptrn=FOLD_FILE_PATTERN):

    pathlib.Path(config.BATCH_DIRECTORY).mkdir(parents=True, exist_ok=True)
    blob_names = [b.name for b in _blob_client.list_blobs(config.CONTAINER_NAME)]

    results = []
    for i in range(count):
        blob_name = ptrn.format(i)
        if not blob_name in blob_names:
            raise RuntimeError("incomplete blob set: missing blob {}".format(blob_name))
        out_path = os.path.join(config.BATCH_DIRECTORY, blob_name)
        with _blob_client.get_blob_to_stream(
            config.CONTAINER_NAME, blob_name, out_path
        ) as blob:
            results[i] = load(blob, Loader=Loader)
    return results 

def load_yaml(config_file):
  """
  Attempt to load yaml configuration from an opened file
  """
  import yaml
  try:
    from yaml import CLoader as Loader
  except ImportError:
    from yaml import Loader
  out = yaml.load(config_file, Loader=Loader)
  if out is None:
    return {}
  return out 

def load_yaml(path):
    with open(path, "r") as f:
        content = yaml.load(f, Loader=yaml.CLoader)
        return content 

def load_labels(fpath, lang):
    with open(fpath, 'r', encoding="UTF-8") as f:
        labels = load(f, Loader=Loader)
        return labels[lang] 

def load_yaml(path):
    with open(path) as f:
        return yaml.load(f.read(), Loader=Loader) 

def iter_yaml_lz4_reports(fn):
    """Iterate YAML reports from a lz4 file
    """
    assert str(fn).endswith("lz4")

    fd = lz4frame.open(fn)
    blobgen = stream_yaml_blobs(fd)

    off, header = next(blobgen)
    headsha = hashlib.sha1(header)
    # XXX: bad header kills whole bucket
    header = yaml.load(header, Loader=CLoader)
    if not header.get("report_id"):
        header["report_id"] = generate_report_id(header)

    for off, entry in blobgen:
        entry_len = len(entry)
        esha = headsha.copy()
        esha.update(entry)
        esha = esha.digest()
        try:
            entry = yaml.load(entry, Loader=CLoader)
            if not entry:  # e.g. '---\nnull\n...\n'
                continue
            if "test_start_time" in entry and "test_start_time" in header:
                header.pop("test_start_time")
            entry.update(header)
            yield off, entry_len, esha, entry, None
        except Exception as exc:
            yield off, entry_len, esha, None, exc

    fd.close()


## Entry points 

def parse_file(cls, path):
        try:
            if hasattr(path, 'read') and callable(path.read):
                # File-like obj
                conf = yaml.load(path.read(), Loader=_Loader)
            else:
                with io.open(path) as f:
                    conf = yaml.load(f.read(), Loader=_Loader)
        except yaml.error.MarkedYAMLError as e:
            raise InvalidSpecification(str(e))

        return cls.parse(conf) 

def load_gt(path):
    with open(path, "r") as f:
        gts = yaml.load(f, Loader=yaml.CLoader)
        for im_id, gts_im in gts.items():
            for gt in gts_im:
                if "cam_R_m2c" in gt.keys():
                    gt["cam_R_m2c"] = np.array(gt["cam_R_m2c"]).reshape((3, 3))
                if "cam_t_m2c" in gt.keys():
                    gt["cam_t_m2c"] = np.array(gt["cam_t_m2c"]).reshape((3, 1))
    return gts 

def load(self, formatter=None):
        try:
            fd = open(self.filepath, 'r')
        except IOError:
            if self.strict is True:
                raise
            else:
                return

        _, file_extension = os.path.splitext(self.filepath)

        if file_extension.lower() in JSON_EXTENSIONS:
            import json
            self.data = dict((self.format(k, formatter), v) for k, v in json.load(fd).items())
        elif file_extension.lower() in YAML_EXTENSIONS:
            from yaml import load as yload, dump as ydump
            try:
                from yaml import CLoader as Loader
            except ImportError:
                from yaml import Loader
            self.data = dict((self.format(k, formatter), v) for k, v in yload(fd, Loader=Loader).items())
        elif file_extension.lower() in PYTHON_EXTENSIONS:
            mod = imp.load_source('mod', self.filepath)
            self.data = dict((self.format(k, formatter), v) for k, v in vars(mod).items() if k.isupper())
        else:
            raise ValueError("Unhandled file extension {0}".format(file_extension))

        fd.close() 

def load_info(path):
    with open(path, "r") as f:
        info = yaml.load(f, Loader=yaml.CLoader)
        for eid in info.keys():
            if "cam_K" in info[eid].keys():
                info[eid]["cam_K"] = np.array(info[eid]["cam_K"]).reshape((3, 3))
            if "cam_R_w2c" in info[eid].keys():
                info[eid]["cam_R_w2c"] = np.array(info[eid]["cam_R_w2c"]).reshape((3, 3))
            if "cam_t_w2c" in info[eid].keys():
                info[eid]["cam_t_w2c"] = np.array(info[eid]["cam_t_w2c"]).reshape((3, 1))
    return info 

def load_cam_params(path):
    with open(path, "r") as f:
        c = yaml.load(f, Loader=yaml.CLoader)
    cam = {
        "im_size": (c["width"], c["height"]),
        "K": np.array([[c["fx"], 0.0, c["cx"]], [0.0, c["fy"], c["cy"]], [0.0, 0.0, 1.0]]),
    }
    if "depth_scale" in c.keys():
        cam["depth_scale"] = float(c["depth_scale"])
    return cam