Python yaml.FullLoader() Examples

def import_files(self):
        print("self.manifest_path", self.manifest_path)
        self.manifest = yaml.load(open("{}/manifest.yml".format(self.manifest_path)), Loader=yaml.FullLoader)
        sys.path.append(self.manifest_path)
        """
        don't remove the import below, this will be the cti_transformations.py,
        which is one of the required file to run the job. This file will be provided by the 
        user during the run.
        """
        try:
            import ib_functions
        except Exception as e:
            ib_functions = None
        self.ib_functions = ib_functions
        print ("self.ib_functions is {}".format(ib_functions))
        # print("manifest is {}".format(self.manifest))
        # print("ib_functions is {}".format(self.ib_functions)) 

def find_descriptor_schemas(self, schema_file):
        """Find descriptor schemas in given path."""
        if not schema_file.lower().endswith((".yml", ".yaml")):
            return []

        with open(schema_file) as fn:
            schemas = yaml.load(fn, Loader=yaml.FullLoader)
        if not schemas:
            self.stderr.write("Could not read YAML file {}".format(schema_file))
            return []

        descriptor_schemas = []
        for schema in schemas:
            if "schema" not in schema:
                continue

            descriptor_schemas.append(schema)

        return descriptor_schemas 

def load_config(yaml_file):
    logger.debug("Loading configuration from %s" % yaml_file)
    fd = None
    try:
        fd = open(yaml_file)
    except Exception as e:
        logger.error(
            "Error reading configuration file %s, ignoring..." % yaml_file
        )
        return

    try:
        return yaml.load(fd, Loader=yaml.FullLoader)
    except Exception as e:
        logger.error(
            "Error parsing configuration file %s: %s" % (yaml_file, e)
        )
        return 

def load_configs_from_yaml(yaml_file: str) -> Dict:
    """
    从yaml配置文件中加载参数，这里会将嵌套的二级映射调整为一级映射

    Args:
        yaml_file: yaml】文件路径

    Returns:
        yaml文件中的配置字典
    """
    yaml_config = yaml.load(open(yaml_file, encoding='utf-8'), Loader=yaml.FullLoader)
    configs_dict = {}
    for sub_k, sub_v in yaml_config.items():
        # 读取嵌套的参数
        if isinstance(sub_v, dict):
            for k, v in sub_v.items():
                if k in configs_dict.keys():
                    raise ValueError(f'Duplicate parameter : {k}')
                configs_dict[k] = v
        else:
            configs_dict[sub_k] = sub_v
    return configs_dict 

def test_improper_metrics_repo(improper_metrics_repo):
    runner.main(cache_dir, out_dir, None, None, False, True, repositories_file,
                True, None, None, None, None, 'dev')

    path = os.path.join(out_dir, "glean", improper_repo_name, "metrics")
    with open(path, 'r') as data:
        metrics = json.load(data)

    # should be empty output, since it was an improper file
    assert not metrics

    with open(EMAIL_FILE, 'r') as email_file:
        emails = yaml.load(email_file, Loader=yaml.FullLoader)

    # should send 1 email
    assert len(emails) == 1 

def load_scenario(scenario_str) -> Scenario:
    """
    Load a scenario from a yaml string.
    :param scenario_str:
    :return:
    """
    loaded = yaml.load(scenario_str, Loader=yaml.FullLoader)
    evts = []
    for evt in loaded["events"]:
        id_evt = evt["id"]
        if "actions" in evt:
            actions = []
            for a in evt["actions"]:
                args = dict(a)
                args.pop("type")
                actions.append(EventAction(a["type"], **args))
            evts.append(DcopEvent(id_evt, actions=actions))
        elif "delay" in evt:
            evts.append(DcopEvent(id_evt, delay=evt["delay"]))

    return Scenario(evts) 

def add_single_video_yaml(yamlfile,videofile):
    yamlPath = yamlfile
    cap = cv2.VideoCapture(videofile)
    width = int(cap.get(3))  # float
    height = int(cap.get(4))  # float
    cropLine = [0, width, 0, height]
    cropLine = str(cropLine)
    currCropLinePath = cropLine.strip("[]")
    currCropLinePath = currCropLinePath.replace("'", "")
    with open(yamlPath) as f:
        read_yaml = yaml.load(f, Loader=yaml.FullLoader)

    read_yaml["video_sets"].update({videofile: {'crop': currCropLinePath}})

    with open(yamlPath, 'w') as outfile:
        yaml.dump(read_yaml, outfile, default_flow_style=False) 

def test_regularize_parameters():

    params_yaml = """
params:
  stop_cycle: 100
  variant: [A, B, C]
  probability: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]
"""

    params = yaml.load(params_yaml, Loader=yaml.FullLoader)

    reg_params = regularize_parameters(params["params"])
    assert len(reg_params) == 3

    assert len(reg_params["stop_cycle"]) == 1
    for v in reg_params["stop_cycle"]:
        assert isinstance(v, str)

    assert len(reg_params["variant"]) == 3
    for v in reg_params["variant"]:
        assert isinstance(v, str)

    assert len(reg_params["probability"]) == 6
    for v in reg_params["probability"]:
        assert isinstance(v, str) 

def update_init_weight(yamlfile,initweights):
    yamlPath=yamlfile
    initweights,initw_filetype = os.path.splitext(initweights)

    with open(yamlPath) as f:
        read_yaml = yaml.load(f, Loader=yaml.FullLoader)

    iteration = read_yaml['iteration']

    yamlfiledirectory = os.path.dirname(yamlfile)
    iterationfolder = yamlfiledirectory +'\\dlc-models\\iteration-' +str(iteration)
    projectfolder = os.listdir(iterationfolder)
    projectfolder = projectfolder[0]


    posecfg = iterationfolder + '\\' + projectfolder +'\\train\\' + 'pose_cfg.yaml'

    with open(posecfg) as g:
        read_cfg = yaml.load(g, Loader=yaml.FullLoader)

    read_cfg['init_weights'] = str(initweights)

    with open(posecfg, 'w') as outfile:
        yaml.dump(read_cfg, outfile, default_flow_style=False)
    print(os.path.basename(initweights),'selected') 

def main(args):
    with open(args.config) as f:
        if version.parse(yaml.version >= "5.1"):
            config = yaml.load(f, Loader=yaml.FullLoader)
        else:
            config = yaml.load(f)

    for k, v in config.items():
        setattr(args, k, v)

    # exp path
    if not hasattr(args, 'exp_path'):
        args.exp_path = os.path.dirname(args.config)

    # dist init
    if mp.get_start_method(allow_none=True) != 'spawn':
        mp.set_start_method('spawn', force=True)
    dist_init(args.launcher, backend='nccl')

    # train
    trainer = Trainer(args)
    trainer.run() 

def generatetempyaml_multi(yamlfile,videolist):

    #copy yaml and rename
    tempyaml = os.path.dirname(yamlfile) +'\\temp.yaml'
    shutil.copy(yamlfile,tempyaml)


    deeplabcut.add_new_videos(tempyaml,videolist,copy_videos=True)

    with open(tempyaml) as f:
        read_yaml = yaml.load(f, Loader=yaml.FullLoader)

    original_videosets = read_yaml['video_sets'].keys()

    keys=[]
    for i in original_videosets:
        keys.append(i)


    read_yaml['video_sets'].pop(keys[0],None)

    with open(tempyaml, 'w') as outfile:
        yaml.dump(read_yaml, outfile, default_flow_style=False) 

def main():
    exp_dir = os.path.dirname(args.config)
    
    with open(args.config) as f:
        if version.parse(yaml.version >= "5.1"):
            config = yaml.load(f, Loader=yaml.FullLoader)
        else:
            config = yaml.load(f)

    for k, v in config.items():
        setattr(args, k, v)
    
    model = models.modules.__dict__[args.model['module']['arch']](args.model['module'])
    model = torch.nn.DataParallel(model)
    
    ckpt_path = exp_dir + '/checkpoints/ckpt_iter_{}.pth.tar'.format(args.iter)
    save_path = exp_dir + '/checkpoints/convert_iter_{}.pth.tar'.format(args.iter)
    ckpt = torch.load(ckpt_path)
    weight = ckpt['state_dict']
    model.load_state_dict(weight, strict=True)
    model = model.module.image_encoder
    
    torch.save(model.state_dict(), save_path) 

def load_dcop(dcop_str: str, main_dir=None) -> DCOP:
    loaded = yaml.load(dcop_str, Loader=yaml.FullLoader)

    if "name" not in loaded:
        raise ValueError("Missing name in dcop string")
    if "objective" not in loaded or loaded["objective"] not in ["min", "max"]:
        raise ValueError("Objective is mandatory and must be min or max")

    dcop = DCOP(
        loaded["name"],
        loaded["objective"],
        loaded["description"] if "description" in loaded else "",
    )

    dcop.domains = _build_domains(loaded)
    dcop.variables = _build_variables(loaded, dcop)
    dcop.external_variables = _build_external_variables(loaded, dcop)
    dcop._constraints = _build_constraints(loaded, dcop, main_dir)
    dcop._agents_def = _build_agents(loaded)
    dcop.dist_hints = _build_dist_hints(loaded, dcop)
    return dcop 

def test_run_batches_direct_and_iteration(mock_run_batch):
    with tempfile.TemporaryDirectory() as tmpdirname:

        definition = f"""
sets:
  set1:
     path: {tmpdirname}
     iterations: 3
batches:
  batch1:
    command: test
    """
        conf = yaml.load(definition, Loader=yaml.FullLoader)
        run_batches(conf, simulate=False)

        assert mock_run_batch.call_count == 3 

def load_config(config_path):
    """ Load a configuration file, also reading any component configs """
    with open(str(config_path)) as config_fd:
        config = yaml.load(config_fd, Loader=yaml.FullLoader)
    for component in config:
        if isinstance(config[component], dict) and "path" in config[component]:
            component_path = CONFIG_ROOT / config[component]["path"]
            with open(str(component_path)) as component_fd:
                component_config = yaml.load(component_fd, Loader=yaml.FullLoader)
            component_config.update(config[component])
            config[component] = component_config
            if "name" not in config[component]:
                config[component]["name"] = component_path.stem
    if "name" not in config:
        config["name"] = config_path.stem
    return config 

def test_runtime_overrides_influx(path_to_config_files):
    expanded_template = _generate_test_workflow_yaml(
        path_to_config_files, "config-test-runtime-resource.yaml"
    )
    templates = expanded_template["spec"]["templates"]
    influx_task = [
        task for task in templates if task["name"] == "gordo-influx-statefulset"
    ][0]
    influx_statefulset_definition = yaml.load(
        influx_task["resource"]["manifest"], Loader=yaml.FullLoader
    )
    influx_resource = influx_statefulset_definition["spec"]["template"]["spec"][
        "containers"
    ][0]["resources"]
    # We use yaml overriden memory (both request and limits).
    assert influx_resource["requests"]["memory"] == "321M"

    # This was specified to 120 in the config file, but is bumped to match the
    # request
    assert influx_resource["limits"]["memory"] == "321M"
    # requests.cpu is default
    assert influx_resource["requests"]["cpu"] == "520m"
    assert influx_resource["limits"]["cpu"] == "10040m" 

def test_dataset_from_config_checks_dates():
    """
    A dataset needs to have train_start_date properly before train_end_date
    """
    element_str = """
        dataset:
          resolution: 2T
          tags:
            - GRA-YE  -23-0751X.PV
            - GRA-TE  -23-0698.PV
            - GRA-PIT -23-0619B.PV
          train_start_date: 2018-05-10T15:05:50+02:00
          train_end_date: 2018-05-10T15:05:50+02:00
    """
    dataset_config = yaml.load(element_str, Loader=yaml.FullLoader)["dataset"]
    with pytest.raises(ValueError):
        TimeSeriesDataset.from_dict(dataset_config) 

def get_dict_from_yaml(config_file: Union[str, io.StringIO]) -> dict:
    """
    Read a config file or file like object of YAML into a dict
    """
    # We must override the default constructor for timestamp to ensure the result
    # has tzinfo. Yaml loader never adds tzinfo, but converts to UTC.
    yaml.FullLoader.add_constructor(
        tag="tag:yaml.org,2002:timestamp", constructor=_timestamp_constructor
    )
    if hasattr(config_file, "read"):
        yaml_content = yaml.load(config_file, Loader=yaml.FullLoader)
    else:
        try:
            path_to_config_file = os.path.abspath(config_file)  # type: ignore
            with open(path_to_config_file, "r") as yamlfile:  # type: ignore
                yaml_content = yaml.load(yamlfile, Loader=yaml.FullLoader)
        except FileNotFoundError:
            raise FileNotFoundError(
                f"Unable to find config file <{path_to_config_file}>"
            )
    # Handle multiple versions of workflow config structure
    if "spec" in yaml_content:
        yaml_content = yaml_content["spec"]["config"]

    return yaml_content 

def readYaml(env=None):
    """Load the skelebot.yaml, with environment overrride if present, into the Config object"""

    yamlData = None
    cwd = os.getcwd()
    cfgFile = FILE_PATH.format(path=cwd)
    if os.path.isfile(cfgFile):
        with open(cfgFile, 'r') as stream:
            yamlData = yaml.load(stream, Loader=yaml.FullLoader)
            if (env is not None):
                envFile = ENV_FILE_PATH.format(path=cwd, env=env)
                if os.path.isfile(envFile):
                    with open(envFile, 'r') as stream:
                        overrideYaml = yaml.load(stream, Loader=yaml.FullLoader)
                        yamlData = override(yamlData, overrideYaml)
                else:
                    raise RuntimeError("Environment Not Found")

    return yamlData 

def LoadConfig(self):
        file = open(self.__config_path, 'r', encoding='utf-8')
        content = file.read()
        # config = yaml.load(content, Loader=yaml.FullLoader)
        config = yaml.load(content)
        self.__image_classes = config[self.__image_classes_key]
        self.__feature_classes = config[self.__feature_classes_key]
        file.close() 

def config(self):
        with open(self.conf_yaml, 'r') as f:
            return yaml.load(f, Loader=yaml.FullLoader) 

def read_yaml_template(template=None):
    """Read yaml file."""
    if template is None:
        template = os.path.join(os.path.dirname(__file__), "topsApp-template.yml")
    with open(template, "r") as outfile:
        defaults = yaml.load(outfile, Loader=yaml.FullLoader)

    return defaults 

def load(self, model_dir, configs, initialize=False, print=True):
        save_config_file = os.path.join(model_dir, self.CONFIG_FILE_NAME)
        if os.path.exists(save_config_file):
            configs = [save_config_file] + configs
        elif not initialize:
            raise ValueError("{} is an invalid model directory".format(model_dir))

        for cfg in configs:
            kv = [s.strip() for s in cfg.split("=", 1)]
            if len(kv) == 1:
                if not os.path.exists(cfg):
                    raise ValueError("The file '{}' doesn't exist.".format(cfg))
                obj = yaml.load(open(cfg).read(), Loader=yaml.FullLoader)
                for k, v in obj.items():
                    self[k] = v
            else:
                k, v = kv
                try:
                    v = int(v)
                except ValueError:
                    try:
                        v = float(v)
                    except ValueError:
                        v_norm = v.lower().strip()
                        if v_norm == 'true':
                            v = True
                        elif v_norm == 'false':
                            v = False
                        elif v_norm == 'null':
                            v = None
                self[k] = v

            if not os.path.exists(save_config_file) and initialize:
                self.save(model_dir)

            if print:
                logging.info("All configurations:\n" + repr(self)) 

def run_distribute(filename, distribution, graph=None, algo=None):
    """
    Run the distribute cli command with the given parameters
    """
    filename = instance_path(filename)
    algo_opt = '' if algo is None else '-a ' + algo
    graph_opt = '' if graph is None else '-g ' + graph
    cmd = 'pydcop distribute -d {distribution} {graph_opt} ' \
          '{algo_opt} {file}'.format(distribution=distribution,
                                     graph_opt=graph_opt,
                                     algo_opt=algo_opt,
                                     file=filename)
    output = check_output(cmd, stderr=STDOUT, timeout=10, shell=True)
    return yaml.load(output.decode(encoding='utf-8'), Loader=yaml.FullLoader) 

def preprocess(args):
    with open(args.config) as f:
        config = yaml.load(f, Loader=yaml.FullLoader)

    # Make directory if not exist
    os.makedirs(args.output_dir, exist_ok=True)
    print('')
    print('[INFO] Root directory:', args.data_dir)

    AP = AudioProcessor(**config['audio'])
    executor = ProcessPoolExecutor(max_workers=args.n_jobs)
    fid = []
    text = []
    wav = []
    futures = []
    with open(args.old_meta, encoding='utf-8') as f:
        for line in f:
            parts = line.strip().split('|')
            fpath = os.path.join(args.data_dir, '%s.wav' % parts[0])
            text = parts[2]
            job = executor.submit(partial(process_utterance, fpath, text, args.output_dir, AP))
            futures += [job]

    print('[INFO] Preprocessing', end=' => ')
    print(len(futures), 'audio files found')
    results = [future.result() for future in tqdm(futures)]
    fpath_meta = os.path.join(args.output_dir, 'ljspeech_meta.txt')
    with open(fpath_meta, 'w') as f:
        for x in results:
            s = map(lambda x: str(x), x)
            f.write('|'.join(s) + '\n') 

def __init__(self, asr, emb_decoder, beam_size, min_len_ratio, max_len_ratio,
                 lm_path='', lm_config='', lm_weight=0.0, ctc_weight=0.0):
        super().__init__()
        # Setup
        self.beam_size = beam_size
        self.min_len_ratio = min_len_ratio
        self.max_len_ratio = max_len_ratio
        self.asr = asr

        # ToDo : implement pure ctc decode
        assert self.asr.enable_att

        # Additional decoding modules
        self.apply_ctc = ctc_weight > 0
        if self.apply_ctc:
            assert self.asr.ctc_weight > 0, 'ASR was not trained with CTC decoder'
            self.ctc_w = ctc_weight
            self.ctc_beam_size = int(CTC_BEAM_RATIO * self.beam_size)

        self.apply_lm = lm_weight > 0
        if self.apply_lm:
            self.lm_w = lm_weight
            self.lm_path = lm_path
            lm_config = yaml.load(open(lm_config, 'r'), Loader=yaml.FullLoader)
            self.lm = RNNLM(self.asr.vocab_size, **lm_config['model'])
            self.lm.load_state_dict(torch.load(
                self.lm_path, map_location='cpu')['model'])
            self.lm.eval()

        self.apply_emb = emb_decoder is not None
        if self.apply_emb:
            self.emb_decoder = emb_decoder 

def update_from_file(self, config_file):
        with codecs.open(config_file, 'r', 'utf-8') as file:
            loaded_cfg = yaml.load(file, Loader=yaml.FullLoader)
        self.update_from_other_cfg(loaded_cfg) 

def run_graph(filename, graph):
    filename = instance_path(filename)
    cmd = 'pydcop graph -g {graph} {file}'.format(graph=graph,
                                                   file=filename)
    output = check_output(cmd, stderr=STDOUT, timeout=10, shell=True)
    return yaml.load(output.decode(encoding='utf-8'), Loader=yaml.FullLoader) 

def load_replica_dist(dist_str: str) -> ReplicaDistribution:
    loaded = yaml.load(dist_str, Loader=yaml.FullLoader)

    if 'replica_dist' not in loaded:
        raise ValueError('Invalid replica distribution file')

    loaded_dist = loaded['replica_dist']

    return ReplicaDistribution(loaded_dist) 

def load_dist(dist_str: str) -> Distribution:
    loaded = yaml.load(dist_str, Loader=yaml.FullLoader)

    if 'distribution' not in loaded:
        raise ValueError('Invalid distribution file')

    loaded_dist = loaded['distribution']

    return Distribution(loaded_dist)