Python yaml.full_load() Examples

def open_configuration_file(sFileName, sJUnitFileName=None):
    '''Attempts to open a configuration file and read it's contents.'''
    try:
        with open(sFileName) as yaml_file:
            tempConfiguration = yaml.full_load(yaml_file)
    except IOError:
        print('ERROR: Could not find configuration file: ' + sFileName)
        write_invalid_configuration_junit_file(sFileName, sJUnitFileName)
        sys.exit(1)
    except yaml.scanner.ScannerError as e:
        print('ERROR: Invalid configuration file: ' + sFileName)
        print(e)
        write_invalid_configuration_junit_file(sFileName, sJUnitFileName)
        exit()
    except yaml.parser.ParserError as e:
        print('ERROR: Invalid configuration file: ' + sFileName)
        print(e)
        write_invalid_configuration_junit_file(sFileName, sJUnitFileName)
        exit()
    return tempConfiguration 

def read_lint_config(repo_path):
    # read .lintconfig.yml
    full_path = os.path.expanduser(repo_path)
    config_filepath = os.path.join(full_path, '.lintconfig.yml')
    acronyms = []
    abbreviations = []
    timeframes = []
    checks = []
    if os.path.isfile(config_filepath):
        with open(config_filepath) as f:
            config = yaml.full_load(f)
            acronyms = config.get('acronyms', acronyms)
            abbreviations = config.get('abbreviations', abbreviations)
            timeframes = config.get('timeframes', timeframes)
            checks = config.get('checks', checks)

    checks = _parse_checks(checks)
    return {'acronyms': acronyms, 'abbreviations': abbreviations, 'timeframes': timeframes, 'checks': checks} 

def conf_to_dict(conf):
	result = None
	# json string?
	try:
		result = json.loads(conf)
	except:
		# json file?
		try:	
			with open(conf) as f:
				result =  json.load(f)
		except:
			# yaml file?
			try:
				with open(conf) as stream:
					result = yaml.full_load(stream)
			except:
				pass
	return result 

def test_build_model_specification(mocker, test_spec, test_user_config):
    expand_user_mock = mocker.patch('vivarium.framework.configuration.Path.expanduser')
    expand_user_mock.return_value = test_user_config
    loaded_model_spec = build_model_specification(test_spec)

    test_data = DEFAULT_PLUGINS.copy()

    with test_spec.open() as f:
        model_data = yaml.full_load(f)

    test_data.update(model_data)

    with test_user_config.open() as f:
        user_data = yaml.full_load(f)

    test_data['configuration'].update(user_data)

    assert loaded_model_spec.to_dict() == test_data 

def _coerce(data: Union[Dict, str, Path, 'ConfigTree'], source: Union[str, None]) -> Tuple[Dict, Union[str, None]]:
        """Coerces data into dictionary format."""
        if isinstance(data, dict):
            return data, source
        elif (isinstance(data, str) and data.endswith(('.yaml', '.yml'))) or isinstance(data, Path):
            source = source if source else str(data)
            with open(data) as f:
                data = f.read()
            data = yaml.full_load(data)
            return data, source
        elif isinstance(data, str):
            data = yaml.full_load(data)
            return data, source
        elif isinstance(data, ConfigTree):
            return data.to_dict(), source
        else:
            raise ConfigurationError(f"ConfigTree can only update from dictionaries, strings, paths and ConfigTrees. "
                                     f"You passed in {type(data)}", value_name=None) 

def validate_model_specification_file(file_path: Union[str, Path]) -> None:
    """Ensures the provided file is a yaml file"""
    file_path = Path(file_path)
    if not file_path.exists():
        raise ConfigurationError('If you provide a model specification file, it must be a file. '
                                 f'You provided {str(file_path)}', value_name=None)

    if file_path.suffix not in ['.yaml', '.yml']:
        raise ConfigurationError(f'Model specification files must be in a yaml format. You provided {file_path.suffix}',
                                 value_name=None)
    # Attempt to load
    with file_path.open() as f:
        raw_spec = yaml.full_load(f)
    top_keys = set(raw_spec.keys())
    valid_keys = {'plugins', 'components', 'configuration'}
    if not top_keys <= valid_keys:
        raise ConfigurationError(f'Model specification contains additional top level '
                                 f'keys {valid_keys.difference(top_keys)}.', value_name=None) 

def get_predefined_styles():
    '''
    Reads all predefined styles and returns a list of names.

    Parameters : None

    Returns : (list of strings)
    '''
    lReturn = []
    sStylePath = os.path.join(os.path.dirname(__file__), 'styles')
    lStyles = os.listdir(sStylePath)
    for sStyle in lStyles:
        if sStyle.endswith('.yaml'):
            with open(os.path.join(sStylePath, sStyle)) as yaml_file:
                tempConfiguration = yaml.full_load(yaml_file)
            lReturn.append(tempConfiguration['name'])
    return lReturn 

def test_set_stylesheet(mock_reddit):
    r = mock_reddit.return_value
    with open(os.path.join(BASE_DIR,"tests", "fixture_data", "stylesheet_0" + ".json"), "r") as f:
        stylesheet = json.loads(f.read())
    r.get_stylesheet.return_value = stylesheet
    r.set_stylesheet.return_value = {"errors":[]}
    patch('praw.')

    experiment_name = "stylesheet_experiment_test"
    with open(os.path.join(BASE_DIR,"config", "experiments", experiment_name + ".yml"), "r") as f:
        experiment_config = yaml.full_load(f)['test']
    controller = StylesheetExperimentController(experiment_name, db_session, r, log)

    for condition in ['special', 'normal']:
        for arm in ["arm_0", "arm_1"]:
            assert (controller.experiment_settings['conditions'][condition]['arms'][arm] in stylesheet['stylesheet'].split("\n"))!=True

    for condition in ['special', 'normal']:
        for arm in ["arm_0", "arm_1"]:
            line_length = len(stylesheet['stylesheet'].split("\n"))
            result_lines = controller.set_stylesheet(condition, arm).split("\n")
            assert controller.experiment_settings['conditions'][condition]['arms'][arm] in result_lines
            assert len(result_lines) == line_length + 3 

def get_experiment_config(self, required_keys, experiment_name):        
        experiment_file_path = os.path.join(BASE_DIR, "config", "experiments", experiment_name) + ".yml"
        with open(experiment_file_path, 'r') as f:
            try:
                experiment_config_all = yaml.full_load(f)
            except yaml.YAMLError as exc:
                self.log.error("{0}: Failure loading experiment yaml {1}".format(
                    self.__class__.__name__, experiment_file_path), str(exc))
                sys.exit(1)
        if(ENV not in experiment_config_all.keys()):
            self.log.error("{0}: Cannot find experiment settings for {1} in {2}".format(
                self.__class__.__name__, ENV, experiment_file_path))
            sys.exit(1)

        experiment_config = experiment_config_all[ENV]
        for key in required_keys:
            if key not in experiment_config.keys():
                self.log.error("{0}: Value missing from {1}: {2}".format(
                    self.__class__.__name__, experiment_file_path, key))
                sys.exit(1)
        return experiment_config 

def test_from_yaml(self):
        stream = StringIO(TEST_JSON)
        d = yaml.full_load(stream)
        d = flatten_dict(d['output_metadata'])
        self.assertEqual(17, len(d))
        self.assertEqual('DCS4COP Sentinel-3 OLCI L2C Data Cube',
                         d.get('title'))
        self.assertEqual('Brockmann Consult GmbH, Royal Belgian Institute for Natural Sciences (RBINS)',
                         d.get('creator_name'))
        self.assertEqual('https://www.brockmann-consult.de, http://odnature.naturalsciences.be/remsem/',
                         d.get('creator_url'))
        self.assertEqual("2018-05-30", d.get('date_created'))
        self.assertEqual("2018-06-01", d.get('date_issued'))
        self.assertEqual(0.0, d.get('geospatial_lon_min'))
        self.assertEqual(5.0, d.get('geospatial_lon_max'))
        self.assertEqual(50.0, d.get('geospatial_lat_min'))
        self.assertEqual(52.5, d.get('geospatial_lat_max'))
        self.assertEqual('degrees_east', d.get('geospatial_lon_units'))
        self.assertEqual('degrees_north', d.get('geospatial_lat_units'))
        self.assertEqual(0.0025, d.get('geospatial_lon_resolution'))
        self.assertEqual(0.0025, d.get('geospatial_lat_resolution'))
        self.assertEqual('2016-10-01', d.get('time_coverage_start'))
        self.assertEqual('2017-10-01T12:00:10', d.get('time_coverage_end'))
        self.assertEqual('P1Y', d.get('time_coverage_duration'))
        self.assertEqual('1D', d.get('time_coverage_resolution')) 

def get_experiment_config(self, required_keys, experiment_name):        
        experiment_file_path = os.path.join(BASE_DIR, "config", "experiments", experiment_name) + ".yml"
        with open(experiment_file_path, 'r') as f:
            try:
                experiment_config_all = yaml.full_load(f)
            except yaml.YAMLError as exc:
                self.log.error("{0}: Failure loading experiment yaml {1}".format(
                    self.__class__.__name__, experiment_file_path), str(exc))
                sys.exit(1)
        if(ENV not in experiment_config_all.keys()):
            self.log.error("{0}: Cannot find experiment settings for {1} in {2}".format(
                self.__class__.__name__, ENV, experiment_file_path))
            sys.exit(1)

        experiment_config = experiment_config_all[ENV]
        for key in required_keys:
            if key not in experiment_config.keys():
                self.log.error("{0}: Value missing from {1}: {2}".format(
                    self.__class__.__name__, experiment_file_path, key))
                sys.exit(1)
        return experiment_config 

def get_experiment_config(self, required_keys, experiment_name): 
        experiment_file_path = os.path.join(BASE_DIR, "config", "experiments", experiment_name) + ".yml"
        with open(experiment_file_path, 'r') as f:
            try:
                experiment_config_all = yaml.full_load(f)
            except yaml.YAMLError as exc:
                self.log.error("{0}: Failure loading experiment yaml {1}".format(
                    self.__class__.__name__, experiment_file_path), str(exc))
                sys.exit(1)
        if(ENV not in experiment_config_all.keys()):
            self.log.error("{0}: Cannot find experiment settings for {1} in {2}".format(
                self.__class__.__name__, ENV, experiment_file_path))
            sys.exit(1)

        experiment_config = experiment_config_all[ENV]
        for key in required_keys:
            if key not in experiment_config.keys():
                self.log.error("{0}: Value missing from {1}: {2}".format(
                    self.__class__.__name__, experiment_file_path, key))
                sys.exit(1)
        return experiment_config 

def insert_admin():
    dirs = os.listdir()
    if 'app' in dirs:
        f = open("app/config.cfg", 'r')
    else:
        f = open("./config.cfg", 'r')
    settings = yaml.full_load(f)
    settings = settings['docker']
    mongo_client = MongoClient(settings['mongo_ip'], settings['mongo_port'])
    mongo_db = mongo_client['userinfo']
    userinfo = mongo_db['userinfo']
    f.close()

    parser = argparse.ArgumentParser(description = "Add admin user to MongoDB")
    parser.add_argument('-p', '--password', type = str, help = "Admin password", required = True)

    if userinfo.find({'username':'admin'}).count() > 0:
        print("Admin already exists")
    else:
        args = parser.parse_args()
        password = args.password
        hashed = hashpw(password.encode('utf-8'), gensalt())
        userinfo.insert_one({"username":"admin","password":hashed,"user_id":0})
        print("Successfully added an admin user with password {}!".format(password)) 

def _parse_docstring(obj, process_doc, from_file_keyword, base_path):
    first_line, other_lines, swag = None, None, None
    full_doc = inspect.getdoc(obj)
    if full_doc:
        if from_file_keyword is not None:
            from_file = _find_from_file(full_doc, from_file_keyword, base_path)
            if from_file:
                full_doc_from_file = _doc_from_file(from_file)
                if full_doc_from_file:
                    full_doc = full_doc_from_file
        line_feed = full_doc.find('\n')
        if line_feed != -1:
            first_line = process_doc(full_doc[:line_feed])
            yaml_sep = full_doc[line_feed+1:].find('---')
            if yaml_sep != -1:
                other_lines = process_doc(full_doc[line_feed+1:line_feed+yaml_sep])
                swag = yaml.full_load(full_doc[line_feed+yaml_sep:])
            else:
                other_lines = process_doc(full_doc[line_feed+1:])
        else:
            first_line = full_doc
    return first_line, other_lines, swag 

def load_type(obj_type, path: str):
    """Load a Kubernetes YAML manifest file for the specified type.

    While Kubernetes manifests can contain multiple object definitions
    in a single file (delimited with the YAML separator '---'), this
    does not currently support those files. This function expects a
    single object definition in the specified manifest file.

    Args:
        path: The path the manifest YAML to load.
        obj_type: The Kubernetes API object type that the YAML
            contents should be loaded into.

    Returns:
        A Kubernetes API object populated with the YAML contents.

    Raises:
        FileNotFoundError: The specified file was not found.
    """
    with open(path, 'r') as f:
        manifest = yaml.full_load(f)

    return new_object(obj_type, manifest) 

def read_yaml_config(config_file):
    config = None
    try:
        with open(config_file, 'r') as stream:
            config = yaml.full_load(stream) if hasattr(yaml, 'full_load') else yaml.load(stream)
    except yaml.YAMLError as exc:
        logger.error("Invalid config_file %s: %s" % (config_file, exc))
    return config 

def load_experiment_config(experiment_name):
    """Load the configuration file for the provided experiment name."""
    try:
        path = Path(BASE_DIR)/'config'/'experiments'/(experiment_name+'.yml')
        with open(str(path)) as f:
            config = yaml.full_load(f)
        return config
    except:
        log_msg = '%s Error reading the experiment configuration: %s'
        log.error(log_msg, LOG_PREFIX, experiment_name)
        raise 

def test_determine_intervention_eligible(mock_reddit):
    r = mock_reddit.return_value
    patch('praw.')

    experiment_name = "stylesheet_experiment_test"
    with open(os.path.join(BASE_DIR,"config", "experiments", experiment_name + ".yml"), "r") as f:
        experiment_config = yaml.full_load(f)['test']

    assert len(db_session.query(Experiment).all()) == 0
    controller = StylesheetExperimentController(experiment_name, db_session, r, log)

    ## in the case with no interventions, confirm eligibility
    assert controller.determine_intervention_eligible() == True

    ## now create an action and confirm ineligibility outside the interval
    experiment_action = ExperimentAction(
        experiment_id = controller.experiment.id,
        praw_key_id = "TEST",
        action = "Intervention:{0}.{1}".format("TEST","TEST"),
        action_object_type = ThingType.STYLESHEET.value,
        action_object_id = None,
        metadata_json  = json.dumps({"arm":"TEST", "condition":"TEST"})
    )
    db_session.add(experiment_action)
    db_session.commit()

    assert controller.determine_intervention_eligible() == False
    
    ## now change the action and confirm eligibility within the interval
    experiment_action.created_at = experiment_action.created_at - datetime.timedelta(seconds=controller.experiment_settings['intervention_interval_seconds'])
    db_session.commit()
    assert controller.determine_intervention_eligible() == True

    ## now change the end date of the experiment and confirm ineligibility
    controller.experiment_settings['end_time'] = str((datetime.datetime.utcnow() - datetime.timedelta(days=1)).replace(tzinfo=pytz.utc))
    #controller.experiment.settings = json.dumps(controller.experiment_settings)
    #db_session.commit()
    assert controller.determine_intervention_eligible() == False 

def test_select_condition(mock_reddit):
    r = mock_reddit.return_value
    patch('praw.')

    experiment_name = "stylesheet_experiment_test"
    with open(os.path.join(BASE_DIR,"config", "experiments", experiment_name + ".yml"), "r") as f:
        experiment_config = yaml.full_load(f)['test']
    controller = StylesheetExperimentController(experiment_name, db_session, r, log)
    
    assert controller.select_condition(current_time = parser.parse("07/21/2017 00:00:00")) == "special"
    assert controller.select_condition(current_time = parser.parse("07/20/2017 00:00:00")) == "normal" 

def parse_file(file: Path) -> dict:
    with file.open() as stream:
        try:
            return yaml.full_load(stream)
        except yaml.YAMLError as e:
            print(f'Could not parse {file}')
            print(e)
            sys.exit(1) 

def run(args):
    if args.sr != 16000:
        raise ValueError("Now only support audio in 16kHz")
    # shape: T x F, complex
    stft_kwargs = {
        "frame_len": args.frame_len,
        "frame_hop": args.frame_hop,
        "window": args.window,
        "center": args.center,
    }
    spectrogram_reader = SpectrogramReader(
        args.wav_scp,
        **stft_kwargs,
        round_power_of_two=args.round_power_of_two)

    if args.conf:
        with open(args.conf, "r") as conf:
            omlsa_conf = yaml.full_load(conf)
            suppressor = iMCRA(**omlsa_conf)
    else:
        suppressor = iMCRA()

    if args.output == "wave":
        with WaveWriter(args.dst_dir, sr=args.sr) as writer:
            for key, stft in spectrogram_reader:
                logger.info(f"Processing utterance {key}...")
                gain = suppressor.run(stft)
                samps = inverse_stft(gain * stft, **stft_kwargs)
                writer.write(key, samps)
    else:
        with NumpyWriter(args.dst_dir) as writer:
            for key, stft in spectrogram_reader:
                logger.info(f"Processing utterance {key}...")
                gain = suppressor.run(stft)
                writer.write(key, gain)
    logger.info(f"Processed {len(spectrogram_reader):d} utterances done") 

def get_config():
    config_file = os.path.abspath(os.path.join(os.path.expanduser("~"), 'config.yaml'))
    with open(config_file, 'r') as file:
        config = yaml.full_load(file)
    return config 

def load_yaml(file):
    try:
        config = yaml.full_load(file.read_text())
        tool_name = str(file.name.replace(".yaml", ""))
        tools[tool_name] = config
    except KeyError as error:  # load dependencies first
        dependency = error.args[0]
        dependency_file = definitions / (dependency + ".yaml")
        load_yaml(dependency_file)
        load_yaml(file) 

def test_read_yaml_loader_binary_mode(self):
        testYaml = read(
            TestReadHelper.TEST_FILE, loader=yaml.full_load, binary_file=True
        )
        self.assertEqual(testYaml["testYaml"], "testing") 

def test_read_yaml_loader_non_binary(self):
        testYaml = read(TestReadHelper.TEST_FILE, loader=yaml.full_load)
        self.assertEqual(testYaml["testYaml"], "testing") 

def read_cfg(path_to_config_file, profile_name):
    cfg = read(path_to_config_file, loader=yaml.full_load)
    if profile_name is not None:
        cfg["profile"] = profile_name
    elif "AWS_PROFILE" in os.environ:
        cfg["profile"] = os.environ["AWS_PROFILE"]
    return cfg 

def load_stt_correction_file(cls, stt_correction_file):
        stt_correction_file_path = Utils.get_real_file_path(stt_correction_file)
        stt_correction_file = open(stt_correction_file_path, "r")
        stt_correction = yaml.full_load(stt_correction_file)

        return stt_correction 

def __init__(self, file_path):
        """
        Load yaml file, with includes statement
        :param file_path: path to the yaml file to load
        """
        # get the parent dir. will be used in case of relative path
        parent_dir = os.path.normpath(file_path + os.sep + os.pardir)

        # load the yaml file
        with open(file_path, "r") as f:
            self.data = yaml.full_load(f)

        if self.data is None:
            raise YAMLFileEmpty("[YAMLLoader] File %s is empty" % file_path)
        # add included brain
        if isinstance(self.data, list):
            for el in self.data:
                if "includes" in el:
                    for inc in el["includes"]:
                        # if the path is relative, we add the root path
                        if not os.path.isabs(inc):  # os.path.isabs returns True if the path is absolute
                            # logger.debug("File path %s is relative, adding the root path" % inc)
                            inc = os.path.join(parent_dir, inc)
                            # logger.debug("New path: %s" % inc)
                        with open(inc, "r") as f:
                            self.update(yaml.full_load(f)) 

def ipmi_fru2dict(fru_str):
    """Convert the ipmitool fru output to a dictionary. The function first
        convert the input string to yaml, then yaml load is used to create a
        dictionary.
    Args:
        fru_str (str): Result of running 'ipmitool fru'
    returns: A dictionary who's keys are the FRUs
    """
    yaml_data = []
    lines = string.splitlines()
    for i, line in enumerate(lines):
        # Strip out any excess white space (including tabs) around the ':'
        line = re.sub(r'\s*:\s*', ': ', line)
        # Check for blank lines
        if re.search(r'^\s*$', line):
            yaml_data.append(line)
            continue
        if i < len(lines) - 1:
            # If indentation is increasing on the following line, then convert
            # the current line to a dictionary key.
            indent = re.search(r'[ \t]*', line).span()[1]
            next_indent = re.search(r'[ \t]*', lines[i + 1]).span()[1]
            if next_indent > indent:
                line = re.sub(r'\s*:\s*', ':', line)
                # if ':' in middle of line take the second half, else
                # take the beginning
                if line.split(':')[1]:
                    line = line.split(':')[1]
                else:
                    line = line.split(':')[0]
                yaml_data.append(line + ':')
            else:
                split = line.split(':', 1)
                # Add quotes around the value to handle non alphanumerics
                line = split[0] + ': "' + split[1] + '"'
                yaml_data.append(line)
    yaml_data = '\n'.join(yaml_data)
    return yaml.full_load(yaml_data) 

def _load_file(self, f):
        """Get values from config file"""
        try:
            with open(f, 'r') as _fo:
                _seria_in = seria.load(_fo)
                _y = _seria_in.dump('yaml')
        except IOError:
            raise FiggypyError("could not open configuration file")
        self.values.update(yaml.full_load(_y))