Python yaml.load() Examples

def main():
    if len(sys.argv) > 2:
        Fatal('usage: %s [containers.yaml]' % sys.argv[0])

    if len(sys.argv) == 2:
        with open(sys.argv[1], 'r') as fp:
            config = yaml.load(fp)
    else:
        config = yaml.load(sys.stdin)

    syslog.openlog(PROGNAME)
    LogInfo('processing container manifest')

    CheckVersion(config)

    all_volumes = LoadVolumes(config.get('volumes', []))
    user_containers = LoadUserContainers(config.get('containers', []),
                                         all_volumes)
    CheckGroupWideConflicts(user_containers)

    if user_containers:
        infra_containers = LoadInfraContainers(user_containers)
        RunContainers(infra_containers + user_containers) 

def __read_settings(self, yaml_file):
        if yaml_file is None:
            yaml_file = os.path.join(ciftify.config.find_ciftify_global(),
                    'ciftify_workflow_settings.yaml')
        if not os.path.exists(yaml_file):
            logger.critical("Settings yaml file {} does not exist"
                "".format(yaml_file))
            sys.exit(1)

        try:
            with open(yaml_file, 'r') as yaml_stream:
                config = yaml.load(yaml_stream, Loader=yaml.SafeLoader)
        except:
            logger.critical("Cannot read yaml config file {}, check formatting."
                    "".format(yaml_file))
            sys.exit(1)

        return config 

def show_cfg(resource_url, escape='##'):
    """
    Write out a grammar file, ignoring escaped and empty lines.

    :type resource_url: str
    :param resource_url: A URL specifying where the resource should be
        loaded from.  The default protocol is "nltk:", which searches
        for the file in the the NLTK data package.
    :type escape: str
    :param escape: Prepended string that signals lines to be ignored
    """
    resource_url = normalize_resource_url(resource_url)
    resource_val = load(resource_url, format='text', cache=False)
    lines = resource_val.splitlines()
    for l in lines:
        if l.startswith(escape):
            continue
        if re.match('^$', l):
            continue
        print(l) 

def __read_mode(self, mode):
        logger = logging.getLogger(__name__)
        ciftify_data = config.find_ciftify_global()
        qc_settings = os.path.join(ciftify_data, 'qc_modes.yaml')
        try:
            with open(qc_settings, 'r') as qc_stream:
                qc_modes = yaml.load(qc_stream, Loader=yaml.SafeLoader)
        except:
            logger.error("Cannot read qc_modes file: {}".format(qc_settings))
            sys.exit(1)
        try:
            settings = qc_modes[mode]
        except KeyError:
            logger.error("qc_modes file {} does not define mode {}"
                    "".format(qc_settings, mode))
            sys.exit(1)
        return settings 

def symptom_codes(group):
    """
    Returns CompTIA symptom codes for component group
    """
    if group == '':
        return

    try:
        symptoms = get_remote_symptom_codes(group)
    except Exception as e:
        # ... finally fall back to local static data
        # @FIXME: How do we keep this up to date?
        data = yaml.load(open("servo/fixtures/comptia.yaml", "r"))
        symptoms = data[group]['symptoms']

    codes = [(k, "%s - %s " % (k, symptoms[k])) for k in sorted(symptoms)]
    return codes 

def get_remote_symptom_codes(group):
    """
    Remote lookup for symptom codes
    """
    symptoms = {}
    cache = caches['comptia']
    # First, try to load from global cache (updated every 24h)
    data = cache.get('codes') or {}

    if not data:
        # ... then try to fetch from GSX
        GsxAccount.fallback()
        data = gsxws.comptia.fetch()
        cache.set('codes', data)

    for k, v in data.get(group):
        symptoms[k] = v

    return symptoms 

def readYaml(env=None):
    """Load the skelebot.yaml, with environment overrride if present, into the Config object"""

    yamlData = None
    cwd = os.getcwd()
    cfgFile = FILE_PATH.format(path=cwd)
    if os.path.isfile(cfgFile):
        with open(cfgFile, 'r') as stream:
            yamlData = yaml.load(stream, Loader=yaml.FullLoader)
            if (env is not None):
                envFile = ENV_FILE_PATH.format(path=cwd, env=env)
                if os.path.isfile(envFile):
                    with open(envFile, 'r') as stream:
                        overrideYaml = yaml.load(stream, Loader=yaml.FullLoader)
                        yamlData = override(yamlData, overrideYaml)
                else:
                    raise RuntimeError("Environment Not Found")

    return yamlData 

def import_files(self):
        print("self.manifest_path", self.manifest_path)
        self.manifest = yaml.load(open("{}/manifest.yml".format(self.manifest_path)), Loader=yaml.FullLoader)
        sys.path.append(self.manifest_path)
        """
        don't remove the import below, this will be the cti_transformations.py,
        which is one of the required file to run the job. This file will be provided by the 
        user during the run.
        """
        try:
            import ib_functions
        except Exception as e:
            ib_functions = None
        self.ib_functions = ib_functions
        print ("self.ib_functions is {}".format(ib_functions))
        # print("manifest is {}".format(self.manifest))
        # print("ib_functions is {}".format(self.ib_functions)) 

def get_config(config_file, exp_dir=None):
  """ Construct and snapshot hyper parameters """
  config = edict(yaml.load(open(config_file, 'r')))

  # create hyper parameters
  config.run_id = str(os.getpid())
  config.exp_name = '_'.join([
      config.model.name, config.dataset.name,
      time.strftime('%Y-%b-%d-%H-%M-%S'), config.run_id
  ])

  if exp_dir is not None:
    config.exp_dir = exp_dir

  config.save_dir = os.path.join(config.exp_dir, config.exp_name)

  # snapshot hyperparameters
  mkdir(config.exp_dir)
  mkdir(config.save_dir)

  save_name = os.path.join(config.save_dir, 'config.yaml')
  yaml.dump(edict2dict(config), open(save_name, 'w'), default_flow_style=False)

  return config 

def config(monkeypatch):
    yaml_string = """

wasp:
  setting1: 1  # int
  foo: bar  # string

database:
  username: normal_user

  migration:
    username: migration_user

flat: true  # boolean
flat_with_underscores: hello
"""
    def my_load_config(self):
        self.default_options = yaml.load(io.StringIO(yaml_string))

    monkeypatch.setattr(Config, '_load_config', my_load_config)
    config_ = Config()
    config_.load()
    return config_ 

def make_secondary_dl_yaml(template_yaml, model_json, output_yaml_path):
    with open(template_yaml, 'r') as f:
        model_yaml = yaml.load(f)
    #
    # get the model config:
    json_file = open(model_json, 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = keras.models.model_from_json(loaded_model_json)
    #
    model_yaml["output_schema"]["targets"] = []
    for oname, oshape in zip(loaded_model.output_names, loaded_model.output_shape):
        append_el ={"name":oname , "shape":str(oshape)#replace("None,", "")
        , "doc":"Methylation probability for %s"%oname}
        model_yaml["output_schema"]["targets"].append(append_el)
    #
    with open(output_yaml_path, 'w') as f:
        yaml.dump(model_yaml, f, default_flow_style=False) 

def make_model_yaml(template_yaml, model_json, output_yaml_path):
    #
    with open(template_yaml, 'r') as f:
        model_yaml = yaml.load(f)
    #
    # get the model config:
    json_file = open(model_json, 'r')
    loaded_model_json = json_file.read()
    json_file.close()
    loaded_model = keras.models.model_from_json(loaded_model_json)
    #
    model_yaml["schema"]["targets"] = []
    for oname, oshape in zip(loaded_model.output_names, loaded_model.output_shape):
        append_el ={"name":oname , "shape":str(oshape)#replace("None,", "")
        , "doc":"Methylation probability for %s"%oname}
        model_yaml["schema"]["targets"].append(append_el)
    #
    with open(output_yaml_path, 'w') as f:
        yaml.dump(model_yaml, f, default_flow_style=False) 

def __init__(self, spec_url):
        """Create a new URLLoader.

        Keyword arguments:
        spec_url -- URL where the specification YAML file is located."""
        headers = {'Accept' : 'text/yaml'}
        resp = requests.get(spec_url, headers=headers)
        if resp.status_code == 200:
            self.spec = yaml.load(resp.text)
            self.spec['url'] = spec_url
            self.spec['files'] = {}
            for queryUrl in self.spec['queries']:
                queryNameExt = path.basename(queryUrl)
                queryName = path.splitext(queryNameExt)[0] # Remove extention
                item = {
                    'name': queryName,
                    'download_url': queryUrl
                }
                self.spec['files'][queryNameExt] = item
            del self.spec['queries']
        else:
            raise Exception(resp.text) 

def load_test_checkpoints(model, save_path, logger, use_best=False):
    
    #try:
    if use_best:
        print(save_path.EXPS+save_path.NAME+save_path.BESTMODEL)
        states= torch.load(save_path.EXPS+save_path.NAME+save_path.BESTMODEL) if torch.cuda.is_available() \
            else torch.load(save_path.EXPS+save_path.NAME+save_path.BESTMODEL, map_location=torch.device('cpu'))
    else:   
        states= torch.load(save_path.EXPS+save_path.NAME+save_path.MODEL) if torch.cuda.is_available() \
            else torch.load(save_path.EXPS+save_path.NAME+save_path.MODEL, map_location=torch.device('cpu'))
    #logger.debug("success")
    #try:
    model.load_state_dict(states['model_state'])
    # except:
    #     states_no_module = OrderedDict()
    #     for k, v in states['model_state'].items():
    #         name_no_module = k[7:]
    #         states_no_module[name_no_module] = v
    #     model.load_state_dict(states_no_module)
    logger.info('loading checkpoints success')
    # except:
    #     logger.error("no checkpoints") 

def setup(self, bottom, top):
        layer_params = yaml.load(self.param_str)
        self._layer_params = layer_params
        # default batch_size = 256
        self._batch_size = int(layer_params.get('batch_size', 256))
        self._resize = layer_params.get('resize', -1)
        self._mean_file = layer_params.get('mean_file', None)
        self._source_type = layer_params.get('source_type', 'CSV')
        self._shuffle = layer_params.get('shuffle', False)
        # read image_mean from file and preload all data into memory
        # will read either file or array into self._mean
        self.set_mean()
        self.preload_db()
        self._compressed = self._layer_params.get('compressed', True)
        if not self._compressed:
            self.decompress_data() 

def load_yaml_file(filename, label='config file', loader=yaml.Loader):
    """Load a yaml config file.
    """
    try:
        with open(filename, "r") as file:
            res = yaml.load(file.read(), Loader=loader) or {}
            if not isinstance(res, dict):
                msg = f"Please ensure that {label} consists of key value pairs."
                raise VergeMLError(f"Invalid {label}: {filename}", msg)
            return res
    except yaml.YAMLError as err:
        if hasattr(err, 'problem_mark'):
            mark = getattr(err, 'problem_mark')
            problem = getattr(err, 'problem')
            message = f"Could not read {label} {filename}:"
            message += "\n" + display_err_in_file(filename, mark.line, mark.column, problem)
        elif hasattr(err, 'problem'):
            problem = getattr(err, 'problem')
            message = f"Could not read {label} {filename}: {problem}"
        else:
            message = f"Could not read {label} {filename}: YAML Error"

        suggestion = f"There is a syntax error in your {label} - please fix it and try again."

        raise VergeMLError(message, suggestion)

    except OSError as err:
        msg = "Please ensure the file exists and you have the required access privileges."
        raise VergeMLError(f"Could not open {label} {filename}: {err.strerror}", msg) 

def set_mean(self):
        if self._mean_file:
            if type(self._mean_file) is str:
                # read image mean from file
                try:
                    # if it is a pickle file
                    self._mean = np.load(self._mean_file)
                except (IOError):
                    blob = caffe_pb2.BlobProto()
                    blob_str = open(self._mean_file, 'rb').read()
                    blob.ParseFromString(blob_str)
                    self._mean = np.array(caffe.io.blobproto_to_array(blob))[0]
            else:
                self._mean = self._mean_file
                self._mean = np.array(self._mean)
        else:
            self._mean = None 

def testContainerValidMinimal(self):
        yaml_code = """
      - name: abc123
        image: foo/bar
      - name: abc124
        image: foo/bar
      """
        user = run_containers.LoadUserContainers(yaml.load(yaml_code), [])
        self.assertEqual(2, len(user))
        self.assertEqual('abc123', user[0].name)
        self.assertEqual('abc124', user[1].name)

        infra = run_containers.LoadInfraContainers(user)
        self.assertEqual(1, len(infra))
        self.assertEqual('.net', infra[0].name) 

def __get_settings(self):
        default_config = os.path.join(ciftify.config.find_ciftify_global(),
                'qc_modes.yaml')
        with open(default_config, 'r') as qc_stream:
            settings = yaml.load(qc_stream, Loader=yaml.SafeLoader)
        return settings 

def testVolumeDupName(self):
        yaml_code = """
      - name: abc123
      - name: abc123
      """
        with self.assertRaises(SystemExit):
            run_containers.LoadVolumes(yaml.load(yaml_code)) 

def load_ndarray(path_to_file):
    try:
        with open(path_to_file, 'rb') as f:
            data = np.load(f)
    except Exception as e:
        raise e

    return data 

def read_rst_yaml (filename=None):
    import yaml, os
    if filename is None: 
        path = os.getcwd()
        filename =find('*.yaml', path)
    with open(filename, 'r') as stream:
        try:
            data = yaml.load(stream)
            msg = 'RESTART: Full matrix elements of screened interactions (W_c) was read from {}'.format(filename)
            return data, msg
        except yaml.YAMLError as exc:
            return exc 

def __init__(self, n_dims, template_dir=None, upgrade_templates=False):
        super(CachedFileTemplateAdapter, self).__init__(
            n_dims,
            template_dir=template_dir
        )

        # Handle those before generating cache as we want to load them if
        # upgrade_templates is True
        FileTemplateAdapter.handle_old_templates(
            self, upgrade_templates=upgrade_templates)

        self._cache = {lm_id: FileTemplateAdapter.load_template(self, lm_id)
                       for lm_id in FileTemplateAdapter.template_ids(self)}
        print('cached {} templates ({})'.format(
            len(self._cache), ', '.join(self._cache.keys()))) 

def load_yaml_template(filepath, n_dims):
    with open(filepath) as f:
        data = yaml.load(f.read())

        if 'groups' in data:
            raw_groups = data['groups']
        else:
            raise KeyError(
                "Missing 'groups' or 'template' key in yaml file %s"
                % filepath)

        groups = []

        for index, group in enumerate(raw_groups):

            label = group.get('label', index)  # Allow simple ordered groups

            n = group['points']  # Should raise KeyError by design if missing
            connectivity = group.get('connectivity', [])

            if isinstance(connectivity, list):
                index = parse_connectivity(connectivity, n)
            elif connectivity == 'cycle':
                index = parse_connectivity(
                    ['0:%d' % (n - 1), '%d 0' % (n - 1)], n)
            else:
                index = []  # Couldn't parse connectivity, safe default

            groups.append(Group(label, n, index))

    return build_json(groups, n_dims) 

def parse_yaml(filename):
    with open(filename, 'r') as fin:
        test_data = yaml.load(fin)
    return test_data 

def from_yaml(cls, path: str):
        with open(path, "rb") as fp:
            config = yaml.load(fp, Loader=yaml.FullLoader)

        return TradingContext(config) 

def from_json(cls, path: str):
        with open(path, "rb") as fp:
            config = json.load(fp)

        return TradingContext(config) 

def load_gzip_json(file):
    try:
        with gzip.open(file, 'r') as f:
            data = json.load(f)
    except Exception as e:
        raise e
    return data 

def load_json(file):
    try:
        with open(file, 'r') as f:
            data = json.load(f)
    except Exception as e:
        raise e
    return data 

def testContainerWithoutCommand(self):
        yaml_code = """
      - name: abc123
        image: foo/bar
      """
        x = run_containers.LoadUserContainers(yaml.load(yaml_code), [])
        self.assertEqual(1, len(x))
        self.assertEqual(0, len(x[0].command))