Python os.path() Examples

def preview_filename(self, path):
        """Generate a filename for previews, appending a number when the file already exists.
        """
        if not os.path.exists(path):
            return path

        pcounter = SourcePlugin._COUNTER
        if not path in pcounter:
            pcounter[path] = 1
        fname, ext = os.path.splitext(path)
        counter = pcounter[path]

        while os.path.exists("{}_{}{}".format(fname, counter, ext)):
            counter += 1
        pcounter[path] = counter

        return "{}_{}{}".format(fname, counter, ext) 

def load(self, model_dir, architecture, image_size):
        from keras.models import load_model
        from vergeml.sources.features import get_preprocess_input
        labels_txt = os.path.join(model_dir, "labels.txt")
        if not os.path.exists(labels_txt):
            raise VergeMLError("labels.txt not found: {}".format(labels_txt))

        model_h5 = os.path.join(model_dir, "model.h5")
        if not os.path.exists(model_h5):
            raise VergeMLError("model.h5 not found: {}".format(model_h5))

        with open(labels_txt, "r") as f:
            self.labels = f.read().splitlines()

        self.model = load_model(model_h5)
        self.image_size = image_size
        self.preprocess_input = get_preprocess_input(architecture) 

def predict(self, f, k=5, resize_mode='fill'):
        from keras.preprocessing import image
        from vergeml.img import resize_image

        filename = os.path.basename(f)

        if not os.path.exists(f):
            return dict(filename=filename, prediction=[])

        img = image.load_img(f)
        img = resize_image(img, self.image_size, self.image_size, 'antialias', resize_mode)

        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = self.preprocess_input(x)
        preds = self.model.predict(x)
        pred = self._decode(preds, top=k)[0]
        prediction=[dict(probability=np.asscalar(perc), label=klass) for _, klass, perc in pred]

        return dict(filename=filename, prediction=prediction) 

def _validate_split(self, project_dir):
        """Validate the split configuration for val and test.
        """

        for split in ('val-split', 'test-split'):
            spltype, splval = parse_split(self._config[split])

            if spltype == 'dir':

                # deal with relative paths
                path = os.path.join(project_dir, splval) if not os.path.isabs(splval) else splval

                if not os.path.exists(path):

                    # raise if path does not exist
                    msg = f"Invalid value for option {split} - no such directory: {splval}"
                    suggestion = f"Please set {split} to a percentage, number or directory."

                    raise VergeMLError(msg, suggestion,
                                       hint_key=split, hint_type='value', help_topic='split') 

def _load_trained_model(self):
        """Load a trained models hyperparameters and results
        """

        train_mod_path = os.path.join(self._config['trainings-dir'], self.trained_model)
        if not os.path.exists(train_mod_path):
            raise VergeMLError("Trained model not found: {}".format(self.trained_model))

        # Merge data.yaml
        data_file = os.path.join(self._config['trainings-dir'], self.trained_model, 'data.yaml')
        if not os.path.exists(data_file):
            raise VergeMLError("data.yaml file not found for {}: {}".format(
                self.trained_model, data_file))

        doc = load_yaml_file(data_file, 'data file')
        self._config.update({
            'hyperparameters': doc.get('hyperparameters', {}),
            'results': doc.get('results', {}),
            'model': doc.get('model')
        })

        self.results = _Results(self, data_file)

        return data_file 

def get_custom_architecture(name, trainings_dir, output_layer):
    from keras.models import load_model, Model
    name = name.lstrip("@")
    model = load_model(os.path.join(trainings_dir, name, 'checkpoints', 'model.h5'))
    try:
        if isinstance(output_layer, int):
            layer = model.layers[output_layer]
        else:
            layer = model.get_layer(output_layer)
    except Exception:
        if isinstance(output_layer, int):
            raise VergeMLError(f'output-layer {output_layer} not found - model has only {len(model.layers)} layers.')
        else:
            candidates = list(map(lambda l: l.name, model.layers))
            raise VergeMLError(f'output-layer named {output_layer} not found.',
                               suggestion=did_you_mean(candidates, output_layer))
    model = Model(inputs=model.input, outputs=layer.output)
    return model 

def load_predictions(env, nclasses):
    path = os.path.join(env.stats_dir(), "predictions.csv")

    if not os.path.exists(path):
        raise FileExistsError(path)

    with open(path, newline='') as csvfile:
        y_score = []
        y_test = []
        csv_reader = csv.reader(csvfile, dialect="excel")
        for row in csv_reader:
            assert len(row) == nclasses * 2
            y_score.append(list(map(float, row[:nclasses])))
            y_test.append(list(map(float, row[nclasses:])))
        
        y_score = np.array(y_score)
        y_test = np.array(y_test)

        return y_test, y_score 

def __call__(self, args, env):
        samples_dir = env.get('samples-dir')

        print("Downloading unique objects to {}.".format(samples_dir))

        src_dir = self.download_files([_URL], env=env, dir=env.get('cache-dir'))
        path = os.path.join(src_dir, "ObjectsAll.zip")

        zipf = zipfile.ZipFile(path, 'r')
        zipf.extractall(src_dir)
        zipf.close()

        for file in os.listdir(os.path.join(src_dir, "OBJECTSALL")):
            shutil.copy(os.path.join(src_dir, "OBJECTSALL", file), samples_dir)

        shutil.rmtree(src_dir)

        print("Finished downloading unique objects.") 

def add_base_arguments(parser, default_help):
    import os
    from os.path import join as path_join

    home = os.environ.get('HOME')
    mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '')

    parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help)
    # --jobs supports not passing an argument, in which case the 'const' is used,
    # which is the number of CPU cores on the host system.
    parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help)
    parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help)
    parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help)

    if mono_sources_default:
        parser.add_argument('--mono-sources', default=mono_sources_default, help=default_help)
    else:
        parser.add_argument('--mono-sources', required=True)

    parser.add_argument('--mxe-prefix', default='/usr', help=default_help) 

def find_executable(name) -> str:
    is_windows = os.name == 'nt'
    windows_exts = os.environ['PATHEXT'].split(ENV_PATH_SEP) if is_windows else None
    path_dirs = os.environ['PATH'].split(ENV_PATH_SEP)

    search_dirs = path_dirs + [os.getcwd()] # cwd is last in the list

    for dir in search_dirs:
        path = os.path.join(dir, name)

        if is_windows:
            for extension in windows_exts:
                path_with_ext = path + extension

                if os.path.isfile(path_with_ext) and os.access(path_with_ext, os.X_OK):
                    return path_with_ext
        else:
            if os.path.isfile(path) and os.access(path, os.X_OK):
                return path

    return '' 

def configure(opts: AndroidOpts, product: str, target: str):
    env = { 'ANDROID_API_VERSION': get_api_version_or_min(opts, target) }

    if is_cross(target):
        import llvm

        if is_cross_mxe(target):
            llvm.make(opts, 'llvmwin64')
            setup_android_cross_mxe_template(env, opts, target, host_arch='x86_64')
        else:
            llvm.make(opts, 'llvm64')
            setup_android_cross_template(env, opts, target, host_arch='x86_64')
    else:
        make_standalone_toolchain(opts, target, env['ANDROID_API_VERSION'])
        setup_android_target_template(env, opts, target)

    if not os.path.isfile(path_join(opts.mono_source_root, 'configure')):
        runtime.run_autogen(opts)

    runtime.run_configure(env, opts, product, target) 

def import_helper():
    from os.path import dirname
    import imp
    possible_libs = ["_alf_grammar.win32",
                     "_alf_grammar.ntoarm",
                     "_alf_grammar.ntox86",
                     "_alf_grammar.linux"]
    found_lib = False
    for i in possible_libs:
        fp = None
        try:
            fp, pathname, description = imp.find_module(i, [dirname(__file__)])
            _mod = imp.load_module("_alf_grammar", fp, pathname, description)
            found_lib = True
            break
        except ImportError:
            pass
        finally:
            if fp:
                fp.close()
    if not found_lib:
        raise ImportError("Failed to load _alf_grammar module")
    return _mod 

def cachedir(self):
        """Path to workflow's cache directory.

        The cache directory is a subdirectory of Alfred's own cache directory
        in ``~/Library/Caches``. The full path is:

        ``~/Library/Caches/com.runningwithcrayons.Alfred-X/Workflow Data/<bundle id>``

        ``Alfred-X`` may be ``Alfred-2`` or ``Alfred-3``.

        :returns: full path to workflow's cache directory
        :rtype: ``unicode``

        """
        if self.alfred_env.get('workflow_cache'):
            dirpath = self.alfred_env.get('workflow_cache')

        else:
            dirpath = self._default_cachedir

        return self._create(dirpath) 

def cache_data(self, name, data):
        """Save ``data`` to cache under ``name``.

        If ``data`` is ``None``, the corresponding cache file will be
        deleted.

        :param name: name of datastore
        :param data: data to store. This may be any object supported by
                the cache serializer

        """
        serializer = manager.serializer(self.cache_serializer)

        cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer))

        if data is None:
            if os.path.exists(cache_path):
                os.unlink(cache_path)
                self.logger.debug('deleted cache file: %s', cache_path)
            return

        with atomic_writer(cache_path, 'wb') as file_obj:
            serializer.dump(data, file_obj)

        self.logger.debug('cached data: %s', cache_path) 

def _delete_directory_contents(self, dirpath, filter_func):
        """Delete all files in a directory.

        :param dirpath: path to directory to clear
        :type dirpath: ``unicode`` or ``str``
        :param filter_func function to determine whether a file shall be
            deleted or not.
        :type filter_func ``callable``

        """
        if os.path.exists(dirpath):
            for filename in os.listdir(dirpath):
                if not filter_func(filename):
                    continue
                path = os.path.join(dirpath, filename)
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
                self.logger.debug('deleted : %r', path) 

def __call__(self, args, env):

        # Parse and partition into normal and comparison args.
        args, cargs = _parse_args(args, env)

        # When trainings dir does not exist, print an error and exit
        if not os.path.exists(env.get('trainings-dir')):
            print("No trainings found.", file=sys.stderr)
            return

        info, hyper = _find_trained_models(args, env)

        theader, tdata, left_align = _format_table(args, cargs, info, hyper)

        _output_table(args['output'], theader, tdata, left_align) 

def _find_trained_models(args, env):
    info = {}
    hyper = {}
    train_dir = env.get('trainings-dir')

    for trained_model in os.listdir(train_dir):
        data_yaml = os.path.join(train_dir, trained_model, 'data.yaml')
        if os.path.isfile(data_yaml):
            with open(data_yaml) as file:
                doc = yaml.safe_load(file)
        else:
            doc = {}
        info[trained_model] = {}
        hyper[trained_model] = {}

        if 'model' in doc:
            info[trained_model]['model'] = doc['model']

        if 'results' in doc:
            info[trained_model].update(doc['results'])

        if 'hyperparameters' in doc:
            hyper[trained_model].update(doc['hyperparameters'])

    sort = [s.strip() for s in args['sort'].split(",")]

    info = OrderedDict(sorted(info.items(), reverse=(args['order'] == 'asc'),
                              key=lambda x: [x[1].get(s, 0) for s in sort]))
    return info, hyper 

def scan(self, path, exclude=[]) -> List[str]:
        """Scan path for matching files.

        :param path: the path to scan
        :param exclude: a list of directories to exclude

        :return: a list of sorted filenames
        """
        res = []
        path = path.rstrip("/").rstrip("\\")
        for pat in self.input_patterns:
            res.extend(glob.glob(path + os.sep + pat, recursive=True))

        res = list(filter(lambda p: os.path.isfile(p), res))

        if exclude:
            def excluded(path):
                for e in exclude:
                    if path.startswith(e):
                        return True
                return False

            res = list(filter(lambda p: not excluded(p), res))

        return sorted(res) 

def hash_files(self, files):
        """A default implementation for hash based on files
        """
        if not self._cached_file_state:
            self._cached_file_state = io.BytesIO()

            for (split, files) in files.items():
                for path, _ in files:
                    fstate = "{}{}{}{}".format(split, path, os.path.getmtime(path), os.path.getsize(path))
                    self._cached_file_state.write(fstate.encode('utf-8'))

        return self._cached_file_state.getvalue().decode("utf-8") 

def _save(model, base_model, layers, labels, random_seed, checkpoints_dir):
    from keras.layers import Flatten, Dense
    from keras import Model
    nclasses = len(labels)
    x = Flatten()(base_model.output)
    x = _makenet(x, layers, dropout=None, random_seed=random_seed)
    predictions = Dense(nclasses, activation="softmax", name="predictions")(x)
    model_final = Model(inputs=base_model.input, outputs=predictions)

    for i in range(layers - 1):
        weights = model.get_layer(name='dense_layer_{}'.format(i)).get_weights()
        model_final.get_layer(name='dense_layer_{}'.format(i)).set_weights(weights)

    weights = model.get_layer(name='predictions').get_weights()
    model_final.get_layer(name='predictions').set_weights(weights)

    model_final.save(os.path.join(checkpoints_dir, "model.h5"))
    with open(os.path.join(checkpoints_dir, "labels.txt"), "w") as f:
        f.write("\n".join(labels))
    return model_final 

def get(self, path):
        """Get a value by its path.

        For example, to access the variable 'id' in the dict 'device', use device.id as path.
        """
        return deepcopy(dict_get_path(self._config, path, None)) 

def set(self, path, value):
        """Set a value by its path.
        """
        dict_set_path(self._config, path, value) 

def samples_dir(self):
        """Return the samples_dir or throw an error if it does not exist.
        """
        samples_dir = self._config['samples-dir']
        if not os.path.exists(samples_dir):
            raise VergeMLError(f'Could not find samples directory: {samples_dir}')
        elif not os.path.isdir(samples_dir):
            raise VergeMLError(f'Configured samples-dir is not a directory: {samples_dir}')
        return samples_dir 

def trained_model_dir(self):
        """Return the directory of a trained model (create if it does not exist).
        """
        assert self.trained_model, "You must create a trained model first."
        trained_model_dir = os.path.join(self._config['trainings-dir'], self.trained_model)
        try:
            os.makedirs(trained_model_dir)
        except FileExistsError:
            pass
        return trained_model_dir 

def checkpoints_dir(self):
        """Return the checkpoints directory (create if it does not exist)
        """
        checkpoints_dir = os.path.join(self.trained_model_dir(), "checkpoints")
        try:
            os.makedirs(checkpoints_dir)
        except FileExistsError:
            pass
        return checkpoints_dir 

def __init__(self, env, path):
        self.path = path
        self.env = env
        self.last_sync = None 

def _sync(self, force=False):
        now = datetime.datetime.now()
        if force or not self.last_sync or (now - self.last_sync).total_seconds() > _SYNC_INTV:
            self.last_sync = now
            data = dict(model=self.env.get('model'),
                        hyperparameters=self._convert(self.env.get('hyperparameters')),
                        results=self._convert(self.env.get('results')))
            with open(self.path, "w") as f:
                yaml.dump(data, f, default_flow_style=False) 

def __init__(self, path):
        self.data = {}
        self.path = path
        if os.path.exists(path):
            with open(path) as f:
                self.data = json.load(f) 

def write_preview(self, output_dir: str, split: str, sample: Sample):
        filename = sample.meta['filename']
        
        name = fixext(os.path.basename(filename), sample.x)
        split_dir = os.path.join(output_dir, split)
        if not os.path.exists(split_dir):
            os.makedirs(split_dir)

        path = self.preview_filename(os.path.join(split_dir, name))
        sample.x.save(path)
        return path 

def evaluate_args(architecture, trainings_dir, variant, alpha, size):

    if not architecture.startswith('@') and not architecture in ARCHITECTURES:
        raise VergeMLError("Unknown CNN: {}".format(architecture))
    elif architecture.startswith('@'):
        name = architecture.lstrip('@')
        path = os.path.join(trainings_dir, name, 'checkpoints', 'model.h5')
        if not os.path.isfile(path):
            raise VergeMLError("Unknown CNN: {}".format(architecture))

    if architecture == 'densenet':
        if variant == 'auto':
            variant = DENSENET_VARIANTS[0]
        if not variant in DENSENET_VARIANTS:
            raise VergeMLError("Invalid densenet variant: {}".format(variant))

    elif architecture == 'mobilenet':
        if size not in MOBILENET_SIZES and size != "auto":
            raise VergeMLError("Invalid mobilenet size: {}".format(size))
        if alpha not in MOBILENET_ALPHA_VALUES:
            raise VergeMLError("Invalid alpha value: {}".format(alpha))

    elif architecture == 'mobilenet-v2':
        if size not in MOBILENET_V2_SIZES and size != "auto":
            raise VergeMLError("Invalid mobilenet size: {}".format(size))

        if alpha not in MOBILENET_V2_ALPHA_VALUES:
            raise VergeMLError("Invalid alpha value: {}".format(alpha))

    elif architecture == 'nasnet':
        if not variant in NASNET_VARIANTS and variant != 'auto':
            raise VergeMLError("Invalid nasnet variant: {}".format(variant))