Python os.path() Examples

The following are 30 code examples for showing how to use os.path(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module os , or try the search function .

Example 1
Project: vergeml   Author: mme   File: io.py    License: MIT License 6 votes vote down vote up
def scan(self, path, exclude=[]) -> List[str]:
        """Scan path for matching files.

        :param path: the path to scan
        :param exclude: a list of directories to exclude

        :return: a list of sorted filenames
        """
        res = []
        path = path.rstrip("/").rstrip("\\")
        for pat in self.input_patterns:
            res.extend(glob.glob(path + os.sep + pat, recursive=True))

        res = list(filter(lambda p: os.path.isfile(p), res))

        if exclude:
            def excluded(path):
                for e in exclude:
                    if path.startswith(e):
                        return True
                return False

            res = list(filter(lambda p: not excluded(p), res))

        return sorted(res) 
Example 2
Project: vergeml   Author: mme   File: io.py    License: MIT License 6 votes vote down vote up
def preview_filename(self, path):
        """Generate a filename for previews, appending a number when the file already exists.
        """
        if not os.path.exists(path):
            return path

        pcounter = SourcePlugin._COUNTER
        if not path in pcounter:
            pcounter[path] = 1
        fname, ext = os.path.splitext(path)
        counter = pcounter[path]

        while os.path.exists("{}_{}{}".format(fname, counter, ext)):
            counter += 1
        pcounter[path] = counter

        return "{}_{}{}".format(fname, counter, ext) 
Example 3
Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 6 votes vote down vote up
def load(self, model_dir, architecture, image_size):
        from keras.models import load_model
        from vergeml.sources.features import get_preprocess_input
        labels_txt = os.path.join(model_dir, "labels.txt")
        if not os.path.exists(labels_txt):
            raise VergeMLError("labels.txt not found: {}".format(labels_txt))

        model_h5 = os.path.join(model_dir, "model.h5")
        if not os.path.exists(model_h5):
            raise VergeMLError("model.h5 not found: {}".format(model_h5))

        with open(labels_txt, "r") as f:
            self.labels = f.read().splitlines()

        self.model = load_model(model_h5)
        self.image_size = image_size
        self.preprocess_input = get_preprocess_input(architecture) 
Example 4
Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 6 votes vote down vote up
def predict(self, f, k=5, resize_mode='fill'):
        from keras.preprocessing import image
        from vergeml.img import resize_image

        filename = os.path.basename(f)

        if not os.path.exists(f):
            return dict(filename=filename, prediction=[])

        img = image.load_img(f)
        img = resize_image(img, self.image_size, self.image_size, 'antialias', resize_mode)

        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = self.preprocess_input(x)
        preds = self.model.predict(x)
        pred = self._decode(preds, top=k)[0]
        prediction=[dict(probability=np.asscalar(perc), label=klass) for _, klass, perc in pred]

        return dict(filename=filename, prediction=prediction) 
Example 5
Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 6 votes vote down vote up
def _save(model, base_model, layers, labels, random_seed, checkpoints_dir):
    from keras.layers import Flatten, Dense
    from keras import Model
    nclasses = len(labels)
    x = Flatten()(base_model.output)
    x = _makenet(x, layers, dropout=None, random_seed=random_seed)
    predictions = Dense(nclasses, activation="softmax", name="predictions")(x)
    model_final = Model(inputs=base_model.input, outputs=predictions)

    for i in range(layers - 1):
        weights = model.get_layer(name='dense_layer_{}'.format(i)).get_weights()
        model_final.get_layer(name='dense_layer_{}'.format(i)).set_weights(weights)

    weights = model.get_layer(name='predictions').get_weights()
    model_final.get_layer(name='predictions').set_weights(weights)

    model_final.save(os.path.join(checkpoints_dir, "model.h5"))
    with open(os.path.join(checkpoints_dir, "labels.txt"), "w") as f:
        f.write("\n".join(labels))
    return model_final 
Example 6
Project: vergeml   Author: mme   File: env.py    License: MIT License 6 votes vote down vote up
def _validate_split(self, project_dir):
        """Validate the split configuration for val and test.
        """

        for split in ('val-split', 'test-split'):
            spltype, splval = parse_split(self._config[split])

            if spltype == 'dir':

                # deal with relative paths
                path = os.path.join(project_dir, splval) if not os.path.isabs(splval) else splval

                if not os.path.exists(path):

                    # raise if path does not exist
                    msg = f"Invalid value for option {split} - no such directory: {splval}"
                    suggestion = f"Please set {split} to a percentage, number or directory."

                    raise VergeMLError(msg, suggestion,
                                       hint_key=split, hint_type='value', help_topic='split') 
Example 7
Project: vergeml   Author: mme   File: env.py    License: MIT License 6 votes vote down vote up
def _load_trained_model(self):
        """Load a trained models hyperparameters and results
        """

        train_mod_path = os.path.join(self._config['trainings-dir'], self.trained_model)
        if not os.path.exists(train_mod_path):
            raise VergeMLError("Trained model not found: {}".format(self.trained_model))

        # Merge data.yaml
        data_file = os.path.join(self._config['trainings-dir'], self.trained_model, 'data.yaml')
        if not os.path.exists(data_file):
            raise VergeMLError("data.yaml file not found for {}: {}".format(
                self.trained_model, data_file))

        doc = load_yaml_file(data_file, 'data file')
        self._config.update({
            'hyperparameters': doc.get('hyperparameters', {}),
            'results': doc.get('results', {}),
            'model': doc.get('model')
        })

        self.results = _Results(self, data_file)

        return data_file 
Example 8
Project: vergeml   Author: mme   File: features.py    License: MIT License 6 votes vote down vote up
def get_custom_architecture(name, trainings_dir, output_layer):
    from keras.models import load_model, Model
    name = name.lstrip("@")
    model = load_model(os.path.join(trainings_dir, name, 'checkpoints', 'model.h5'))
    try:
        if isinstance(output_layer, int):
            layer = model.layers[output_layer]
        else:
            layer = model.get_layer(output_layer)
    except Exception:
        if isinstance(output_layer, int):
            raise VergeMLError(f'output-layer {output_layer} not found - model has only {len(model.layers)} layers.')
        else:
            candidates = list(map(lambda l: l.name, model.layers))
            raise VergeMLError(f'output-layer named {output_layer} not found.',
                               suggestion=did_you_mean(candidates, output_layer))
    model = Model(inputs=model.input, outputs=layer.output)
    return model 
Example 9
Project: vergeml   Author: mme   File: __init__.py    License: MIT License 6 votes vote down vote up
def load_predictions(env, nclasses):
    path = os.path.join(env.stats_dir(), "predictions.csv")

    if not os.path.exists(path):
        raise FileExistsError(path)

    with open(path, newline='') as csvfile:
        y_score = []
        y_test = []
        csv_reader = csv.reader(csvfile, dialect="excel")
        for row in csv_reader:
            assert len(row) == nclasses * 2
            y_score.append(list(map(float, row[:nclasses])))
            y_test.append(list(map(float, row[nclasses:])))
        
        y_score = np.array(y_score)
        y_test = np.array(y_test)

        return y_test, y_score 
Example 10
Project: vergeml   Author: mme   File: unique_objects.py    License: MIT License 6 votes vote down vote up
def __call__(self, args, env):
        samples_dir = env.get('samples-dir')

        print("Downloading unique objects to {}.".format(samples_dir))

        src_dir = self.download_files([_URL], env=env, dir=env.get('cache-dir'))
        path = os.path.join(src_dir, "ObjectsAll.zip")

        zipf = zipfile.ZipFile(path, 'r')
        zipf.extractall(src_dir)
        zipf.close()

        for file in os.listdir(os.path.join(src_dir, "OBJECTSALL")):
            shutil.copy(os.path.join(src_dir, "OBJECTSALL", file), samples_dir)

        shutil.rmtree(src_dir)

        print("Finished downloading unique objects.") 
Example 11
Project: godot-mono-builds   Author: godotengine   File: cmd_utils.py    License: MIT License 6 votes vote down vote up
def add_base_arguments(parser, default_help):
    import os
    from os.path import join as path_join

    home = os.environ.get('HOME')
    mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '')

    parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help)
    # --jobs supports not passing an argument, in which case the 'const' is used,
    # which is the number of CPU cores on the host system.
    parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help)
    parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help)
    parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help)

    if mono_sources_default:
        parser.add_argument('--mono-sources', default=mono_sources_default, help=default_help)
    else:
        parser.add_argument('--mono-sources', required=True)

    parser.add_argument('--mxe-prefix', default='/usr', help=default_help) 
Example 12
Project: godot-mono-builds   Author: godotengine   File: os_utils.py    License: MIT License 6 votes vote down vote up
def find_executable(name) -> str:
    is_windows = os.name == 'nt'
    windows_exts = os.environ['PATHEXT'].split(ENV_PATH_SEP) if is_windows else None
    path_dirs = os.environ['PATH'].split(ENV_PATH_SEP)

    search_dirs = path_dirs + [os.getcwd()] # cwd is last in the list

    for dir in search_dirs:
        path = os.path.join(dir, name)

        if is_windows:
            for extension in windows_exts:
                path_with_ext = path + extension

                if os.path.isfile(path_with_ext) and os.access(path_with_ext, os.X_OK):
                    return path_with_ext
        else:
            if os.path.isfile(path) and os.access(path, os.X_OK):
                return path

    return '' 
Example 13
Project: godot-mono-builds   Author: godotengine   File: android.py    License: MIT License 6 votes vote down vote up
def configure(opts: AndroidOpts, product: str, target: str):
    env = { 'ANDROID_API_VERSION': get_api_version_or_min(opts, target) }

    if is_cross(target):
        import llvm

        if is_cross_mxe(target):
            llvm.make(opts, 'llvmwin64')
            setup_android_cross_mxe_template(env, opts, target, host_arch='x86_64')
        else:
            llvm.make(opts, 'llvm64')
            setup_android_cross_template(env, opts, target, host_arch='x86_64')
    else:
        make_standalone_toolchain(opts, target, env['ANDROID_API_VERSION'])
        setup_android_target_template(env, opts, target)

    if not os.path.isfile(path_join(opts.mono_source_root, 'configure')):
        runtime.run_autogen(opts)

    runtime.run_configure(env, opts, product, target) 
Example 14
Project: ALF   Author: blackberry   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def import_helper():
    from os.path import dirname
    import imp
    possible_libs = ["_alf_grammar.win32",
                     "_alf_grammar.ntoarm",
                     "_alf_grammar.ntox86",
                     "_alf_grammar.linux"]
    found_lib = False
    for i in possible_libs:
        fp = None
        try:
            fp, pathname, description = imp.find_module(i, [dirname(__file__)])
            _mod = imp.load_module("_alf_grammar", fp, pathname, description)
            found_lib = True
            break
        except ImportError:
            pass
        finally:
            if fp:
                fp.close()
    if not found_lib:
        raise ImportError("Failed to load _alf_grammar module")
    return _mod 
Example 15
Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def cachedir(self):
        """Path to workflow's cache directory.

        The cache directory is a subdirectory of Alfred's own cache directory
        in ``~/Library/Caches``. The full path is:

        ``~/Library/Caches/com.runningwithcrayons.Alfred-X/Workflow Data/<bundle id>``

        ``Alfred-X`` may be ``Alfred-2`` or ``Alfred-3``.

        :returns: full path to workflow's cache directory
        :rtype: ``unicode``

        """
        if self.alfred_env.get('workflow_cache'):
            dirpath = self.alfred_env.get('workflow_cache')

        else:
            dirpath = self._default_cachedir

        return self._create(dirpath) 
Example 16
Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def cache_data(self, name, data):
        """Save ``data`` to cache under ``name``.

        If ``data`` is ``None``, the corresponding cache file will be
        deleted.

        :param name: name of datastore
        :param data: data to store. This may be any object supported by
                the cache serializer

        """
        serializer = manager.serializer(self.cache_serializer)

        cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer))

        if data is None:
            if os.path.exists(cache_path):
                os.unlink(cache_path)
                self.logger.debug('deleted cache file: %s', cache_path)
            return

        with atomic_writer(cache_path, 'wb') as file_obj:
            serializer.dump(data, file_obj)

        self.logger.debug('cached data: %s', cache_path) 
Example 17
Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def _delete_directory_contents(self, dirpath, filter_func):
        """Delete all files in a directory.

        :param dirpath: path to directory to clear
        :type dirpath: ``unicode`` or ``str``
        :param filter_func function to determine whether a file shall be
            deleted or not.
        :type filter_func ``callable``

        """
        if os.path.exists(dirpath):
            for filename in os.listdir(dirpath):
                if not filter_func(filename):
                    continue
                path = os.path.join(dirpath, filename)
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
                self.logger.debug('deleted : %r', path) 
Example 18
Project: vergeml   Author: mme   File: ls.py    License: MIT License 5 votes vote down vote up
def __call__(self, args, env):

        # Parse and partition into normal and comparison args.
        args, cargs = _parse_args(args, env)

        # When trainings dir does not exist, print an error and exit
        if not os.path.exists(env.get('trainings-dir')):
            print("No trainings found.", file=sys.stderr)
            return

        info, hyper = _find_trained_models(args, env)

        theader, tdata, left_align = _format_table(args, cargs, info, hyper)

        _output_table(args['output'], theader, tdata, left_align) 
Example 19
Project: vergeml   Author: mme   File: ls.py    License: MIT License 5 votes vote down vote up
def _find_trained_models(args, env):
    info = {}
    hyper = {}
    train_dir = env.get('trainings-dir')

    for trained_model in os.listdir(train_dir):
        data_yaml = os.path.join(train_dir, trained_model, 'data.yaml')
        if os.path.isfile(data_yaml):
            with open(data_yaml) as file:
                doc = yaml.safe_load(file)
        else:
            doc = {}
        info[trained_model] = {}
        hyper[trained_model] = {}

        if 'model' in doc:
            info[trained_model]['model'] = doc['model']

        if 'results' in doc:
            info[trained_model].update(doc['results'])

        if 'hyperparameters' in doc:
            hyper[trained_model].update(doc['hyperparameters'])

    sort = [s.strip() for s in args['sort'].split(",")]

    info = OrderedDict(sorted(info.items(), reverse=(args['order'] == 'asc'),
                              key=lambda x: [x[1].get(s, 0) for s in sort]))
    return info, hyper 
Example 20
Project: vergeml   Author: mme   File: io.py    License: MIT License 5 votes vote down vote up
def hash_files(self, files):
        """A default implementation for hash based on files
        """
        if not self._cached_file_state:
            self._cached_file_state = io.BytesIO()

            for (split, files) in files.items():
                for path, _ in files:
                    fstate = "{}{}{}{}".format(split, path, os.path.getmtime(path), os.path.getsize(path))
                    self._cached_file_state.write(fstate.encode('utf-8'))

        return self._cached_file_state.getvalue().decode("utf-8") 
Example 21
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def get(self, path):
        """Get a value by its path.

        For example, to access the variable 'id' in the dict 'device', use device.id as path.
        """
        return deepcopy(dict_get_path(self._config, path, None)) 
Example 22
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def set(self, path, value):
        """Set a value by its path.
        """
        dict_set_path(self._config, path, value) 
Example 23
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def samples_dir(self):
        """Return the samples_dir or throw an error if it does not exist.
        """
        samples_dir = self._config['samples-dir']
        if not os.path.exists(samples_dir):
            raise VergeMLError(f'Could not find samples directory: {samples_dir}')
        elif not os.path.isdir(samples_dir):
            raise VergeMLError(f'Configured samples-dir is not a directory: {samples_dir}')
        return samples_dir 
Example 24
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def trained_model_dir(self):
        """Return the directory of a trained model (create if it does not exist).
        """
        assert self.trained_model, "You must create a trained model first."
        trained_model_dir = os.path.join(self._config['trainings-dir'], self.trained_model)
        try:
            os.makedirs(trained_model_dir)
        except FileExistsError:
            pass
        return trained_model_dir 
Example 25
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def checkpoints_dir(self):
        """Return the checkpoints directory (create if it does not exist)
        """
        checkpoints_dir = os.path.join(self.trained_model_dir(), "checkpoints")
        try:
            os.makedirs(checkpoints_dir)
        except FileExistsError:
            pass
        return checkpoints_dir 
Example 26
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def __init__(self, env, path):
        self.path = path
        self.env = env
        self.last_sync = None 
Example 27
Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def _sync(self, force=False):
        now = datetime.datetime.now()
        if force or not self.last_sync or (now - self.last_sync).total_seconds() > _SYNC_INTV:
            self.last_sync = now
            data = dict(model=self.env.get('model'),
                        hyperparameters=self._convert(self.env.get('hyperparameters')),
                        results=self._convert(self.env.get('results')))
            with open(self.path, "w") as f:
                yaml.dump(data, f, default_flow_style=False) 
Example 28
Project: vergeml   Author: mme   File: results.py    License: MIT License 5 votes vote down vote up
def __init__(self, path):
        self.data = {}
        self.path = path
        if os.path.exists(path):
            with open(path) as f:
                self.data = json.load(f) 
Example 29
Project: vergeml   Author: mme   File: image.py    License: MIT License 5 votes vote down vote up
def write_preview(self, output_dir: str, split: str, sample: Sample):
        filename = sample.meta['filename']
        
        name = fixext(os.path.basename(filename), sample.x)
        split_dir = os.path.join(output_dir, split)
        if not os.path.exists(split_dir):
            os.makedirs(split_dir)

        path = self.preview_filename(os.path.join(split_dir, name))
        sample.x.save(path)
        return path 
Example 30
Project: vergeml   Author: mme   File: features.py    License: MIT License 5 votes vote down vote up
def evaluate_args(architecture, trainings_dir, variant, alpha, size):

    if not architecture.startswith('@') and not architecture in ARCHITECTURES:
        raise VergeMLError("Unknown CNN: {}".format(architecture))
    elif architecture.startswith('@'):
        name = architecture.lstrip('@')
        path = os.path.join(trainings_dir, name, 'checkpoints', 'model.h5')
        if not os.path.isfile(path):
            raise VergeMLError("Unknown CNN: {}".format(architecture))

    if architecture == 'densenet':
        if variant == 'auto':
            variant = DENSENET_VARIANTS[0]
        if not variant in DENSENET_VARIANTS:
            raise VergeMLError("Invalid densenet variant: {}".format(variant))

    elif architecture == 'mobilenet':
        if size not in MOBILENET_SIZES and size != "auto":
            raise VergeMLError("Invalid mobilenet size: {}".format(size))
        if alpha not in MOBILENET_ALPHA_VALUES:
            raise VergeMLError("Invalid alpha value: {}".format(alpha))

    elif architecture == 'mobilenet-v2':
        if size not in MOBILENET_V2_SIZES and size != "auto":
            raise VergeMLError("Invalid mobilenet size: {}".format(size))

        if alpha not in MOBILENET_V2_ALPHA_VALUES:
            raise VergeMLError("Invalid alpha value: {}".format(alpha))

    elif architecture == 'nasnet':
        if not variant in NASNET_VARIANTS and variant != 'auto':
            raise VergeMLError("Invalid nasnet variant: {}".format(variant))