Python os.path() Examples

The following are 30 code examples of os.path(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module os , or try the search function .
Example #1
Source Project: vergeml   Author: mme   File: io.py    License: MIT License 7 votes vote down vote up
def preview_filename(self, path):
        """Generate a filename for previews, appending a number when the file already exists.
        """
        if not os.path.exists(path):
            return path

        pcounter = SourcePlugin._COUNTER
        if not path in pcounter:
            pcounter[path] = 1
        fname, ext = os.path.splitext(path)
        counter = pcounter[path]

        while os.path.exists("{}_{}{}".format(fname, counter, ext)):
            counter += 1
        pcounter[path] = counter

        return "{}_{}{}".format(fname, counter, ext) 
Example #2
Source Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 6 votes vote down vote up
def load(self, model_dir, architecture, image_size):
        from keras.models import load_model
        from vergeml.sources.features import get_preprocess_input
        labels_txt = os.path.join(model_dir, "labels.txt")
        if not os.path.exists(labels_txt):
            raise VergeMLError("labels.txt not found: {}".format(labels_txt))

        model_h5 = os.path.join(model_dir, "model.h5")
        if not os.path.exists(model_h5):
            raise VergeMLError("model.h5 not found: {}".format(model_h5))

        with open(labels_txt, "r") as f:
            self.labels = f.read().splitlines()

        self.model = load_model(model_h5)
        self.image_size = image_size
        self.preprocess_input = get_preprocess_input(architecture) 
Example #3
Source Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 6 votes vote down vote up
def predict(self, f, k=5, resize_mode='fill'):
        from keras.preprocessing import image
        from vergeml.img import resize_image

        filename = os.path.basename(f)

        if not os.path.exists(f):
            return dict(filename=filename, prediction=[])

        img = image.load_img(f)
        img = resize_image(img, self.image_size, self.image_size, 'antialias', resize_mode)

        x = image.img_to_array(img)
        x = np.expand_dims(x, axis=0)
        x = self.preprocess_input(x)
        preds = self.model.predict(x)
        pred = self._decode(preds, top=k)[0]
        prediction=[dict(probability=np.asscalar(perc), label=klass) for _, klass, perc in pred]

        return dict(filename=filename, prediction=prediction) 
Example #4
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 6 votes vote down vote up
def _validate_split(self, project_dir):
        """Validate the split configuration for val and test.
        """

        for split in ('val-split', 'test-split'):
            spltype, splval = parse_split(self._config[split])

            if spltype == 'dir':

                # deal with relative paths
                path = os.path.join(project_dir, splval) if not os.path.isabs(splval) else splval

                if not os.path.exists(path):

                    # raise if path does not exist
                    msg = f"Invalid value for option {split} - no such directory: {splval}"
                    suggestion = f"Please set {split} to a percentage, number or directory."

                    raise VergeMLError(msg, suggestion,
                                       hint_key=split, hint_type='value', help_topic='split') 
Example #5
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 6 votes vote down vote up
def _load_trained_model(self):
        """Load a trained models hyperparameters and results
        """

        train_mod_path = os.path.join(self._config['trainings-dir'], self.trained_model)
        if not os.path.exists(train_mod_path):
            raise VergeMLError("Trained model not found: {}".format(self.trained_model))

        # Merge data.yaml
        data_file = os.path.join(self._config['trainings-dir'], self.trained_model, 'data.yaml')
        if not os.path.exists(data_file):
            raise VergeMLError("data.yaml file not found for {}: {}".format(
                self.trained_model, data_file))

        doc = load_yaml_file(data_file, 'data file')
        self._config.update({
            'hyperparameters': doc.get('hyperparameters', {}),
            'results': doc.get('results', {}),
            'model': doc.get('model')
        })

        self.results = _Results(self, data_file)

        return data_file 
Example #6
Source Project: vergeml   Author: mme   File: features.py    License: MIT License 6 votes vote down vote up
def get_custom_architecture(name, trainings_dir, output_layer):
    from keras.models import load_model, Model
    name = name.lstrip("@")
    model = load_model(os.path.join(trainings_dir, name, 'checkpoints', 'model.h5'))
    try:
        if isinstance(output_layer, int):
            layer = model.layers[output_layer]
        else:
            layer = model.get_layer(output_layer)
    except Exception:
        if isinstance(output_layer, int):
            raise VergeMLError(f'output-layer {output_layer} not found - model has only {len(model.layers)} layers.')
        else:
            candidates = list(map(lambda l: l.name, model.layers))
            raise VergeMLError(f'output-layer named {output_layer} not found.',
                               suggestion=did_you_mean(candidates, output_layer))
    model = Model(inputs=model.input, outputs=layer.output)
    return model 
Example #7
Source Project: vergeml   Author: mme   File: __init__.py    License: MIT License 6 votes vote down vote up
def load_predictions(env, nclasses):
    path = os.path.join(env.stats_dir(), "predictions.csv")

    if not os.path.exists(path):
        raise FileExistsError(path)

    with open(path, newline='') as csvfile:
        y_score = []
        y_test = []
        csv_reader = csv.reader(csvfile, dialect="excel")
        for row in csv_reader:
            assert len(row) == nclasses * 2
            y_score.append(list(map(float, row[:nclasses])))
            y_test.append(list(map(float, row[nclasses:])))
        
        y_score = np.array(y_score)
        y_test = np.array(y_test)

        return y_test, y_score 
Example #8
Source Project: vergeml   Author: mme   File: unique_objects.py    License: MIT License 6 votes vote down vote up
def __call__(self, args, env):
        samples_dir = env.get('samples-dir')

        print("Downloading unique objects to {}.".format(samples_dir))

        src_dir = self.download_files([_URL], env=env, dir=env.get('cache-dir'))
        path = os.path.join(src_dir, "ObjectsAll.zip")

        zipf = zipfile.ZipFile(path, 'r')
        zipf.extractall(src_dir)
        zipf.close()

        for file in os.listdir(os.path.join(src_dir, "OBJECTSALL")):
            shutil.copy(os.path.join(src_dir, "OBJECTSALL", file), samples_dir)

        shutil.rmtree(src_dir)

        print("Finished downloading unique objects.") 
Example #9
Source Project: godot-mono-builds   Author: godotengine   File: cmd_utils.py    License: MIT License 6 votes vote down vote up
def add_base_arguments(parser, default_help):
    import os
    from os.path import join as path_join

    home = os.environ.get('HOME')
    mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '')

    parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help)
    # --jobs supports not passing an argument, in which case the 'const' is used,
    # which is the number of CPU cores on the host system.
    parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help)
    parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help)
    parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help)

    if mono_sources_default:
        parser.add_argument('--mono-sources', default=mono_sources_default, help=default_help)
    else:
        parser.add_argument('--mono-sources', required=True)

    parser.add_argument('--mxe-prefix', default='/usr', help=default_help) 
Example #10
Source Project: godot-mono-builds   Author: godotengine   File: os_utils.py    License: MIT License 6 votes vote down vote up
def mkdir_p(path):
    if not os.path.exists(path):
        print('creating directory: ' + path)
        os.makedirs(path)


# Remove files and/or directories recursively 
Example #11
Source Project: godot-mono-builds   Author: godotengine   File: os_utils.py    License: MIT License 6 votes vote down vote up
def find_executable(name) -> str:
    is_windows = os.name == 'nt'
    windows_exts = os.environ['PATHEXT'].split(ENV_PATH_SEP) if is_windows else None
    path_dirs = os.environ['PATH'].split(ENV_PATH_SEP)

    search_dirs = path_dirs + [os.getcwd()] # cwd is last in the list

    for dir in search_dirs:
        path = os.path.join(dir, name)

        if is_windows:
            for extension in windows_exts:
                path_with_ext = path + extension

                if os.path.isfile(path_with_ext) and os.access(path_with_ext, os.X_OK):
                    return path_with_ext
        else:
            if os.path.isfile(path) and os.access(path, os.X_OK):
                return path

    return '' 
Example #12
Source Project: godot-mono-builds   Author: godotengine   File: android.py    License: MIT License 6 votes vote down vote up
def configure(opts: AndroidOpts, product: str, target: str):
    env = { 'ANDROID_API_VERSION': get_api_version_or_min(opts, target) }

    if is_cross(target):
        import llvm

        if is_cross_mxe(target):
            llvm.make(opts, 'llvmwin64')
            setup_android_cross_mxe_template(env, opts, target, host_arch='x86_64')
        else:
            llvm.make(opts, 'llvm64')
            setup_android_cross_template(env, opts, target, host_arch='x86_64')
    else:
        make_standalone_toolchain(opts, target, env['ANDROID_API_VERSION'])
        setup_android_target_template(env, opts, target)

    if not os.path.isfile(path_join(opts.mono_source_root, 'configure')):
        runtime.run_autogen(opts)

    runtime.run_configure(env, opts, product, target) 
Example #13
Source Project: ALF   Author: blackberry   File: __init__.py    License: Apache License 2.0 6 votes vote down vote up
def import_helper():
    from os.path import dirname
    import imp
    possible_libs = ["_alf_grammar.win32",
                     "_alf_grammar.ntoarm",
                     "_alf_grammar.ntox86",
                     "_alf_grammar.linux"]
    found_lib = False
    for i in possible_libs:
        fp = None
        try:
            fp, pathname, description = imp.find_module(i, [dirname(__file__)])
            _mod = imp.load_module("_alf_grammar", fp, pathname, description)
            found_lib = True
            break
        except ImportError:
            pass
        finally:
            if fp:
                fp.close()
    if not found_lib:
        raise ImportError("Failed to load _alf_grammar module")
    return _mod 
Example #14
Source Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def cachedir(self):
        """Path to workflow's cache directory.

        The cache directory is a subdirectory of Alfred's own cache directory
        in ``~/Library/Caches``. The full path is:

        ``~/Library/Caches/com.runningwithcrayons.Alfred-X/Workflow Data/<bundle id>``

        ``Alfred-X`` may be ``Alfred-2`` or ``Alfred-3``.

        :returns: full path to workflow's cache directory
        :rtype: ``unicode``

        """
        if self.alfred_env.get('workflow_cache'):
            dirpath = self.alfred_env.get('workflow_cache')

        else:
            dirpath = self._default_cachedir

        return self._create(dirpath) 
Example #15
Source Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def cache_data(self, name, data):
        """Save ``data`` to cache under ``name``.

        If ``data`` is ``None``, the corresponding cache file will be
        deleted.

        :param name: name of datastore
        :param data: data to store. This may be any object supported by
                the cache serializer

        """
        serializer = manager.serializer(self.cache_serializer)

        cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer))

        if data is None:
            if os.path.exists(cache_path):
                os.unlink(cache_path)
                self.logger.debug('deleted cache file: %s', cache_path)
            return

        with atomic_writer(cache_path, 'wb') as file_obj:
            serializer.dump(data, file_obj)

        self.logger.debug('cached data: %s', cache_path) 
Example #16
Source Project: wechat-alfred-workflow   Author: TKkk-iOSer   File: workflow.py    License: MIT License 6 votes vote down vote up
def _delete_directory_contents(self, dirpath, filter_func):
        """Delete all files in a directory.

        :param dirpath: path to directory to clear
        :type dirpath: ``unicode`` or ``str``
        :param filter_func function to determine whether a file shall be
            deleted or not.
        :type filter_func ``callable``

        """
        if os.path.exists(dirpath):
            for filename in os.listdir(dirpath):
                if not filter_func(filename):
                    continue
                path = os.path.join(dirpath, filename)
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
                self.logger.debug('deleted : %r', path) 
Example #17
Source Project: vergeml   Author: mme   File: ls.py    License: MIT License 5 votes vote down vote up
def __call__(self, args, env):

        # Parse and partition into normal and comparison args.
        args, cargs = _parse_args(args, env)

        # When trainings dir does not exist, print an error and exit
        if not os.path.exists(env.get('trainings-dir')):
            print("No trainings found.", file=sys.stderr)
            return

        info, hyper = _find_trained_models(args, env)

        theader, tdata, left_align = _format_table(args, cargs, info, hyper)

        _output_table(args['output'], theader, tdata, left_align) 
Example #18
Source Project: vergeml   Author: mme   File: ls.py    License: MIT License 5 votes vote down vote up
def _find_trained_models(args, env):
    info = {}
    hyper = {}
    train_dir = env.get('trainings-dir')

    for trained_model in os.listdir(train_dir):
        data_yaml = os.path.join(train_dir, trained_model, 'data.yaml')
        if os.path.isfile(data_yaml):
            with open(data_yaml) as file:
                doc = yaml.safe_load(file)
        else:
            doc = {}
        info[trained_model] = {}
        hyper[trained_model] = {}

        if 'model' in doc:
            info[trained_model]['model'] = doc['model']

        if 'results' in doc:
            info[trained_model].update(doc['results'])

        if 'hyperparameters' in doc:
            hyper[trained_model].update(doc['hyperparameters'])

    sort = [s.strip() for s in args['sort'].split(",")]

    info = OrderedDict(sorted(info.items(), reverse=(args['order'] == 'asc'),
                              key=lambda x: [x[1].get(s, 0) for s in sort]))
    return info, hyper 
Example #19
Source Project: vergeml   Author: mme   File: io.py    License: MIT License 5 votes vote down vote up
def scan(self, path, exclude=[]) -> List[str]:
        """Scan path for matching files.

        :param path: the path to scan
        :param exclude: a list of directories to exclude

        :return: a list of sorted filenames
        """
        res = []
        path = path.rstrip("/").rstrip("\\")
        for pat in self.input_patterns:
            res.extend(glob.glob(path + os.sep + pat, recursive=True))

        res = list(filter(lambda p: os.path.isfile(p), res))

        if exclude:
            def excluded(path):
                for e in exclude:
                    if path.startswith(e):
                        return True
                return False

            res = list(filter(lambda p: not excluded(p), res))

        return sorted(res) 
Example #20
Source Project: vergeml   Author: mme   File: io.py    License: MIT License 5 votes vote down vote up
def hash_files(self, files):
        """A default implementation for hash based on files
        """
        if not self._cached_file_state:
            self._cached_file_state = io.BytesIO()

            for (split, files) in files.items():
                for path, _ in files:
                    fstate = "{}{}{}{}".format(split, path, os.path.getmtime(path), os.path.getsize(path))
                    self._cached_file_state.write(fstate.encode('utf-8'))

        return self._cached_file_state.getvalue().decode("utf-8") 
Example #21
Source Project: vergeml   Author: mme   File: imagenet.py    License: MIT License 5 votes vote down vote up
def _save(model, base_model, layers, labels, random_seed, checkpoints_dir):
    from keras.layers import Flatten, Dense
    from keras import Model
    nclasses = len(labels)
    x = Flatten()(base_model.output)
    x = _makenet(x, layers, dropout=None, random_seed=random_seed)
    predictions = Dense(nclasses, activation="softmax", name="predictions")(x)
    model_final = Model(inputs=base_model.input, outputs=predictions)

    for i in range(layers - 1):
        weights = model.get_layer(name='dense_layer_{}'.format(i)).get_weights()
        model_final.get_layer(name='dense_layer_{}'.format(i)).set_weights(weights)

    weights = model.get_layer(name='predictions').get_weights()
    model_final.get_layer(name='predictions').set_weights(weights)

    model_final.save(os.path.join(checkpoints_dir, "model.h5"))
    with open(os.path.join(checkpoints_dir, "labels.txt"), "w") as f:
        f.write("\n".join(labels))
    return model_final 
Example #22
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def get(self, path):
        """Get a value by its path.

        For example, to access the variable 'id' in the dict 'device', use device.id as path.
        """
        return deepcopy(dict_get_path(self._config, path, None)) 
Example #23
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def set(self, path, value):
        """Set a value by its path.
        """
        dict_set_path(self._config, path, value) 
Example #24
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def samples_dir(self):
        """Return the samples_dir or throw an error if it does not exist.
        """
        samples_dir = self._config['samples-dir']
        if not os.path.exists(samples_dir):
            raise VergeMLError(f'Could not find samples directory: {samples_dir}')
        elif not os.path.isdir(samples_dir):
            raise VergeMLError(f'Configured samples-dir is not a directory: {samples_dir}')
        return samples_dir 
Example #25
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def trained_model_dir(self):
        """Return the directory of a trained model (create if it does not exist).
        """
        assert self.trained_model, "You must create a trained model first."
        trained_model_dir = os.path.join(self._config['trainings-dir'], self.trained_model)
        try:
            os.makedirs(trained_model_dir)
        except FileExistsError:
            pass
        return trained_model_dir 
Example #26
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def checkpoints_dir(self):
        """Return the checkpoints directory (create if it does not exist)
        """
        checkpoints_dir = os.path.join(self.trained_model_dir(), "checkpoints")
        try:
            os.makedirs(checkpoints_dir)
        except FileExistsError:
            pass
        return checkpoints_dir 
Example #27
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def __init__(self, env, path):
        self.path = path
        self.env = env
        self.last_sync = None 
Example #28
Source Project: vergeml   Author: mme   File: env.py    License: MIT License 5 votes vote down vote up
def _sync(self, force=False):
        now = datetime.datetime.now()
        if force or not self.last_sync or (now - self.last_sync).total_seconds() > _SYNC_INTV:
            self.last_sync = now
            data = dict(model=self.env.get('model'),
                        hyperparameters=self._convert(self.env.get('hyperparameters')),
                        results=self._convert(self.env.get('results')))
            with open(self.path, "w") as f:
                yaml.dump(data, f, default_flow_style=False) 
Example #29
Source Project: vergeml   Author: mme   File: results.py    License: MIT License 5 votes vote down vote up
def __init__(self, path):
        self.data = {}
        self.path = path
        if os.path.exists(path):
            with open(path) as f:
                self.data = json.load(f) 
Example #30
Source Project: vergeml   Author: mme   File: image.py    License: MIT License 5 votes vote down vote up
def write_preview(self, output_dir: str, split: str, sample: Sample):
        filename = sample.meta['filename']
        
        name = fixext(os.path.basename(filename), sample.x)
        split_dir = os.path.join(output_dir, split)
        if not os.path.exists(split_dir):
            os.makedirs(split_dir)

        path = self.preview_filename(os.path.join(split_dir, name))
        sample.x.save(path)
        return path