Python os.path() Examples
The following are 30 code examples for showing how to use os.path(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.
You may check out the related API usage on the sidebar.
You may also want to check out all available functions/classes of the module
os
, or try the search function
.
Example 1
Project: vergeml Author: mme File: io.py License: MIT License | 6 votes |
def scan(self, path, exclude=[]) -> List[str]: """Scan path for matching files. :param path: the path to scan :param exclude: a list of directories to exclude :return: a list of sorted filenames """ res = [] path = path.rstrip("/").rstrip("\\") for pat in self.input_patterns: res.extend(glob.glob(path + os.sep + pat, recursive=True)) res = list(filter(lambda p: os.path.isfile(p), res)) if exclude: def excluded(path): for e in exclude: if path.startswith(e): return True return False res = list(filter(lambda p: not excluded(p), res)) return sorted(res)
Example 2
Project: vergeml Author: mme File: io.py License: MIT License | 6 votes |
def preview_filename(self, path): """Generate a filename for previews, appending a number when the file already exists. """ if not os.path.exists(path): return path pcounter = SourcePlugin._COUNTER if not path in pcounter: pcounter[path] = 1 fname, ext = os.path.splitext(path) counter = pcounter[path] while os.path.exists("{}_{}{}".format(fname, counter, ext)): counter += 1 pcounter[path] = counter return "{}_{}{}".format(fname, counter, ext)
Example 3
Project: vergeml Author: mme File: imagenet.py License: MIT License | 6 votes |
def load(self, model_dir, architecture, image_size): from keras.models import load_model from vergeml.sources.features import get_preprocess_input labels_txt = os.path.join(model_dir, "labels.txt") if not os.path.exists(labels_txt): raise VergeMLError("labels.txt not found: {}".format(labels_txt)) model_h5 = os.path.join(model_dir, "model.h5") if not os.path.exists(model_h5): raise VergeMLError("model.h5 not found: {}".format(model_h5)) with open(labels_txt, "r") as f: self.labels = f.read().splitlines() self.model = load_model(model_h5) self.image_size = image_size self.preprocess_input = get_preprocess_input(architecture)
Example 4
Project: vergeml Author: mme File: imagenet.py License: MIT License | 6 votes |
def predict(self, f, k=5, resize_mode='fill'): from keras.preprocessing import image from vergeml.img import resize_image filename = os.path.basename(f) if not os.path.exists(f): return dict(filename=filename, prediction=[]) img = image.load_img(f) img = resize_image(img, self.image_size, self.image_size, 'antialias', resize_mode) x = image.img_to_array(img) x = np.expand_dims(x, axis=0) x = self.preprocess_input(x) preds = self.model.predict(x) pred = self._decode(preds, top=k)[0] prediction=[dict(probability=np.asscalar(perc), label=klass) for _, klass, perc in pred] return dict(filename=filename, prediction=prediction)
Example 5
Project: vergeml Author: mme File: env.py License: MIT License | 6 votes |
def _validate_split(self, project_dir): """Validate the split configuration for val and test. """ for split in ('val-split', 'test-split'): spltype, splval = parse_split(self._config[split]) if spltype == 'dir': # deal with relative paths path = os.path.join(project_dir, splval) if not os.path.isabs(splval) else splval if not os.path.exists(path): # raise if path does not exist msg = f"Invalid value for option {split} - no such directory: {splval}" suggestion = f"Please set {split} to a percentage, number or directory." raise VergeMLError(msg, suggestion, hint_key=split, hint_type='value', help_topic='split')
Example 6
Project: vergeml Author: mme File: env.py License: MIT License | 6 votes |
def _load_trained_model(self): """Load a trained models hyperparameters and results """ train_mod_path = os.path.join(self._config['trainings-dir'], self.trained_model) if not os.path.exists(train_mod_path): raise VergeMLError("Trained model not found: {}".format(self.trained_model)) # Merge data.yaml data_file = os.path.join(self._config['trainings-dir'], self.trained_model, 'data.yaml') if not os.path.exists(data_file): raise VergeMLError("data.yaml file not found for {}: {}".format( self.trained_model, data_file)) doc = load_yaml_file(data_file, 'data file') self._config.update({ 'hyperparameters': doc.get('hyperparameters', {}), 'results': doc.get('results', {}), 'model': doc.get('model') }) self.results = _Results(self, data_file) return data_file
Example 7
Project: vergeml Author: mme File: features.py License: MIT License | 6 votes |
def get_custom_architecture(name, trainings_dir, output_layer): from keras.models import load_model, Model name = name.lstrip("@") model = load_model(os.path.join(trainings_dir, name, 'checkpoints', 'model.h5')) try: if isinstance(output_layer, int): layer = model.layers[output_layer] else: layer = model.get_layer(output_layer) except Exception: if isinstance(output_layer, int): raise VergeMLError(f'output-layer {output_layer} not found - model has only {len(model.layers)} layers.') else: candidates = list(map(lambda l: l.name, model.layers)) raise VergeMLError(f'output-layer named {output_layer} not found.', suggestion=did_you_mean(candidates, output_layer)) model = Model(inputs=model.input, outputs=layer.output) return model
Example 8
Project: vergeml Author: mme File: __init__.py License: MIT License | 6 votes |
def load_predictions(env, nclasses): path = os.path.join(env.stats_dir(), "predictions.csv") if not os.path.exists(path): raise FileExistsError(path) with open(path, newline='') as csvfile: y_score = [] y_test = [] csv_reader = csv.reader(csvfile, dialect="excel") for row in csv_reader: assert len(row) == nclasses * 2 y_score.append(list(map(float, row[:nclasses]))) y_test.append(list(map(float, row[nclasses:]))) y_score = np.array(y_score) y_test = np.array(y_test) return y_test, y_score
Example 9
Project: vergeml Author: mme File: unique_objects.py License: MIT License | 6 votes |
def __call__(self, args, env): samples_dir = env.get('samples-dir') print("Downloading unique objects to {}.".format(samples_dir)) src_dir = self.download_files([_URL], env=env, dir=env.get('cache-dir')) path = os.path.join(src_dir, "ObjectsAll.zip") zipf = zipfile.ZipFile(path, 'r') zipf.extractall(src_dir) zipf.close() for file in os.listdir(os.path.join(src_dir, "OBJECTSALL")): shutil.copy(os.path.join(src_dir, "OBJECTSALL", file), samples_dir) shutil.rmtree(src_dir) print("Finished downloading unique objects.")
Example 10
Project: godot-mono-builds Author: godotengine File: cmd_utils.py License: MIT License | 6 votes |
def add_base_arguments(parser, default_help): import os from os.path import join as path_join home = os.environ.get('HOME') mono_sources_default = os.environ.get('MONO_SOURCE_ROOT', '') parser.add_argument('--verbose-make', action='store_true', default=False, help=default_help) # --jobs supports not passing an argument, in which case the 'const' is used, # which is the number of CPU cores on the host system. parser.add_argument('--jobs', '-j', nargs='?', const=str(os.cpu_count()), default='1', help=default_help) parser.add_argument('--configure-dir', default=path_join(home, 'mono-configs'), help=default_help) parser.add_argument('--install-dir', default=path_join(home, 'mono-installs'), help=default_help) if mono_sources_default: parser.add_argument('--mono-sources', default=mono_sources_default, help=default_help) else: parser.add_argument('--mono-sources', required=True) parser.add_argument('--mxe-prefix', default='/usr', help=default_help)
Example 11
Project: godot-mono-builds Author: godotengine File: os_utils.py License: MIT License | 6 votes |
def find_executable(name) -> str: is_windows = os.name == 'nt' windows_exts = os.environ['PATHEXT'].split(ENV_PATH_SEP) if is_windows else None path_dirs = os.environ['PATH'].split(ENV_PATH_SEP) search_dirs = path_dirs + [os.getcwd()] # cwd is last in the list for dir in search_dirs: path = os.path.join(dir, name) if is_windows: for extension in windows_exts: path_with_ext = path + extension if os.path.isfile(path_with_ext) and os.access(path_with_ext, os.X_OK): return path_with_ext else: if os.path.isfile(path) and os.access(path, os.X_OK): return path return ''
Example 12
Project: godot-mono-builds Author: godotengine File: android.py License: MIT License | 6 votes |
def configure(opts: AndroidOpts, product: str, target: str): env = { 'ANDROID_API_VERSION': get_api_version_or_min(opts, target) } if is_cross(target): import llvm if is_cross_mxe(target): llvm.make(opts, 'llvmwin64') setup_android_cross_mxe_template(env, opts, target, host_arch='x86_64') else: llvm.make(opts, 'llvm64') setup_android_cross_template(env, opts, target, host_arch='x86_64') else: make_standalone_toolchain(opts, target, env['ANDROID_API_VERSION']) setup_android_target_template(env, opts, target) if not os.path.isfile(path_join(opts.mono_source_root, 'configure')): runtime.run_autogen(opts) runtime.run_configure(env, opts, product, target)
Example 13
Project: ALF Author: blackberry File: __init__.py License: Apache License 2.0 | 6 votes |
def import_helper(): from os.path import dirname import imp possible_libs = ["_alf_grammar.win32", "_alf_grammar.ntoarm", "_alf_grammar.ntox86", "_alf_grammar.linux"] found_lib = False for i in possible_libs: fp = None try: fp, pathname, description = imp.find_module(i, [dirname(__file__)]) _mod = imp.load_module("_alf_grammar", fp, pathname, description) found_lib = True break except ImportError: pass finally: if fp: fp.close() if not found_lib: raise ImportError("Failed to load _alf_grammar module") return _mod
Example 14
Project: wechat-alfred-workflow Author: TKkk-iOSer File: workflow.py License: MIT License | 6 votes |
def cachedir(self): """Path to workflow's cache directory. The cache directory is a subdirectory of Alfred's own cache directory in ``~/Library/Caches``. The full path is: ``~/Library/Caches/com.runningwithcrayons.Alfred-X/Workflow Data/<bundle id>`` ``Alfred-X`` may be ``Alfred-2`` or ``Alfred-3``. :returns: full path to workflow's cache directory :rtype: ``unicode`` """ if self.alfred_env.get('workflow_cache'): dirpath = self.alfred_env.get('workflow_cache') else: dirpath = self._default_cachedir return self._create(dirpath)
Example 15
Project: wechat-alfred-workflow Author: TKkk-iOSer File: workflow.py License: MIT License | 6 votes |
def cache_data(self, name, data): """Save ``data`` to cache under ``name``. If ``data`` is ``None``, the corresponding cache file will be deleted. :param name: name of datastore :param data: data to store. This may be any object supported by the cache serializer """ serializer = manager.serializer(self.cache_serializer) cache_path = self.cachefile('%s.%s' % (name, self.cache_serializer)) if data is None: if os.path.exists(cache_path): os.unlink(cache_path) self.logger.debug('deleted cache file: %s', cache_path) return with atomic_writer(cache_path, 'wb') as file_obj: serializer.dump(data, file_obj) self.logger.debug('cached data: %s', cache_path)
Example 16
Project: wechat-alfred-workflow Author: TKkk-iOSer File: workflow.py License: MIT License | 6 votes |
def _delete_directory_contents(self, dirpath, filter_func): """Delete all files in a directory. :param dirpath: path to directory to clear :type dirpath: ``unicode`` or ``str`` :param filter_func function to determine whether a file shall be deleted or not. :type filter_func ``callable`` """ if os.path.exists(dirpath): for filename in os.listdir(dirpath): if not filter_func(filename): continue path = os.path.join(dirpath, filename) if os.path.isdir(path): shutil.rmtree(path) else: os.unlink(path) self.logger.debug('deleted : %r', path)
Example 17
Project: vergeml Author: mme File: ls.py License: MIT License | 5 votes |
def __call__(self, args, env): # Parse and partition into normal and comparison args. args, cargs = _parse_args(args, env) # When trainings dir does not exist, print an error and exit if not os.path.exists(env.get('trainings-dir')): print("No trainings found.", file=sys.stderr) return info, hyper = _find_trained_models(args, env) theader, tdata, left_align = _format_table(args, cargs, info, hyper) _output_table(args['output'], theader, tdata, left_align)
Example 18
Project: vergeml Author: mme File: ls.py License: MIT License | 5 votes |
def _find_trained_models(args, env): info = {} hyper = {} train_dir = env.get('trainings-dir') for trained_model in os.listdir(train_dir): data_yaml = os.path.join(train_dir, trained_model, 'data.yaml') if os.path.isfile(data_yaml): with open(data_yaml) as file: doc = yaml.safe_load(file) else: doc = {} info[trained_model] = {} hyper[trained_model] = {} if 'model' in doc: info[trained_model]['model'] = doc['model'] if 'results' in doc: info[trained_model].update(doc['results']) if 'hyperparameters' in doc: hyper[trained_model].update(doc['hyperparameters']) sort = [s.strip() for s in args['sort'].split(",")] info = OrderedDict(sorted(info.items(), reverse=(args['order'] == 'asc'), key=lambda x: [x[1].get(s, 0) for s in sort])) return info, hyper
Example 19
Project: vergeml Author: mme File: io.py License: MIT License | 5 votes |
def hash_files(self, files): """A default implementation for hash based on files """ if not self._cached_file_state: self._cached_file_state = io.BytesIO() for (split, files) in files.items(): for path, _ in files: fstate = "{}{}{}{}".format(split, path, os.path.getmtime(path), os.path.getsize(path)) self._cached_file_state.write(fstate.encode('utf-8')) return self._cached_file_state.getvalue().decode("utf-8")
Example 20
Project: vergeml Author: mme File: imagenet.py License: MIT License | 5 votes |
def _save(model, base_model, layers, labels, random_seed, checkpoints_dir): from keras.layers import Flatten, Dense from keras import Model nclasses = len(labels) x = Flatten()(base_model.output) x = _makenet(x, layers, dropout=None, random_seed=random_seed) predictions = Dense(nclasses, activation="softmax", name="predictions")(x) model_final = Model(inputs=base_model.input, outputs=predictions) for i in range(layers - 1): weights = model.get_layer(name='dense_layer_{}'.format(i)).get_weights() model_final.get_layer(name='dense_layer_{}'.format(i)).set_weights(weights) weights = model.get_layer(name='predictions').get_weights() model_final.get_layer(name='predictions').set_weights(weights) model_final.save(os.path.join(checkpoints_dir, "model.h5")) with open(os.path.join(checkpoints_dir, "labels.txt"), "w") as f: f.write("\n".join(labels)) return model_final
Example 21
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def get(self, path): """Get a value by its path. For example, to access the variable 'id' in the dict 'device', use device.id as path. """ return deepcopy(dict_get_path(self._config, path, None))
Example 22
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def set(self, path, value): """Set a value by its path. """ dict_set_path(self._config, path, value)
Example 23
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def samples_dir(self): """Return the samples_dir or throw an error if it does not exist. """ samples_dir = self._config['samples-dir'] if not os.path.exists(samples_dir): raise VergeMLError(f'Could not find samples directory: {samples_dir}') elif not os.path.isdir(samples_dir): raise VergeMLError(f'Configured samples-dir is not a directory: {samples_dir}') return samples_dir
Example 24
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def trained_model_dir(self): """Return the directory of a trained model (create if it does not exist). """ assert self.trained_model, "You must create a trained model first." trained_model_dir = os.path.join(self._config['trainings-dir'], self.trained_model) try: os.makedirs(trained_model_dir) except FileExistsError: pass return trained_model_dir
Example 25
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def checkpoints_dir(self): """Return the checkpoints directory (create if it does not exist) """ checkpoints_dir = os.path.join(self.trained_model_dir(), "checkpoints") try: os.makedirs(checkpoints_dir) except FileExistsError: pass return checkpoints_dir
Example 26
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def __init__(self, env, path): self.path = path self.env = env self.last_sync = None
Example 27
Project: vergeml Author: mme File: env.py License: MIT License | 5 votes |
def _sync(self, force=False): now = datetime.datetime.now() if force or not self.last_sync or (now - self.last_sync).total_seconds() > _SYNC_INTV: self.last_sync = now data = dict(model=self.env.get('model'), hyperparameters=self._convert(self.env.get('hyperparameters')), results=self._convert(self.env.get('results'))) with open(self.path, "w") as f: yaml.dump(data, f, default_flow_style=False)
Example 28
Project: vergeml Author: mme File: results.py License: MIT License | 5 votes |
def __init__(self, path): self.data = {} self.path = path if os.path.exists(path): with open(path) as f: self.data = json.load(f)
Example 29
Project: vergeml Author: mme File: image.py License: MIT License | 5 votes |
def write_preview(self, output_dir: str, split: str, sample: Sample): filename = sample.meta['filename'] name = fixext(os.path.basename(filename), sample.x) split_dir = os.path.join(output_dir, split) if not os.path.exists(split_dir): os.makedirs(split_dir) path = self.preview_filename(os.path.join(split_dir, name)) sample.x.save(path) return path
Example 30
Project: vergeml Author: mme File: features.py License: MIT License | 5 votes |
def evaluate_args(architecture, trainings_dir, variant, alpha, size): if not architecture.startswith('@') and not architecture in ARCHITECTURES: raise VergeMLError("Unknown CNN: {}".format(architecture)) elif architecture.startswith('@'): name = architecture.lstrip('@') path = os.path.join(trainings_dir, name, 'checkpoints', 'model.h5') if not os.path.isfile(path): raise VergeMLError("Unknown CNN: {}".format(architecture)) if architecture == 'densenet': if variant == 'auto': variant = DENSENET_VARIANTS[0] if not variant in DENSENET_VARIANTS: raise VergeMLError("Invalid densenet variant: {}".format(variant)) elif architecture == 'mobilenet': if size not in MOBILENET_SIZES and size != "auto": raise VergeMLError("Invalid mobilenet size: {}".format(size)) if alpha not in MOBILENET_ALPHA_VALUES: raise VergeMLError("Invalid alpha value: {}".format(alpha)) elif architecture == 'mobilenet-v2': if size not in MOBILENET_V2_SIZES and size != "auto": raise VergeMLError("Invalid mobilenet size: {}".format(size)) if alpha not in MOBILENET_V2_ALPHA_VALUES: raise VergeMLError("Invalid alpha value: {}".format(alpha)) elif architecture == 'nasnet': if not variant in NASNET_VARIANTS and variant != 'auto': raise VergeMLError("Invalid nasnet variant: {}".format(variant))