Python os.walk() Examples
The following are 30
code examples of os.walk().
You can vote up the ones you like or vote down the ones you don't like,
and go to the original project or source file by following the links above each example.
You may also want to check out all available functions/classes of the module
os
, or try the search function
.
Example #1
Source File: create_dataset.py From cat-bbs with MIT License | 8 votes |
def find_image_filepaths(dataset_dir): """Load image filepaths from the 10k cats dataset.""" result = [] for root, dirs, files in os.walk(dataset_dir): if "/CAT_" in root: for name in files: fp = os.path.join(root, name) if name.endswith(".jpg") and os.path.isfile("%s.cat" % (fp,)): result.append(fp) return result
Example #2
Source File: test_pep8.py From hydrus with MIT License | 8 votes |
def test_pep8(self): """Test method to check PEP8 compliance over the entire project.""" self.file_structure = dirname(dirname(abspath(__file__))) print("Testing for PEP8 compliance of python files in {}".format( self.file_structure)) style = pep8.StyleGuide() style.options.max_line_length = 100 # Set this to desired maximum line length filenames = [] # Set this to desired folder location for root, _, files in os.walk(self.file_structure): python_files = [f for f in files if f.endswith( '.py') and "examples" not in root] for file in python_files: if len(root.split('samples')) != 2: # Ignore samples directory filename = '{0}/{1}'.format(root, file) filenames.append(filename) check = style.check_files(filenames) self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' % check.total_errors)
Example #3
Source File: start_oa.py From incubator-spot with Apache License 2.0 | 6 votes |
def validate_parameters_values(args,logger): logger.info("Validating input parameter values") #date. is_date_ok = True if len(args.date) == 8 else False # type dirs = os.walk(script_path).next()[1] is_type_ok = True if args.type in dirs else False #limit try: int(args.limit) is_limit_ok = True except ValueError: is_limit_ok = False if not is_date_ok: logger.error("date parameter is not correct, please validate it") if not is_type_ok: logger.error("type parameter is not supported, please select a valid type") if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit") if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1)
Example #4
Source File: run.py From fullrmc with GNU Affero General Public License v3.0 | 6 votes |
def ML_run(): LOGGER.force_log("info", "machine learning selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False) # load fresh engine engine = create_engine() # delete existing log files MLLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and MLSelLog in fn] [os.remove(l) for l in MLLogs] # set log file name LOGGER.set_log_file_basename(MLSelLog) # set smart group selector engine.set_group_selector(SmartRandomSelector(engine)) # run engine engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None) LOGGER.force_log("info", "machine learning selection finished", stdout=True, file=False) ########################################################################################## ##################################### RUN SIMULATION ###################################
Example #5
Source File: run.py From fullrmc with GNU Affero General Public License v3.0 | 6 votes |
def normal_run(): LOGGER.force_log("info", "normal selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False) # load fresh engine engine = create_engine() # delete existing log files normalLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and normalSelLog in fn] [os.remove(l) for l in normalLogs] # set log file name LOGGER.set_log_file_basename(normalSelLog) # set random group selector engine.set_group_selector(RandomSelector(engine)) # run engine engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None) LOGGER.force_log("info", "normal selection finished", stdout=True, file=False) ############### run machine learning selector ###############
Example #6
Source File: archive.py From CAMISIM with Apache License 2.0 | 6 votes |
def zip_stream(src_dir, output_stream): """ @param src_dir: @type src_dir: str @param output_stream: @type output_stream: zipfile.ZipFile @return: """ root_path = os.path.dirname(src_dir) assert os.path.isdir(src_dir), "Invalid, not a directory: '{}'".format(src_dir) for root, directories, files in os.walk(src_dir): for file_name in files: file_path = os.path.join(root, file_name) relative_path = os.path.relpath(file_path, root_path) output_stream.write(file_path, arcname=relative_path)
Example #7
Source File: data.py From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 | 6 votes |
def make_dataset(dir, class_to_idx): images = [] dir = os.path.expanduser(dir) for target in sorted(os.listdir(dir)): d = os.path.join(dir, target) if not os.path.isdir(d): continue for root, _, fnames in sorted(os.walk(d)): for fname in sorted(fnames): if is_image_file(fname): path = os.path.join(root, fname) item = (path, class_to_idx[target]) images.append(item) return images
Example #8
Source File: collectdReportMetrics.py From InsightAgent with Apache License 2.0 | 6 votes |
def remove_old_files_and_update_filesnames(filenames): all_directories = os.listdir(csvpath) # aggregate cou for collectd version < 5.5 aggregate_cpu = False # remove old csv files in datadir remove_old_files(os.path.join(home_path, data_dir), 'csv') for each_dir in all_directories: # remove old collectd log files remove_old_files(os.path.join(csvpath, each_dir), None) if "disk" in each_dir: filenames[each_dir + "/disk_octets-"] = [each_dir + '_DiskWrite', each_dir + '_DiskRead'] if "interface" in each_dir: filenames[each_dir + "/if_octets-"] = [each_dir + '_NetworkIn', each_dir + '_NetworkOut'] for fEntry in os.walk(os.path.join(csvpath)): if "cpu-" in fEntry[0]: aggregate_cpu = True filenames['aggregation-cpu-average/cpu-system-'] = ['CPU'] return aggregate_cpu
Example #9
Source File: env_tools.py From OpenFermion-Cirq with Apache License 2.0 | 6 votes |
def get_unhidden_ungenerated_python_files(directory: str) -> Iterable[str]: """Iterates through relevant python files within the given directory. Args: directory: The top-level directory to explore. Yields: File paths. """ for dirpath, dirnames, filenames in os.walk(directory, topdown=True): if os.path.split(dirpath)[-1].startswith('.'): dirnames.clear() continue for filename in filenames: if filename.endswith('.py') and not filename.endswith('_pb2.py'): yield os.path.join(dirpath, filename)
Example #10
Source File: getlogs_evtx.py From InsightAgent with Apache License 2.0 | 6 votes |
def get_file_list_for_directory(root_path='/', file_name_regex=''): if root_path is None or len(root_path) == 0: return [] if root_path[-1] != '/': if os.path.exists(root_path) and (not file_name_regex or (file_name_regex and re.match(file_name_regex, root_path))): return [root_path] else: return [] file_list = [] for path, subdirs, files in os.walk(root_path): for name in files: if not file_name_regex or (file_name_regex and re.match(file_name_regex, name)): file_list.append(os.path.join(path, name)) return file_list
Example #11
Source File: flakiness_checker.py From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 | 6 votes |
def find_test_path(test_file): """Searches for the test file and returns the path if found As a default, the currend working directory is the top of the search. If a directory was provided as part of the argument, the directory will be joined with cwd unless it was an absolute path, in which case, the absolute path will be used instead. """ test_file += ".py" test_path = os.path.split(test_file) top = os.path.join(os.getcwd(), test_path[0]) for (path, dirs, files) in os.walk(top): if test_path[1] in files: return os.path.join(path, test_path[1]) raise FileNotFoundError("Could not find " + test_path[1] + "in directory: " + top)
Example #12
Source File: smatch-table.py From smatch with MIT License | 6 votes |
def get_names(file_dir, files): """ Get the annotator name list based on a list of files Args: file_dir: AMR file folder files: a list of AMR names, e.g. nw_wsj_0001_1 Returns: a list of user names who annotate all the files """ # for each user, check if they have files available # return user name list total_list = [] name_list = [] get_sub = False for path, subdir, dir_files in os.walk(file_dir): if not get_sub: total_list = subdir[:] get_sub = True else: break for user in total_list: has_file = True for f in files: file_path = file_dir + user + "/" + f + ".txt" if not os.path.exists(file_path): has_file = False break if has_file: name_list.append(user) if len(name_list) == 0: print("********Error: Cannot find any user who completes the files*************", file=ERROR_LOG) return name_list
Example #13
Source File: utils.py From incubator-spot with Apache License 2.0 | 6 votes |
def validate_data_source(cls,pipeline_type): dirs = os.walk("{0}/pipelines/".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))).next()[1] is_type_ok = True if pipeline_type in dirs else False return is_type_ok #class NewFileEvent(FileSystemEventHandler): # # pipeline_instance = None # def __init__(self,pipeline_instance): # self.pipeline_instance = pipeline_instance # # def on_moved(self,event): # if not event.is_directory: # self.pipeline_instance.new_file_detected(event.dest_path) # # def on_created(self,event): # if not event.is_directory: # self.pipeline_instance.new_file_detected(event.src_path)
Example #14
Source File: lambda_function_builder.py From sqs-s3-logger with Apache License 2.0 | 5 votes |
def archive(src_dir, output_file): with zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED) as f: src_dir_len = len(src_dir) for root, _, files in os.walk(src_dir): for file in files: fn = os.path.join(root, file) f.write(fn, fn[src_dir_len+1:])
Example #15
Source File: plot.py From cs294-112_hws with MIT License | 5 votes |
def get_datasets(fpath, condition=None): unit = 0 datasets = [] for root, dir, files in os.walk(fpath): if 'log.txt' in files: param_path = open(os.path.join(root,'params.json')) params = json.load(param_path) exp_name = params['exp_name'] log_path = os.path.join(root,'log.txt') experiment_data = pd.read_table(log_path) experiment_data.insert( len(experiment_data.columns), 'Unit', unit ) experiment_data.insert( len(experiment_data.columns), 'Condition', condition or exp_name ) datasets.append(experiment_data) unit += 1 return datasets
Example #16
Source File: getmessages_kafka2.py From InsightAgent with Apache License 2.0 | 5 votes |
def get_file_list_for_directory(root_path='/', file_name_regex_c=''): root_path = os.path.expanduser(root_path) if os.path.isdir(root_path): file_list = [] for path, subdirs, files in os.walk(root_path): for name in files: if check_regex(file_name_regex_c, name): file_list.append(os.path.join(path, name)) return file_list elif os.path.isfile(root_path): if check_regex(file_name_regex_c, root_path): return [root_path] return []
Example #17
Source File: getmessages_file_replay.py From InsightAgent with Apache License 2.0 | 5 votes |
def json_gather_list_values(l, fields, remove=False): sub_field_value = [] # treat each item in the list as a potential tree to walk down for sub_value in l: fields_copy = list(fields[i] for i in range(len(fields))) json_value = json_format_field_value( _get_json_field_helper( sub_value, fields_copy, allow_list=True, remove=remove)) if len(json_value) != 0: sub_field_value.append(json_value) # return the full list of field values return sub_field_value
Example #18
Source File: tools.py From delocate with BSD 2-Clause "Simplified" License | 5 votes |
def dir2zip(in_dir, zip_fname): """ Make a zip file `zip_fname` with contents of directory `in_dir` The recorded filenames are relative to `in_dir`, so doing a standard zip unpack of the resulting `zip_fname` in an empty directory will result in the original directory contents. Parameters ---------- in_dir : str Directory path containing files to go in the zip archive zip_fname : str Filename of zip archive to write """ z = zipfile.ZipFile(zip_fname, 'w', compression=zipfile.ZIP_DEFLATED) for root, dirs, files in os.walk(in_dir): for file in files: in_fname = pjoin(root, file) in_stat = os.stat(in_fname) # Preserve file permissions, but allow copy info = zipfile.ZipInfo(in_fname) info.filename = relpath(in_fname, in_dir) if os.path.sep == '\\': # Make the path unix friendly on windows. # PyPI won't accept wheels with windows path separators info.filename = relpath(in_fname, in_dir).replace('\\', '/') # Set time from modification time info.date_time = time.localtime(in_stat.st_mtime) # See https://stackoverflow.com/questions/434641/how-do-i-set-permissions-attributes-on-a-file-in-a-zip-file-using-pythons-zip/48435482#48435482 # noqa: E501 # Also set regular file permissions perms = stat.S_IMODE(in_stat.st_mode) | stat.S_IFREG info.external_attr = perms << 16 with open_readable(in_fname, 'rb') as fobj: contents = fobj.read() z.writestr(info, contents, zipfile.ZIP_DEFLATED) z.close()
Example #19
Source File: plot.py From cs294-112_hws with MIT License | 5 votes |
def get_datasets(fpath, condition=None): unit = 0 datasets = [] for root, dir, files in os.walk(fpath): if 'log.txt' in files: param_path = open(os.path.join(root,'params.json')) params = json.load(param_path) exp_name = params['exp_name'] log_path = os.path.join(root,'log.txt') experiment_data = pd.read_table(log_path) experiment_data.insert( len(experiment_data.columns), 'Unit', unit ) experiment_data.insert( len(experiment_data.columns), 'Condition', condition or exp_name ) datasets.append(experiment_data) unit += 1 return datasets
Example #20
Source File: getlogs_tcpdump.py From InsightAgent with Apache License 2.0 | 5 votes |
def get_file_list_for_directory(root_path='/', file_name_regex_c=''): root_path = os.path.expanduser(root_path) if os.path.isdir(root_path): file_list = [] for path, subdirs, files in os.walk(root_path): for name in files: if check_regex(file_name_regex_c, name): file_list.append(os.path.join(path, name)) return file_list elif os.path.isfile(root_path): if check_regex(file_name_regex_c, root_path): return [root_path] return []
Example #21
Source File: plot_3.py From cs294-112_hws with MIT License | 5 votes |
def get_datasets(fpath, condition=None): unit = 0 datasets = [] for root, dir, files in os.walk(fpath): if 'log.txt' in files: param_path = open(os.path.join(root,'params.json')) params = json.load(param_path) exp_name = params['exp_name'] log_path = os.path.join(root,'log.txt') experiment_data = pd.read_table(log_path) experiment_data.insert( len(experiment_data.columns), 'Unit', unit ) experiment_data.insert( len(experiment_data.columns), 'Condition', condition or exp_name ) datasets.append(experiment_data) unit += 1 datasets = pd.concat(datasets, ignore_index=True) return datasets
Example #22
Source File: plot.py From cs294-112_hws with MIT License | 5 votes |
def get_datasets(fpath, condition=None): unit = 0 datasets = [] for root, dir, files in os.walk(fpath): if 'log.txt' in files: param_path = open(os.path.join(root,'params.json')) params = json.load(param_path) exp_name = params['exp_name'] log_path = os.path.join(root,'log.txt') experiment_data = pd.read_table(log_path) experiment_data.insert( len(experiment_data.columns), 'Unit', unit ) experiment_data.insert( len(experiment_data.columns), 'Condition', condition or exp_name ) datasets.append(experiment_data) unit += 1 return datasets
Example #23
Source File: plot.py From cs294-112_hws with MIT License | 5 votes |
def get_datasets(fpath, condition=None): unit = 0 datasets = [] for root, dir, files in os.walk(fpath): if 'log.txt' in files: param_path = open(os.path.join(root,'params.json')) params = json.load(param_path) exp_name = params['exp_name'] log_path = os.path.join(root,'log.txt') experiment_data = pd.read_table(log_path) experiment_data.insert( len(experiment_data.columns), 'Unit', unit ) experiment_data.insert( len(experiment_data.columns), 'Condition', condition or exp_name ) datasets.append(experiment_data) unit += 1 return datasets
Example #24
Source File: setup.py From edx_xblock_scorm with Apache License 2.0 | 5 votes |
def package_data(pkg, roots): """Generic function to find package_data. All of the files under each of the `roots` will be declared as package data for package `pkg`. """ data = [] for root in roots: for dirname, _, files in os.walk(os.path.join(pkg, root)): for fname in files: data.append(os.path.relpath(os.path.join(dirname, fname), pkg)) return {pkg: data}
Example #25
Source File: predict.py From Face-Recognition with MIT License | 5 votes |
def get_prediction_images(prediction_dir): files = [x[2] for x in os.walk(prediction_dir)][0] l = [] exts = [".jpg", ".jpeg", ".png"] for file in files: _, ext = os.path.splitext(file) if ext.lower() in exts: l.append(os.path.join(prediction_dir, file)) return l
Example #26
Source File: create_encodings.py From Face-Recognition with MIT License | 5 votes |
def _get_each_labels_files(training_dir_path): return [x[2] for x in os.walk(training_dir_path)][1:]
Example #27
Source File: create_encodings.py From Face-Recognition with MIT License | 5 votes |
def _get_training_labels(training_dir_path): return [x[1] for x in os.walk(training_dir_path)][0]
Example #28
Source File: create_encodings.py From Face-Recognition with MIT License | 5 votes |
def _get_training_dirs(training_dir_path): return [x[0] for x in os.walk(training_dir_path)][1:]
Example #29
Source File: test_fuse.py From delocate with BSD 2-Clause "Simplified" License | 5 votes |
def assert_same_tree(tree1, tree2): for dirpath, dirnames, filenames in os.walk(tree1): tree2_dirpath = pjoin(tree2, relpath(dirpath, tree1)) for dname in dirnames: assert_true(isdir(pjoin(tree2_dirpath, dname))) for fname in filenames: tree1_path = pjoin(dirpath, fname) with open_readable(tree1_path, 'rb') as fobj: contents1 = fobj.read() with open_readable(pjoin(tree2_dirpath, fname), 'rb') as fobj: contents2 = fobj.read() if fname == 'RECORD': # Record can have different line orders assert_record_equal(contents1, contents2) else: assert_equal(contents1, contents2)
Example #30
Source File: getmessages_mariadb.py From InsightAgent with Apache License 2.0 | 5 votes |
def get_file_list_for_directory(root_path='/', file_name_regex_c=''): root_path = os.path.expanduser(root_path) if os.path.isdir(root_path): file_list = [] for path, subdirs, files in os.walk(root_path): for name in files: if check_regex(file_name_regex_c, name): file_list.append(os.path.join(path, name)) return file_list elif os.path.isfile(root_path): if check_regex(file_name_regex_c, root_path): return [root_path] return []