Python os.walk() Examples

def find_image_filepaths(dataset_dir):
    """Load image filepaths from the 10k cats dataset."""
    result = []
    for root, dirs, files in os.walk(dataset_dir):
        if "/CAT_" in root:
            for name in files:
                fp = os.path.join(root, name)
                if name.endswith(".jpg") and os.path.isfile("%s.cat" % (fp,)):
                    result.append(fp)
    return result 

def test_pep8(self):
        """Test method to check PEP8 compliance over the entire project."""
        self.file_structure = dirname(dirname(abspath(__file__)))
        print("Testing for PEP8 compliance of python files in {}".format(
            self.file_structure))
        style = pep8.StyleGuide()
        style.options.max_line_length = 100  # Set this to desired maximum line length
        filenames = []
        # Set this to desired folder location
        for root, _, files in os.walk(self.file_structure):
            python_files = [f for f in files if f.endswith(
                '.py') and "examples" not in root]
            for file in python_files:
                if len(root.split('samples')) != 2:     # Ignore samples directory
                    filename = '{0}/{1}'.format(root, file)
                    filenames.append(filename)
        check = style.check_files(filenames)
        self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' %
                         check.total_errors) 

def validate_data_source(cls,pipeline_type):
        dirs = os.walk("{0}/pipelines/".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))).next()[1]
        is_type_ok = True if pipeline_type in dirs else False
        return is_type_ok


#class NewFileEvent(FileSystemEventHandler):
#
#    pipeline_instance = None
#    def __init__(self,pipeline_instance):
#        self.pipeline_instance = pipeline_instance
#
#    def on_moved(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.dest_path)
#
#    def on_created(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.src_path) 

def validate_parameters_values(args,logger):

    logger.info("Validating input parameter values")

    #date.
    is_date_ok = True if len(args.date) == 8 else False

    # type
    dirs = os.walk(script_path).next()[1]
    is_type_ok = True if args.type in dirs else False

    #limit
    try:
        int(args.limit)
        is_limit_ok = True
    except ValueError:
        is_limit_ok = False

    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
    if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
    if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
    if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1) 

def get_names(file_dir, files):
    """
    Get the annotator name list based on a list of files
    Args:
    file_dir: AMR file folder
    files: a list of AMR names, e.g. nw_wsj_0001_1

    Returns:
   a list of user names who annotate all the files
    """
    # for each user, check if they have files available
    # return user name list
    total_list = []
    name_list = []
    get_sub = False
    for path, subdir, dir_files in os.walk(file_dir):
        if not get_sub:
            total_list = subdir[:]
            get_sub = True
        else:
            break
    for user in total_list:
        has_file = True
        for f in files:
            file_path = file_dir + user + "/" + f + ".txt"
            if not os.path.exists(file_path):
                has_file = False
                break
        if has_file:
            name_list.append(user)
    if len(name_list) == 0:
        print("********Error: Cannot find any user who completes the files*************", file=ERROR_LOG)
    return name_list 

def zip_stream(src_dir, output_stream):
        """

        @param src_dir:
        @type src_dir: str
        @param output_stream:
        @type output_stream: zipfile.ZipFile
        @return:
        """
        root_path = os.path.dirname(src_dir)
        assert os.path.isdir(src_dir), "Invalid, not a directory: '{}'".format(src_dir)
        for root, directories, files in os.walk(src_dir):
            for file_name in files:
                file_path = os.path.join(root, file_name)
                relative_path = os.path.relpath(file_path, root_path)
                output_stream.write(file_path, arcname=relative_path) 

def normal_run():
    LOGGER.force_log("info", "normal selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    normalLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and normalSelLog in fn]
    [os.remove(l) for l in normalLogs]
    # set log file name
    LOGGER.set_log_file_basename(normalSelLog)
    # set random group selector
    engine.set_group_selector(RandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "normal selection finished", stdout=True, file=False)

############### run machine learning selector ############### 

def ML_run():
    LOGGER.force_log("info", "machine learning selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    MLLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and MLSelLog in fn]
    [os.remove(l) for l in MLLogs]
    # set log file name
    LOGGER.set_log_file_basename(MLSelLog)
    # set smart group selector
    engine.set_group_selector(SmartRandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "machine learning selection finished", stdout=True, file=False)

##########################################################################################
#####################################  RUN SIMULATION  ################################### 

def make_dataset(dir, class_to_idx):
    images = []
    dir = os.path.expanduser(dir)
    for target in sorted(os.listdir(dir)):
        d = os.path.join(dir, target)
        if not os.path.isdir(d):
            continue

        for root, _, fnames in sorted(os.walk(d)):
            for fname in sorted(fnames):
                if is_image_file(fname):
                    path = os.path.join(root, fname)
                    item = (path, class_to_idx[target])
                    images.append(item)

    return images 

def find_test_path(test_file):
    """Searches for the test file and returns the path if found
    As a default, the currend working directory is the top of the search.
    If a directory was provided as part of the argument, the directory will be
    joined with cwd unless it was an absolute path, in which case, the
    absolute path will be used instead. 
    """
    test_file += ".py"
    test_path = os.path.split(test_file)
    top = os.path.join(os.getcwd(), test_path[0])

    for (path, dirs, files) in os.walk(top):
        if test_path[1] in files:
            return  os.path.join(path, test_path[1])
    raise FileNotFoundError("Could not find " + test_path[1] + 
                            "in directory: " + top) 

def get_unhidden_ungenerated_python_files(directory: str) -> Iterable[str]:
    """Iterates through relevant python files within the given directory.

    Args:
        directory: The top-level directory to explore.

    Yields:
        File paths.
    """
    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
        if os.path.split(dirpath)[-1].startswith('.'):
            dirnames.clear()
            continue

        for filename in filenames:
            if filename.endswith('.py') and not filename.endswith('_pb2.py'):
                yield os.path.join(dirpath, filename) 

def get_file_list_for_directory(root_path='/', file_name_regex=''):
    if root_path is None or len(root_path) == 0:
        return []

    if root_path[-1] != '/':
        if os.path.exists(root_path) and (not file_name_regex or (file_name_regex and re.match(file_name_regex, root_path))):
            return [root_path]
        else:
            return []

    file_list = []
    for path, subdirs, files in os.walk(root_path):
        for name in files:
            if not file_name_regex or (file_name_regex and re.match(file_name_regex, name)):
                file_list.append(os.path.join(path, name))
    return file_list 

def remove_old_files_and_update_filesnames(filenames):
    all_directories = os.listdir(csvpath)
    # aggregate cou for collectd version < 5.5
    aggregate_cpu = False
    # remove old csv files in datadir
    remove_old_files(os.path.join(home_path, data_dir), 'csv')

    for each_dir in all_directories:
        # remove old collectd log files
        remove_old_files(os.path.join(csvpath, each_dir), None)

        if "disk" in each_dir:
            filenames[each_dir + "/disk_octets-"] = [each_dir +
                                                     '_DiskWrite', each_dir + '_DiskRead']
        if "interface" in each_dir:
            filenames[each_dir + "/if_octets-"] = [each_dir +
                                                   '_NetworkIn', each_dir + '_NetworkOut']

    for fEntry in os.walk(os.path.join(csvpath)):
        if "cpu-" in fEntry[0]:
            aggregate_cpu = True
            filenames['aggregation-cpu-average/cpu-system-'] = ['CPU']

    return aggregate_cpu 

def rm_full_dir(path, ignore_errors=False):
    """
    This function is used to remove a directory and all files and
    directories within it (like `rm -rf`).
    """
    if os.path.isdir(path):
        try:
            os.chmod(path, os.stat(path).st_mode | stat.S_IRWXU
                                                 & ~stat.S_ISVTX)
        except OSError:
            pass
        f_last = 0
        while True:
            f_count = 0
            for root, d_names, f_names in os.walk(path):
                try:
                    os.chmod(root, os.stat(root).st_mode | stat.S_IRWXU
                                                         & ~stat.S_ISVTX)
                except OSError:
                    pass
                for fs_name in f_names + d_names:
                    target = os.path.join(root, fs_name)
                    try:
                        os.chmod(target, os.stat(target).st_mode
                                              | stat.S_IRWXU
                                              & ~stat.S_ISVTX)
                    except OSError:
                        pass
                    f_count += 1
                f_count += 1
            # do this until we get the same count twice, ie. all files we can
            # chmod our way into have been found
            if f_last == f_count:
                break
            f_last = f_count
        shutil.rmtree(path, ignore_errors) 

def get_files(*bases):
        """
        List all files in a data directory.
        """
        for base in bases:
            basedir, _ = base.split(".", 1)
            base = os.path.join(os.path.dirname(__file__), *base.split("."))

            rem = len(os.path.dirname(base)) + len(basedir) + 2

            for root, dirs, files in os.walk(base):
                for name in files:
                    yield os.path.join(basedir, root, name)[rem:] 

def get_files(paths,
              exts=(),
              ignore_exts=DEFAULT_IGNORE_EXTENSIONS,
              ignore_folders=DEFAULT_IGNORE_FOLDERS):
    """Return all files matching the defined conditions."""
    all_files = []
    for path in paths:
        if os.path.isfile(path):
            all_files.append(path)
        else:
            for root, folders, files in os.walk(path):
                # Chop out hidden directories
                new_folders = []
                for folder in folders:
                    if folder[0] != '.':
                        tests = [
                            folder != ignore_folder
                            for ignore_folder in ignore_folders
                        ]
                        if all(tests):
                            new_folders.append(folder)
                folders[:] = new_folders

                # Chop out hidden files and walk files
                files = [f for f in files if f[0] != '.']
                for file in files:
                    tests, pass_tests = [True], [True]
                    if ignore_exts:
                        tests = [
                            not file.endswith('.' + ext) for ext in ignore_exts
                        ]
                    if exts:
                        pass_tests = [file.endswith('.' + ext) for ext in exts]

                    if all(tests) and any(pass_tests):
                        all_files.append(os.path.join(root, file))

    return list(sorted(all_files)) 

def _verify_submission_size(self):
    submission_size = 0
    for dirname, _, filenames in os.walk(self._extracted_submission_dir):
      for f in filenames:
        submission_size += os.path.getsize(os.path.join(dirname, f))
    logging.info('Unpacked submission size: %d', submission_size)
    if submission_size > MAX_SUBMISSION_SIZE_UNPACKED:
      logging.error('Submission size exceeding limit %d',
                    MAX_SUBMISSION_SIZE_UNPACKED)
    return submission_size <= MAX_SUBMISSION_SIZE_UNPACKED 

def _verify_submission_size(self):
    submission_size = 0
    for dirname, _, filenames in os.walk(self._extracted_submission_dir):
      for f in filenames:
        submission_size += os.path.getsize(os.path.join(dirname, f))
    logging.info('Unpacked submission size: %d', submission_size)
    if submission_size > MAX_SUBMISSION_SIZE_UNPACKED:
      logging.error('Submission size exceeding limit %d',
                    MAX_SUBMISSION_SIZE_UNPACKED)
    return submission_size <= MAX_SUBMISSION_SIZE_UNPACKED 

def dynamic_download_and_Synthesizing(illust_id, title=None, prefix=None):
    tag = 'Dynamic_Download_And_Synthesizing'
    d_json_data = 'https://www.pixiv.net/ajax/illust/' + str(illust_id) + '/ugoira_meta'
    d_json_decoded = json.loads(get_text_from_url(d_json_data))['body']
    src_zip_url = d_json_decoded['originalSrc']
    src_mime_type = d_json_decoded['mime_type']
    src_img_delay = int(d_json_decoded['frames'][0]['delay']) / 1000
    src_saved_path = save_path + 'TEMP' + global_symbol + str(illust_id) + global_symbol + \
                     src_zip_url.split('/')[-1]
    src_saved_dir = save_path + 'TEMP' + global_symbol + str(illust_id) + global_symbol
    src_final_dir = save_path + 'Dynamic' + global_symbol
    download_thread(src_zip_url, save_path, None, 'TEMP' + global_symbol + str(illust_id))
    while not os.path.exists(src_saved_path + '.done'):
        time.sleep(1)
        print_with_tag(tag, 'Waiting for complete...')
    print_with_tag(tag, ['Zip target downloaded:', src_saved_path])
    with zipfile.ZipFile(src_saved_path, 'r') as zip_file:
        zip_file.extractall(path=src_saved_dir)
    # get each frame
    sort_by_num = []
    frames = []
    for root, dirs, files in os.walk(src_saved_dir):
        for file in files:
            if file.endswith('jpg') or file.endswith('png'):
                sort_by_num.append(src_saved_dir + global_symbol + file)
    sort_by_num.sort()
    print_with_tag(tag, 'Reading each frame..')
    for each_frame in sort_by_num:
        frames.append(imageio.imread(each_frame))
    gif_save_dir = save_path + str(prefix) + global_symbol + year_month + str(
        day) + global_symbol + 'D-' + str(illust_id) + global_symbol
    gif_name_format = re.sub('[\/:*?"<>|]', '_', str(title)) + '-' + str(illust_id) + '.gif'
    if not os.path.exists(gif_save_dir):
        os.makedirs(gif_save_dir)
    print_with_tag(tag, 'Synthesizing dynamic images..')
    try:
        imageio.mimsave(gif_save_dir + gif_name_format, frames, duration=src_img_delay)
    except Exception as e:
        print_with_tag(tag, [gif_save_dir + gif_name_format])
        print_with_tag(tag, e)
        exit() 

def replace_syslinux_modules(syslinux_version, under_this_dir):
    # Replace modules files extracted from iso with corresponding
    # version provided by multibootusb.
    modules_src_dir = os.path.join(
        multibootusb_host_dir(), "syslinux", "modules", syslinux_version)

    for dirpath, dirnames, filenames in os.walk(under_this_dir):
        for fname in filenames:
            if not fname.lower().endswith('.c32'):
                continue
            dst_path = os.path.join(under_this_dir, dirpath, fname)
            src_path = os.path.join(modules_src_dir, fname)
            if not os.path.exists(src_path):
                log("Suitable replacement of '%s' is not bundled. "
                    "Trying to unlzma." % fname)
                try:
                    with lzma.open(dst_path) as f:
                        expanded = f.read()
                except lzma.LZMAError:
                    continue
                except (OSError, IOError) as e:
                    log("%s while accessing %s." % (e, dst_path))
                    continue
                with open(dst_path, 'wb') as f:
                    f.write(expanded)
                log("Successfully decompressed %s." % fname)
                continue
            try:
                os.remove(dst_path)
                shutil.copy(src_path, dst_path)
                log("Replaced %s module" % fname)
            except (OSError, IOError) as err:
                log(err)
                log("Could not update " + fname) 

def get_data(_dir):
    """
    Get path to all files, including sub directories
    :param _dir: Path to top level directory
    :return: Path to files as list
    """
    data = []
    for dirpath, dirnames, filenames in os.walk(_dir):
        for f in filenames:
            cfg_file = os.path.join(dirpath, f)
            data.append(cfg_file)
    return data 

def ls_images(self, relative):
        path = self.to_abs(relative)
        for root, dirs, files in os.walk(path):
            for f in files:
                if self.is_image(f):
                    reldir = os.path.relpath(root, self.root)
                    rel = os.path.join(reldir, f)
                    yield rel 

def recursive_glob(rootdir='.', suffix=''):
    """Performs recursive glob with given suffix and rootdir
        :param rootdir is the root directory
        :param suffix is the suffix to be searched
    """
    return [os.path.join(looproot, filename)
        for looproot, _, filenames in os.walk(rootdir)
        for filename in filenames if filename.endswith(suffix)]

#####################################################
# 解码分割图
##################################################### 

def pytest_generate_tests(metafunc):
    if 'scenario' in metafunc.fixturenames:
        scenarios = {}
        for root, dirnames, filenames in os.walk('tests/scenarios'):
            for filename in fnmatch.filter(filenames, '*.yml'):
                filepath = os.path.join(root, filename)
                scenarios.update(yaml.load(io.open(filepath, encoding='utf-8')) or {})
        params = []
        for name in sorted(scenarios):
            params.append([name, scenarios[name]])
        metafunc.parametrize('name, scenario', params) 

def get_frames(root_path):
   """Get path to all the frame in view SAX and contain complete frames"""
   ret = []
   for root, _, files in os.walk(root_path):
       root=root.replace('\\','/')
       files=[s for s in files if ".dcm" in s]
       if len(files) == 0 or not files[0].endswith(".dcm") or root.find("sax") == -1:
           continue
       prefix = files[0].rsplit('-', 1)[0]
       fileset = set(files)
       expected = ["%s-%04d.dcm" % (prefix, i + 1) for i in range(30)]
       if all(x in fileset for x in expected):
           ret.append([root + "/" + x for x in expected])
   # sort for reproduciblity
   return sorted(ret, key = lambda x: x[0]) 

def filepath_enumerate(paths):
    """Enumerate the file paths of all subfiles of the list of paths"""
    out = []
    for path in paths:
        if os.path.isfile(path):
            out.append(path)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    out.append(os.path.normpath(os.path.join(root, name)))
    return out

# pylint: disable=useless-object-inheritance 

def main():
    """Main entry function."""
    if len(sys.argv) < 3:
        print('Usage: <project-name> <filetype> <list-of-path to traverse>')
        print('\tfiletype can be python/cpp/all')
        exit(-1)
    _HELPER.project_name = sys.argv[1]
    file_type = sys.argv[2]
    allow_type = []
    if file_type == 'python' or file_type == 'all':
        allow_type += [x for x in PYTHON_SUFFIX]
    if file_type == 'cpp' or file_type == 'all':
        allow_type += [x for x in CXX_SUFFIX]
    allow_type = set(allow_type)
    if os.name != 'nt':
        sys.stderr = codecs.StreamReaderWriter(sys.stderr,
                                               codecs.getreader('utf8'),
                                               codecs.getwriter('utf8'),
                                               'replace')
    for path in sys.argv[3:]:
        if os.path.isfile(path):
            process(path, allow_type)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    process(os.path.join(root, name), allow_type)

    nerr = _HELPER.print_summary(sys.stderr)
    sys.exit(nerr > 0) 

def list_image(root, recursive, exts):
    """Traverses the root of directory that contains images and
    generates image list iterator.
    Parameters
    ----------
    root: string
    recursive: bool
    exts: string
    Returns
    -------
    image iterator that contains all the image under the specified path
    """

    i = 0
    if recursive:
        cat = {}
        for path, dirs, files in os.walk(root, followlinks=True):
            dirs.sort()
            files.sort()
            for fname in files:
                fpath = os.path.join(path, fname)
                suffix = os.path.splitext(fname)[1].lower()
                if os.path.isfile(fpath) and (suffix in exts):
                    if path not in cat:
                        cat[path] = len(cat)
                    yield (i, os.path.relpath(fpath, root), cat[path])
                    i += 1
        for k, v in sorted(cat.items(), key=lambda x: x[1]):
            print(os.path.relpath(k, root), v)
    else:
        for fname in sorted(os.listdir(root)):
            fpath = os.path.join(root, fname)
            suffix = os.path.splitext(fname)[1].lower()
            if os.path.isfile(fpath) and (suffix in exts):
                yield (i, os.path.relpath(fpath, root), 0)
                i += 1 

def process_folder(root, action):
    excepts = []
    for root, _, files in os.walk(root):
        for f in files:
            fname = os.path.normpath(os.path.join(root, f))
            if not process_file(fname, action):
                excepts.append(fname)
    if action == 'check' and excepts:
        logging.warning('The following files do not contain a valid license, '+
                        'you can use `python tools/license_header.py add [file]` to add'+
                        'them automatically: ')
        for x in excepts:
            logging.warning(x)
        return False
    return True 

def get_size(self, start_path):
		total_size = 0
		for dirpath, dirnames, filenames in walk(start_path):
			for f in filenames:
				fp = path.join(dirpath, f)
				total_size += path.getsize(fp)
				
		return total_size