Python os.walk() Examples

The following are 30 code examples for showing how to use os.walk(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may want to check out the right sidebar which shows the related API usage.

You may also want to check out all available functions/classes of the module os , or try the search function .

Example 1
Project: incubator-spot   Author: apache   File: utils.py    License: Apache License 2.0 6 votes vote down vote up
def validate_data_source(cls,pipeline_type):
        dirs = os.walk("{0}/pipelines/".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))).next()[1]
        is_type_ok = True if pipeline_type in dirs else False
        return is_type_ok


#class NewFileEvent(FileSystemEventHandler):
#
#    pipeline_instance = None
#    def __init__(self,pipeline_instance):
#        self.pipeline_instance = pipeline_instance
#
#    def on_moved(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.dest_path)
#
#    def on_created(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.src_path) 
Example 2
Project: incubator-spot   Author: apache   File: start_oa.py    License: Apache License 2.0 6 votes vote down vote up
def validate_parameters_values(args,logger):

    logger.info("Validating input parameter values")

    #date.
    is_date_ok = True if len(args.date) == 8 else False

    # type
    dirs = os.walk(script_path).next()[1]
    is_type_ok = True if args.type in dirs else False

    #limit
    try:
        int(args.limit)
        is_limit_ok = True
    except ValueError:
        is_limit_ok = False

    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
    if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
    if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
    if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1) 
Example 3
Project: hydrus   Author: HTTP-APIs   File: test_pep8.py    License: MIT License 6 votes vote down vote up
def test_pep8(self):
        """Test method to check PEP8 compliance over the entire project."""
        self.file_structure = dirname(dirname(abspath(__file__)))
        print("Testing for PEP8 compliance of python files in {}".format(
            self.file_structure))
        style = pep8.StyleGuide()
        style.options.max_line_length = 100  # Set this to desired maximum line length
        filenames = []
        # Set this to desired folder location
        for root, _, files in os.walk(self.file_structure):
            python_files = [f for f in files if f.endswith(
                '.py') and "examples" not in root]
            for file in python_files:
                if len(root.split('samples')) != 2:     # Ignore samples directory
                    filename = '{0}/{1}'.format(root, file)
                    filenames.append(filename)
        check = style.check_files(filenames)
        self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' %
                         check.total_errors) 
Example 4
Project: CAMISIM   Author: CAMI-challenge   File: archive.py    License: Apache License 2.0 6 votes vote down vote up
def zip_stream(src_dir, output_stream):
        """

        @param src_dir:
        @type src_dir: str
        @param output_stream:
        @type output_stream: zipfile.ZipFile
        @return:
        """
        root_path = os.path.dirname(src_dir)
        assert os.path.isdir(src_dir), "Invalid, not a directory: '{}'".format(src_dir)
        for root, directories, files in os.walk(src_dir):
            for file_name in files:
                file_path = os.path.join(root, file_name)
                relative_path = os.path.relpath(file_path, root_path)
                output_stream.write(file_path, arcname=relative_path) 
Example 5
Project: fullrmc   Author: bachiraoun   File: run.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def normal_run():
    LOGGER.force_log("info", "normal selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    normalLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and normalSelLog in fn]
    [os.remove(l) for l in normalLogs]
    # set log file name
    LOGGER.set_log_file_basename(normalSelLog)
    # set random group selector
    engine.set_group_selector(RandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "normal selection finished", stdout=True, file=False)

############### run machine learning selector ############### 
Example 6
Project: fullrmc   Author: bachiraoun   File: run.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def ML_run():
    LOGGER.force_log("info", "machine learning selection %i started... DON'T INTERRUPT"%numberOfSteps, stdout=True, file=False)
    # load fresh engine
    engine = create_engine()
    # delete existing log files
    MLLogs = [fn for fn in next(os.walk("."))[2] if ".log" in fn and MLSelLog in fn]
    [os.remove(l) for l in MLLogs]
    # set log file name
    LOGGER.set_log_file_basename(MLSelLog)
    # set smart group selector
    engine.set_group_selector(SmartRandomSelector(engine))
    # run engine
    engine.run(numberOfSteps=numberOfSteps, saveFrequency=2*numberOfSteps, restartPdb=None)
    LOGGER.force_log("info", "machine learning selection finished", stdout=True, file=False)

##########################################################################################
#####################################  RUN SIMULATION  ################################### 
Example 7
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: data.py    License: Apache License 2.0 6 votes vote down vote up
def make_dataset(dir, class_to_idx):
    images = []
    dir = os.path.expanduser(dir)
    for target in sorted(os.listdir(dir)):
        d = os.path.join(dir, target)
        if not os.path.isdir(d):
            continue

        for root, _, fnames in sorted(os.walk(d)):
            for fname in sorted(fnames):
                if is_image_file(fname):
                    path = os.path.join(root, fname)
                    item = (path, class_to_idx[target])
                    images.append(item)

    return images 
Example 8
def find_test_path(test_file):
    """Searches for the test file and returns the path if found
    As a default, the currend working directory is the top of the search.
    If a directory was provided as part of the argument, the directory will be
    joined with cwd unless it was an absolute path, in which case, the
    absolute path will be used instead. 
    """
    test_file += ".py"
    test_path = os.path.split(test_file)
    top = os.path.join(os.getcwd(), test_path[0])

    for (path, dirs, files) in os.walk(top):
        if test_path[1] in files:
            return  os.path.join(path, test_path[1])
    raise FileNotFoundError("Could not find " + test_path[1] + 
                            "in directory: " + top) 
Example 9
Project: OpenFermion-Cirq   Author: quantumlib   File: env_tools.py    License: Apache License 2.0 6 votes vote down vote up
def get_unhidden_ungenerated_python_files(directory: str) -> Iterable[str]:
    """Iterates through relevant python files within the given directory.

    Args:
        directory: The top-level directory to explore.

    Yields:
        File paths.
    """
    for dirpath, dirnames, filenames in os.walk(directory, topdown=True):
        if os.path.split(dirpath)[-1].startswith('.'):
            dirnames.clear()
            continue

        for filename in filenames:
            if filename.endswith('.py') and not filename.endswith('_pb2.py'):
                yield os.path.join(dirpath, filename) 
Example 10
Project: InsightAgent   Author: insightfinder   File: getlogs_evtx.py    License: Apache License 2.0 6 votes vote down vote up
def get_file_list_for_directory(root_path='/', file_name_regex=''):
    if root_path is None or len(root_path) == 0:
        return []

    if root_path[-1] != '/':
        if os.path.exists(root_path) and (not file_name_regex or (file_name_regex and re.match(file_name_regex, root_path))):
            return [root_path]
        else:
            return []

    file_list = []
    for path, subdirs, files in os.walk(root_path):
        for name in files:
            if not file_name_regex or (file_name_regex and re.match(file_name_regex, name)):
                file_list.append(os.path.join(path, name))
    return file_list 
Example 11
Project: InsightAgent   Author: insightfinder   File: collectdReportMetrics.py    License: Apache License 2.0 6 votes vote down vote up
def remove_old_files_and_update_filesnames(filenames):
    all_directories = os.listdir(csvpath)
    # aggregate cou for collectd version < 5.5
    aggregate_cpu = False
    # remove old csv files in datadir
    remove_old_files(os.path.join(home_path, data_dir), 'csv')

    for each_dir in all_directories:
        # remove old collectd log files
        remove_old_files(os.path.join(csvpath, each_dir), None)

        if "disk" in each_dir:
            filenames[each_dir + "/disk_octets-"] = [each_dir +
                                                     '_DiskWrite', each_dir + '_DiskRead']
        if "interface" in each_dir:
            filenames[each_dir + "/if_octets-"] = [each_dir +
                                                   '_NetworkIn', each_dir + '_NetworkOut']

    for fEntry in os.walk(os.path.join(csvpath)):
        if "cpu-" in fEntry[0]:
            aggregate_cpu = True
            filenames['aggregation-cpu-average/cpu-system-'] = ['CPU']

    return aggregate_cpu 
Example 12
Project: cat-bbs   Author: aleju   File: create_dataset.py    License: MIT License 5 votes vote down vote up
def find_image_filepaths(dataset_dir):
    """Load image filepaths from the 10k cats dataset."""
    result = []
    for root, dirs, files in os.walk(dataset_dir):
        if "/CAT_" in root:
            for name in files:
                fp = os.path.join(root, name)
                if name.endswith(".jpg") and os.path.isfile("%s.cat" % (fp,)):
                    result.append(fp)
    return result 
Example 13
Project: aegea   Author: kislyuk   File: cloudinit.py    License: Apache License 2.0 5 votes vote down vote up
def get_bootstrap_files(rootfs_skel_dirs, dest="cloudinit"):
    manifest = OrderedDict()
    aegea_conf = os.getenv("AEGEA_CONFIG_FILE")
    targz = io.BytesIO()
    tar = tarfile.open(mode="w:gz", fileobj=targz) if dest == "tarfile" else None

    for rootfs_skel_dir in rootfs_skel_dirs:
        if rootfs_skel_dir == "auto":
            fn = os.path.join(os.path.dirname(__file__), "..", "rootfs.skel")
        elif aegea_conf:
            # FIXME: not compatible with colon-separated AEGEA_CONFIG_FILE
            fn = os.path.join(os.path.dirname(aegea_conf), rootfs_skel_dir)
        elif os.path.exists(rootfs_skel_dir):
            fn = os.path.abspath(os.path.normpath(rootfs_skel_dir))
        else:
            raise Exception("rootfs_skel directory {} not found".format(fn))
        logger.debug("Trying rootfs.skel: %s" % fn)
        if not os.path.exists(fn):
            raise Exception("rootfs_skel directory {} not found".format(fn))
        for root, dirs, files in os.walk(fn):
            for file_ in files:
                path = os.path.join("/", os.path.relpath(root, fn), file_)
                if dest == "cloudinit":
                    add_file_to_cloudinit_manifest(os.path.join(root, file_), path, manifest)
                elif dest == "tarfile":
                    tar.add(os.path.join(root, file_), path)

    if dest == "cloudinit":
        return list(manifest.values())
    elif dest == "tarfile":
        tar.close()
        return targz.getvalue() 
Example 14
Project: ALF   Author: blackberry   File: __init__.py    License: Apache License 2.0 5 votes vote down vote up
def rm_full_dir(path, ignore_errors=False):
    """
    This function is used to remove a directory and all files and
    directories within it (like `rm -rf`).
    """
    if os.path.isdir(path):
        try:
            os.chmod(path, os.stat(path).st_mode | stat.S_IRWXU
                                                 & ~stat.S_ISVTX)
        except OSError:
            pass
        f_last = 0
        while True:
            f_count = 0
            for root, d_names, f_names in os.walk(path):
                try:
                    os.chmod(root, os.stat(root).st_mode | stat.S_IRWXU
                                                         & ~stat.S_ISVTX)
                except OSError:
                    pass
                for fs_name in f_names + d_names:
                    target = os.path.join(root, fs_name)
                    try:
                        os.chmod(target, os.stat(target).st_mode
                                              | stat.S_IRWXU
                                              & ~stat.S_ISVTX)
                    except OSError:
                        pass
                    f_count += 1
                f_count += 1
            # do this until we get the same count twice, ie. all files we can
            # chmod our way into have been found
            if f_last == f_count:
                break
            f_last = f_count
        shutil.rmtree(path, ignore_errors) 
Example 15
Project: smatch   Author: snowblink14   File: smatch-table.py    License: MIT License 5 votes vote down vote up
def get_names(file_dir, files):
    """
    Get the annotator name list based on a list of files
    Args:
    file_dir: AMR file folder
    files: a list of AMR names, e.g. nw_wsj_0001_1

    Returns:
   a list of user names who annotate all the files
    """
    # for each user, check if they have files available
    # return user name list
    total_list = []
    name_list = []
    get_sub = False
    for path, subdir, dir_files in os.walk(file_dir):
        if not get_sub:
            total_list = subdir[:]
            get_sub = True
        else:
            break
    for user in total_list:
        has_file = True
        for f in files:
            file_path = file_dir + user + "/" + f + ".txt"
            if not os.path.exists(file_path):
                has_file = False
                break
        if has_file:
            name_list.append(user)
    if len(name_list) == 0:
        print("********Error: Cannot find any user who completes the files*************", file=ERROR_LOG)
    return name_list 
Example 16
Project: django-click   Author: GaretJax   File: setup.py    License: MIT License 5 votes vote down vote up
def get_files(*bases):
        """
        List all files in a data directory.
        """
        for base in bases:
            basedir, _ = base.split(".", 1)
            base = os.path.join(os.path.dirname(__file__), *base.split("."))

            rem = len(os.path.dirname(base)) + len(basedir) + 2

            for root, dirs, files in os.walk(base):
                for name in files:
                    yield os.path.join(basedir, root, name)[rem:] 
Example 17
Project: ciocheck   Author: ContinuumIO   File: utils.py    License: MIT License 5 votes vote down vote up
def get_files(paths,
              exts=(),
              ignore_exts=DEFAULT_IGNORE_EXTENSIONS,
              ignore_folders=DEFAULT_IGNORE_FOLDERS):
    """Return all files matching the defined conditions."""
    all_files = []
    for path in paths:
        if os.path.isfile(path):
            all_files.append(path)
        else:
            for root, folders, files in os.walk(path):
                # Chop out hidden directories
                new_folders = []
                for folder in folders:
                    if folder[0] != '.':
                        tests = [
                            folder != ignore_folder
                            for ignore_folder in ignore_folders
                        ]
                        if all(tests):
                            new_folders.append(folder)
                folders[:] = new_folders

                # Chop out hidden files and walk files
                files = [f for f in files if f[0] != '.']
                for file in files:
                    tests, pass_tests = [True], [True]
                    if ignore_exts:
                        tests = [
                            not file.endswith('.' + ext) for ext in ignore_exts
                        ]
                    if exts:
                        pass_tests = [file.endswith('.' + ext) for ext in exts]

                    if all(tests) and any(pass_tests):
                        all_files.append(os.path.join(root, file))

    return list(sorted(all_files)) 
Example 18
def _verify_submission_size(self):
    submission_size = 0
    for dirname, _, filenames in os.walk(self._extracted_submission_dir):
      for f in filenames:
        submission_size += os.path.getsize(os.path.join(dirname, f))
    logging.info('Unpacked submission size: %d', submission_size)
    if submission_size > MAX_SUBMISSION_SIZE_UNPACKED:
      logging.error('Submission size exceeding limit %d',
                    MAX_SUBMISSION_SIZE_UNPACKED)
    return submission_size <= MAX_SUBMISSION_SIZE_UNPACKED 
Example 19
def _verify_submission_size(self):
    submission_size = 0
    for dirname, _, filenames in os.walk(self._extracted_submission_dir):
      for f in filenames:
        submission_size += os.path.getsize(os.path.join(dirname, f))
    logging.info('Unpacked submission size: %d', submission_size)
    if submission_size > MAX_SUBMISSION_SIZE_UNPACKED:
      logging.error('Submission size exceeding limit %d',
                    MAX_SUBMISSION_SIZE_UNPACKED)
    return submission_size <= MAX_SUBMISSION_SIZE_UNPACKED 
Example 20
Project: Starx_Pixiv_Collector   Author: SuzukiHonoka   File: start.py    License: MIT License 5 votes vote down vote up
def dynamic_download_and_Synthesizing(illust_id, title=None, prefix=None):
    tag = 'Dynamic_Download_And_Synthesizing'
    d_json_data = 'https://www.pixiv.net/ajax/illust/' + str(illust_id) + '/ugoira_meta'
    d_json_decoded = json.loads(get_text_from_url(d_json_data))['body']
    src_zip_url = d_json_decoded['originalSrc']
    src_mime_type = d_json_decoded['mime_type']
    src_img_delay = int(d_json_decoded['frames'][0]['delay']) / 1000
    src_saved_path = save_path + 'TEMP' + global_symbol + str(illust_id) + global_symbol + \
                     src_zip_url.split('/')[-1]
    src_saved_dir = save_path + 'TEMP' + global_symbol + str(illust_id) + global_symbol
    src_final_dir = save_path + 'Dynamic' + global_symbol
    download_thread(src_zip_url, save_path, None, 'TEMP' + global_symbol + str(illust_id))
    while not os.path.exists(src_saved_path + '.done'):
        time.sleep(1)
        print_with_tag(tag, 'Waiting for complete...')
    print_with_tag(tag, ['Zip target downloaded:', src_saved_path])
    with zipfile.ZipFile(src_saved_path, 'r') as zip_file:
        zip_file.extractall(path=src_saved_dir)
    # get each frame
    sort_by_num = []
    frames = []
    for root, dirs, files in os.walk(src_saved_dir):
        for file in files:
            if file.endswith('jpg') or file.endswith('png'):
                sort_by_num.append(src_saved_dir + global_symbol + file)
    sort_by_num.sort()
    print_with_tag(tag, 'Reading each frame..')
    for each_frame in sort_by_num:
        frames.append(imageio.imread(each_frame))
    gif_save_dir = save_path + str(prefix) + global_symbol + year_month + str(
        day) + global_symbol + 'D-' + str(illust_id) + global_symbol
    gif_name_format = re.sub('[\/:*?"<>|]', '_', str(title)) + '-' + str(illust_id) + '.gif'
    if not os.path.exists(gif_save_dir):
        os.makedirs(gif_save_dir)
    print_with_tag(tag, 'Synthesizing dynamic images..')
    try:
        imageio.mimsave(gif_save_dir + gif_name_format, frames, duration=src_img_delay)
    except Exception as e:
        print_with_tag(tag, [gif_save_dir + gif_name_format])
        print_with_tag(tag, e)
        exit() 
Example 21
Project: multibootusb   Author: mbusb   File: install.py    License: GNU General Public License v2.0 5 votes vote down vote up
def replace_syslinux_modules(syslinux_version, under_this_dir):
    # Replace modules files extracted from iso with corresponding
    # version provided by multibootusb.
    modules_src_dir = os.path.join(
        multibootusb_host_dir(), "syslinux", "modules", syslinux_version)

    for dirpath, dirnames, filenames in os.walk(under_this_dir):
        for fname in filenames:
            if not fname.lower().endswith('.c32'):
                continue
            dst_path = os.path.join(under_this_dir, dirpath, fname)
            src_path = os.path.join(modules_src_dir, fname)
            if not os.path.exists(src_path):
                log("Suitable replacement of '%s' is not bundled. "
                    "Trying to unlzma." % fname)
                try:
                    with lzma.open(dst_path) as f:
                        expanded = f.read()
                except lzma.LZMAError:
                    continue
                except (OSError, IOError) as e:
                    log("%s while accessing %s." % (e, dst_path))
                    continue
                with open(dst_path, 'wb') as f:
                    f.write(expanded)
                log("Successfully decompressed %s." % fname)
                continue
            try:
                os.remove(dst_path)
                shutil.copy(src_path, dst_path)
                log("Replaced %s module" % fname)
            except (OSError, IOError) as err:
                log(err)
                log("Could not update " + fname) 
Example 22
Project: multibootusb   Author: mbusb   File: setup.py    License: GNU General Public License v2.0 5 votes vote down vote up
def get_data(_dir):
    """
    Get path to all files, including sub directories
    :param _dir: Path to top level directory
    :return: Path to files as list
    """
    data = []
    for dirpath, dirnames, filenames in os.walk(_dir):
        for f in filenames:
            cfg_file = os.path.join(dirpath, f)
            data.append(cfg_file)
    return data 
Example 23
Project: mlimages   Author: icoxfog417   File: file_api.py    License: MIT License 5 votes vote down vote up
def ls_images(self, relative):
        path = self.to_abs(relative)
        for root, dirs, files in os.walk(path):
            for f in files:
                if self.is_image(f):
                    reldir = os.path.relpath(root, self.root)
                    rel = os.path.join(reldir, f)
                    yield rel 
Example 24
Project: DeepLab_v3_plus   Author: songdejia   File: util.py    License: MIT License 5 votes vote down vote up
def recursive_glob(rootdir='.', suffix=''):
    """Performs recursive glob with given suffix and rootdir
        :param rootdir is the root directory
        :param suffix is the suffix to be searched
    """
    return [os.path.join(looproot, filename)
        for looproot, _, filenames in os.walk(rootdir)
        for filename in filenames if filename.endswith(suffix)]

#####################################################
# 解码分割图
##################################################### 
Example 25
Project: goodtables-py   Author: frictionlessdata   File: conftest.py    License: MIT License 5 votes vote down vote up
def pytest_generate_tests(metafunc):
    if 'scenario' in metafunc.fixturenames:
        scenarios = {}
        for root, dirnames, filenames in os.walk('tests/scenarios'):
            for filename in fnmatch.filter(filenames, '*.yml'):
                filepath = os.path.join(root, filename)
                scenarios.update(yaml.load(io.open(filepath, encoding='utf-8')) or {})
        params = []
        for name in sorted(scenarios):
            params.append([name, scenarios[name]])
        metafunc.parametrize('name, scenario', params) 
Example 26
def get_frames(root_path):
   """Get path to all the frame in view SAX and contain complete frames"""
   ret = []
   for root, _, files in os.walk(root_path):
       root=root.replace('\\','/')
       files=[s for s in files if ".dcm" in s]
       if len(files) == 0 or not files[0].endswith(".dcm") or root.find("sax") == -1:
           continue
       prefix = files[0].rsplit('-', 1)[0]
       fileset = set(files)
       expected = ["%s-%04d.dcm" % (prefix, i + 1) for i in range(30)]
       if all(x in fileset for x in expected):
           ret.append([root + "/" + x for x in expected])
   # sort for reproduciblity
   return sorted(ret, key = lambda x: x[0]) 
Example 27
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: lint.py    License: Apache License 2.0 5 votes vote down vote up
def filepath_enumerate(paths):
    """Enumerate the file paths of all subfiles of the list of paths"""
    out = []
    for path in paths:
        if os.path.isfile(path):
            out.append(path)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    out.append(os.path.normpath(os.path.join(root, name)))
    return out

# pylint: disable=useless-object-inheritance 
Example 28
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: lint.py    License: Apache License 2.0 5 votes vote down vote up
def main():
    """Main entry function."""
    if len(sys.argv) < 3:
        print('Usage: <project-name> <filetype> <list-of-path to traverse>')
        print('\tfiletype can be python/cpp/all')
        exit(-1)
    _HELPER.project_name = sys.argv[1]
    file_type = sys.argv[2]
    allow_type = []
    if file_type == 'python' or file_type == 'all':
        allow_type += [x for x in PYTHON_SUFFIX]
    if file_type == 'cpp' or file_type == 'all':
        allow_type += [x for x in CXX_SUFFIX]
    allow_type = set(allow_type)
    if os.name != 'nt':
        sys.stderr = codecs.StreamReaderWriter(sys.stderr,
                                               codecs.getreader('utf8'),
                                               codecs.getwriter('utf8'),
                                               'replace')
    for path in sys.argv[3:]:
        if os.path.isfile(path):
            process(path, allow_type)
        else:
            for root, dirs, files in os.walk(path):
                for name in files:
                    process(os.path.join(root, name), allow_type)

    nerr = _HELPER.print_summary(sys.stderr)
    sys.exit(nerr > 0) 
Example 29
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: im2rec.py    License: Apache License 2.0 5 votes vote down vote up
def list_image(root, recursive, exts):
    """Traverses the root of directory that contains images and
    generates image list iterator.
    Parameters
    ----------
    root: string
    recursive: bool
    exts: string
    Returns
    -------
    image iterator that contains all the image under the specified path
    """

    i = 0
    if recursive:
        cat = {}
        for path, dirs, files in os.walk(root, followlinks=True):
            dirs.sort()
            files.sort()
            for fname in files:
                fpath = os.path.join(path, fname)
                suffix = os.path.splitext(fname)[1].lower()
                if os.path.isfile(fpath) and (suffix in exts):
                    if path not in cat:
                        cat[path] = len(cat)
                    yield (i, os.path.relpath(fpath, root), cat[path])
                    i += 1
        for k, v in sorted(cat.items(), key=lambda x: x[1]):
            print(os.path.relpath(k, root), v)
    else:
        for fname in sorted(os.listdir(root)):
            fpath = os.path.join(root, fname)
            suffix = os.path.splitext(fname)[1].lower()
            if os.path.isfile(fpath) and (suffix in exts):
                yield (i, os.path.relpath(fpath, root), 0)
                i += 1 
Example 30
def process_folder(root, action):
    excepts = []
    for root, _, files in os.walk(root):
        for f in files:
            fname = os.path.normpath(os.path.join(root, f))
            if not process_file(fname, action):
                excepts.append(fname)
    if action == 'check' and excepts:
        logging.warning('The following files do not contain a valid license, '+
                        'you can use `python tools/license_header.py add [file]` to add'+
                        'them automatically: ')
        for x in excepts:
            logging.warning(x)
        return False
    return True