Python os.scandir() Examples

def traverse_files(path):
    """
    遍历
    :param path:
    :return:
    """
    for item in os.scandir(path):

        file_name = get_file_name(item.path)[2]
        # 判断是文件夹还是文件
        if item.is_dir():
            # print(item.path)
            # 删除build文件夹
            if file_name == 'build':
                remove_dir(item.path)
            else:
                traverse_files(item.path) 

def get_dir_size(start_path = '.'):
    total_size = 0
    if 'scandir' in dir(os):
        # using fast 'os.scandir' method (new in version 3.5)
        for entry in os.scandir(start_path):
            if entry.is_dir(follow_symlinks = False):
                total_size += get_dir_size(entry.path)
            elif entry.is_file(follow_symlinks = False):
                total_size += entry.stat().st_size
    else:
        # using slow, but compatible 'os.listdir' method
        print (start_path)
        for entry in os.listdir(start_path):
            full_path = os.path.abspath(os.path.join(start_path, entry))
            if os.path.isdir(full_path):
                total_size += get_dir_size(full_path)
            elif os.path.isfile(full_path):
                total_size += os.path.getsize(full_path)
    return total_size

############################################################
###  main ()
############################################################ 

def organize_junk():
    for entry in os.scandir():
        if entry.is_dir():
            continue
        file_path = Path(entry)
        file_format = file_path.suffix.lower()
        if file_format in FILE_FORMATS:
            directory_path = Path(FILE_FORMATS[file_format])
            directory_path.mkdir(exist_ok=True)
            file_path.rename(directory_path.joinpath(file_path))

    try:
        os.mkdir("OTHER-FILES")
    except:
        pass

    for dir in os.scandir():
        try:
            if dir.is_dir():
                os.rmdir(dir)
            else:
                os.rename(os.getcwd() + '/' + str(Path(dir)), os.getcwd() + '/OTHER-FILES/' + str(Path(dir)))
        except:
            pass 

def select_file(path, ext=''):
    file_list = []
    with os.scandir(path) as ls:
        for item in ls:
            if item.is_file():
                filename = str(item.name)
                if ext == '':
                    file_list.append(filename)
                    interrogative = "Select a file: "
                elif filename.endswith(ext):
                    file_list.append(filename)
                    interrogative = "Select "+ext+" file: "
    i = 1
    for file in file_list:
        print("["+str(i)+"] "+file)
        i += 1
    selected = validate_selection(interrogative, file_list)
    return selected 

def walk_output_dir(self, skip_recurse=None):
        """Generate a list of files on disk (even if not in exported files)

        Don't recurse into directories in `skip_recurse`.
        Generate paths with forward slashes on all platforms.
        """
        # os.walk() handles symlinked directories as directories
        # due to this, it's simpler (and faster) to recurse ourselves
        if not os.path.exists(self.output):
            return
        if skip_recurse is None:
            skip_recurse = []
        to_visit = ['']
        while to_visit:
            base = to_visit.pop()
            with os.scandir(f"{self.output}/{base}") as scan_it:
                for entry in scan_it:
                    if entry.is_symlink() or entry.is_file(follow_symlinks=False):
                        yield f"{base}{entry.name}"
                    elif entry.is_dir():
                        path = f"{base}{entry.name}"
                        if path not in skip_recurse:
                            to_visit.append(f"{path}/") 

def _archive_dir(self):
        # the archive expands to a directory named expected-[COMMIT SHA1];
        # we'd like to put this under a stable name
        expected_dirs = [
            de for de in os.scandir(self.source_folder)
            if de.is_dir() and fnmatch(de.name, "expected-*")
        ]
        return expected_dirs[0].name 

def organize_junk():
    for entry in os.scandir():
        if entry.is_dir():
            continue
        file_path = Path(entry.name)
        file_format = file_path.suffix.lower()
        if file_format in FILE_FORMATS:
            directory_path = Path(FILE_FORMATS[file_format])
            directory_path.mkdir(exist_ok=True)
            file_path.rename(directory_path.joinpath(file_path))

    try:
        os.mkdir("OTHER-FILES")
    except:
        pass

    for dir in os.scandir():
        try:
            if dir.is_dir():
                os.rmdir(dir)
            else:
                os.rename(os.getcwd() + '/' + str(Path(dir)), os.getcwd() + '/OTHER-FILES/' + str(Path(dir)))
        except:
            pass 

def find_corresponding_video_file(file_name: str, root_path: str, relative_directory: str):
        """
        This function will attempt to find nfo file's corresponding video file
        """
        if not file_name.endswith('.nfo'):
            return file_name

        filename, _ = os.path.splitext(file_name)
        for f in os.scandir(os.path.join(root_path, relative_directory)):
            _f, _ext = os.path.splitext(f.name)
            if _f == filename and _ext != '.nfo':
                return f.name
        
        # by default just return input file name since nothing is found
        print('[WARNING] cannot find video file for {} in {}'.format(
            file_name, os.path.join(root_path, relative_directory)
        ))
        return file_name 

def __init__(
        self,
        img_dir,
        box_dir,
        n_anchor,
        resolution=DEFAULT_RESOLUTION,
        transform=DEFAULT_TRANSFORM,
    ):
        super().__init__()

        self.img_files = list(sorted(scandir(img_dir), key=lambda f: f.name))
        self.box_files = list(sorted(scandir(box_dir), key=lambda f: f.name))

        self.n_anchor = n_anchor
        # sizes(heights) of anchors have been reduced from the original paper
        # now they are like: [5, 7, 10, 14, 20, ...] (times sqrt(2) each time)
        self.anchors = torch.tensor([5 * (2 ** (i / 2)) for i in range(n_anchor)])

        self.resolution = resolution
        self.grid_resolution = [i // 16 for i in resolution]

        self.transform = transform 

def traverse(path, ignore_files=None):
    if not os.path.exists(path):
        return
    if ignore_files is None:
        ignore_files = []

    for item in scandir(path):
        full_path = os.path.join(path, item.name)
        spec = pathspec.PathSpec.from_lines(
            pathspec.patterns.GitWildMatchPattern, ignore_files
        )
        if spec.match_file(full_path):
            logger.debug("Ignoring %s", item)
            continue

        if item.is_dir():
            for result in traverse(item.path, ignore_files):
                yield os.path.join(item.name, result)
        else:
            yield item.name 

def walk(path, parent_dir=None):
        import importlib
        import os

        for entry in os.scandir(path):

            if entry.is_file() and entry.name.endswith(".py"):
                filename, _ = os.path.splitext(entry.name)
                is_init = filename == "__init__"

                if parent_dir:
                    module = parent_dir if is_init else f"{parent_dir}.{filename}"
                else:
                    if is_init:
                        continue
                    module = filename

                importlib.reload(eval(module))

            elif entry.is_dir() and not entry.name.startswith((".", "__")):
                dirname = f"{parent_dir}.{entry.name}" if parent_dir else entry.name
                walk(entry.path, parent_dir=dirname) 

def get_image_paths( directory ):
    return [ x.path for x in os.scandir( directory ) if x.name.endswith(".jpg") or x.name.endswith(".png") ] 

def _load(self, path):
        corpus = audiomate.Corpus(path=path)

        for directory, type_name in DIRECTORIES.items():
            source_directory = os.path.join(path, directory)

            if not os.path.isdir(source_directory):
                continue

            it = os.scandir(source_directory)

            for entry in it:
                if not entry.name.endswith('.wav'):
                    continue

                file_path = os.path.join(source_directory, entry.name)
                file_idx = entry.name[0:-4]  # chop of .wav
                utterance_idx = file_idx  # every file is a separate utterance

                corpus.new_file(file_path, track_idx=file_idx, copy_file=False)
                utterance = corpus.new_utterance(utterance_idx, file_idx)
                utterance.set_label_list(annotations.LabelList.create_single(type_name, idx=audiomate.corpus.LL_DOMAIN))

        return corpus 

def render_ebook_directory(fpath):
	print("Invoking scandir")
	items = os.scandir(fpath)
	items = [
		{
			'name'      : item.name,
			'fullpath'  : os.path.join(fpath, item.name),
			'directory' : fpath,
			'is_dir'    : item.is_dir(),
			'is_file'   : item.is_file(),
			'relp'      : urllib.parse.urljoin("/epub-reader/", urllib.parse.quote(os.path.relpath(os.path.join(fpath, item.name), settings.EBOOK_STORAGE_DIR)))
		}
		for item in items
	]

	items.sort(key=lambda x: x['name'])
	print("Query done. Rendering with %s items" % (len(items), ))
	return render_template('book-reader/reader-dir.html',
		fpath = fpath,
		items = items,
		) 

def _iterdir(dirname, dironly):
    if not dirname:
        if isinstance(dirname, bytes):
            dirname = bytes(os.curdir, 'ASCII')
        else:
            dirname = os.curdir
    try:
        it = scandir(dirname)
        for entry in it:
            try:
                if not dironly or entry.is_dir():
                    yield entry.name
            except OSError:
                pass
    except OSError:
        return

# Recursively yields relative pathnames inside a literal directory. 

def hash_dirs(dirs: Tuple[str, str]) -> bytes:
    def hash_dir(dirname, ext, paths):
        with os.scandir(dirname) as it:
            for entry in it:
                if entry.is_file() and entry.name.endswith(ext):
                    paths.append(entry.path)
                elif entry.is_dir():
                    hash_dir(entry.path, ext, paths)

    paths = []
    for dirname, ext in dirs:
        hash_dir(dirname, ext, paths)

    h = hashlib.sha1()  # sha1 is the fastest one.
    for path in sorted(paths):
        with open(path, 'rb') as f:
            h.update(f.read())

    return h.digest() 

def files(startFolder=None):
    extensions = (  '.12d', '.cvg', '.dat', '.vwf', '.out', '.nam', '.inp1', '.inp2',
                    '.sta', '.equ', '.eig', '.stm', '.mtx', '.net', '.inp0', '.rin',
                    '.fcv', 'dummy' )
    if not startFolder:
        startFolder = os.getcwd()
    for f in os.scandir(startFolder):
        if f.is_dir(): # if folder
            files(f.path)
        elif f.is_file() and f.name.endswith(extensions):
            try:
                os.remove(f.path)
                sys.__stdout__.write('Delelted: ' + f.path + '\n')
            except:
                sys.__stdout__.write(f.path + ': ' + sys.exc_info()[1][1] + '\n')


# Cleaup old result files 

def get_dir_size(start_path = '.'):
    total_size = 0
    if 'scandir' in dir(os):
        # using fast 'os.scandir' method (new in version 3.5)
        for entry in os.scandir(start_path):
            if entry.is_dir(follow_symlinks = False):
                total_size += get_dir_size(entry.path)
            elif entry.is_file(follow_symlinks = False):
                total_size += entry.stat().st_size
    else:
        # using slow, but compatible 'os.listdir' method
        print (start_path)
        for entry in os.listdir(start_path):
            full_path = os.path.abspath(os.path.join(start_path, entry))
            if os.path.isdir(full_path):
                total_size += get_dir_size(full_path)
            elif os.path.isfile(full_path):
                total_size += os.path.getsize(full_path)
    return total_size

############################################################
###  main ()
############################################################ 

def get_dir_size(start_path = '.'):
    total_size = 0
    if 'scandir' in dir(os):
        # using fast 'os.scandir' method (new in version 3.5)
        for entry in os.scandir(start_path):
            if entry.is_dir(follow_symlinks = False):
                total_size += get_dir_size(entry.path)
            elif entry.is_file(follow_symlinks = False):
                total_size += entry.stat().st_size
    else:
        # using slow, but compatible 'os.listdir' method
        print (start_path)
        for entry in os.listdir(start_path):
            full_path = os.path.abspath(os.path.join(start_path, entry))
            if os.path.isdir(full_path):
                total_size += get_dir_size(full_path)
            elif os.path.isfile(full_path):
                total_size += os.path.getsize(full_path)
    return total_size

############################################################
###  main ()
############################################################ 

def data_reader(input_dir, shuffle=True):
  """Read images from input_dir then shuffle them
  Args:
    input_dir: string, path of input dir, e.g., /path/to/dir
  Returns:
    file_paths: list of strings
  """
  file_paths = []

  for img_file in scandir(input_dir):
    if img_file.name.endswith('.jpg') and img_file.is_file():
      file_paths.append(img_file.path)

  if shuffle:
    # Shuffle the ordering of all image files in order to guarantee
    # random ordering of the images with respect to label in the
    # saved TFRecord files. Make the randomization repeatable.
    shuffled_index = list(range(len(file_paths)))
    random.seed(12345)
    random.shuffle(shuffled_index)

    file_paths = [file_paths[i] for i in shuffled_index]

  return file_paths 

def scan_dir(path):
    target_list = [f.path for f in os.scandir(path) if f.is_dir()]
    target_names = []

    for idx, target in enumerate(target_list):
        if "pycache" not in target:
            target_names.append(os.path.basename(os.path.normpath(target)))

    return target_names 

def find_rest_files(self, path: str):
        def scan(path):
            with os.scandir(path) as it:
                for entry in it:
                    if entry.is_file() and entry.name.endswith('.rst'):
                        files.append(entry.path)
                    if entry.is_dir():
                        scan(entry.path)

        files = []
        scan(path)
        return files 

def scan_all_files_in(start_folder, ext):
    all_files = []
    for f in os.scandir(start_folder):
        if f.is_dir():
            for ff in scan_all_files_in(f.path, ext):
                all_files.append(ff)
        elif f.is_file() and f.name.endswith(ext):
            all_files.append(f.path)
    return sorted(all_files)[:limit]


# Submits all INP models starting from folder 

def results():
    extensions = ('.frd', '.vtk', '.vtu')
    for f in os.scandir('.'):
        if f.name.endswith(extensions):
            try:
                os.remove(f.path)
                sys.__stdout__.write('Delelted: ' + f.path + '\n')
            except:
                sys.__stdout__.write(f.path + ': ' + sys.exc_info()[1][1] + '\n') 

def _get_kinds_for_sub(*, bids_basename, bids_root, sub, ses=None):
    """Retrieve available data kinds for a specific subject and session."""
    subject_dir = op.join(bids_root, f'sub-{sub}')
    if ses is not None:
        subject_dir = op.join(subject_dir, f'ses-{ses}')

    # TODO We do this to ensure we don't accidentally pick up any "spurious"
    # TODO sub-directories. But is that really necessary with valid BIDS data?
    kinds_in_dataset = get_kinds(bids_root=bids_root)
    subdirs = [f.name for f in os.scandir(subject_dir) if f.is_dir()]
    available_kinds = [s for s in subdirs if s in kinds_in_dataset]
    return available_kinds 

def select_dem_from_dir(gdir, dem_source=None, keep_dem_folders=False):
    """Select a DEM from the ones available in an RGI-TOPO glacier directory.

    Parameters
    ----------
    gdir : :py:class:`oggm.GlacierDirectory`
        the glacier directory
    dem_source : str
        the source to pick from (default: NASADEM and COPDEM)
    keep_dem_folders : bool
        the default is to delete the other DEM directories to save space.
        Set this to True to prevent that (e.g. for sensitivity tests)
    """

    # Start by deleting noise
    for fn in ['log.txt', 'diagnostics.json']:
        fp = os.path.join(gdir.dir, fn)
        if os.path.exists(fp):
            os.remove(fp)

    sources = [f.name for f in os.scandir(gdir.dir) if f.is_dir()]

    if dem_source is None:
        if 'NASADEM' in sources:
            dem_source = 'NASADEM'
        else:
            dem_source = 'COPDEM'

    if dem_source not in sources:
        raise RuntimeError('source {} not in folder'.format(dem_source))

    shutil.copyfile(os.path.join(gdir.dir, dem_source, 'dem.tif'),
                    gdir.get_filepath('dem'))
    shutil.copyfile(os.path.join(gdir.dir, dem_source, 'dem_source.txt'),
                    gdir.get_filepath('dem_source'))

    if not keep_dem_folders:
        for source in sources:
            shutil.rmtree(os.path.join(gdir.dir, source)) 

def get_sorted_filenames(dirname, ext):
    return (
        de.name
        for de in sorted(os.scandir(dirname), key=lambda de: de.name)
        if de.is_file() and de.name.endswith('.' + ext)
    ) 

def scandir(dir_path, suffix=None):
        for entry in os.scandir(dir_path):
            if not entry.is_file():
                continue
            filename = entry.name
            if suffix is None:
                yield filename
            elif filename.endswith(suffix):
                yield filename 

def scandir(dir_path, suffix=None, recursive=False):
    """Scan a directory to find the interested files.

    Args:
        dir_path (str | obj:`Path`): Path of the directory.
        suffix (str | tuple(str), optional): File suffix that we are
            interested in. Default: None.
        recursive (bool, optional): If set to True, recursively scan the
            directory. Default: False.

    Returns:
        A generator for all the interested files with relative pathes.
    """
    if isinstance(dir_path, (str, Path)):
        dir_path = str(dir_path)
    else:
        raise TypeError('"dir_path" must be a string or Path object')

    if (suffix is not None) and not isinstance(suffix, (str, tuple)):
        raise TypeError('"suffix" must be a string or tuple of strings')

    root = dir_path

    def _scandir(dir_path, suffix, recursive):
        for entry in os.scandir(dir_path):
            if not entry.name.startswith('.') and entry.is_file():
                rel_path = osp.relpath(entry.path, root)
                if suffix is None:
                    yield rel_path
                elif rel_path.endswith(suffix):
                    yield rel_path
            else:
                if recursive:
                    yield from _scandir(
                        entry.path, suffix=suffix, recursive=recursive)
                else:
                    continue

    return _scandir(dir_path, suffix=suffix, recursive=recursive) 

def scantree(path):
    """Recursively scan a directory tree
    :param str path: path to scan
    :rtype: os.DirEntry
    :return: DirEntry via generator
    """
    for entry in os.scandir(path):
        yield entry
        if entry.is_dir(follow_symlinks=False):
            yield from scantree(entry.path)