Python os.walk() Examples

The following are code examples for showing how to use os.walk(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: fs_image   Author: facebookincubator   File: test_filesystem_storage.py    MIT License 6 votes vote down vote up
def test_write_and_read_back(self):
        expected_content_count = Counter()
        with self._temp_storage() as storage:
            for writes, _ in self.check_storage_impl(storage):
                expected_content_count[b''.join(writes)] += 1

            # Make a histogram of the contents of the output files
            content_count = Counter()
            for f in itertools.chain.from_iterable(
                [os.path.join(p, f) for f in fs]
                    for p, _, fs in os.walk(storage.base_dir) if fs
            ):
                with open(f, 'rb') as infile:
                    content_count[infile.read()] += 1

            # Did we produce the expected number of each kind of output?
            self.assertEqual(expected_content_count, content_count)

    # This test cannot be in the base since there's no generic way to check
    # if we left a trace on the storage system -- there's no ID to fetch. 
Example 2
Project: fs_image   Author: facebookincubator   File: mount_utils.py    MIT License 6 votes vote down vote up
def mountpoints_from_subvol_meta(subvol: Subvol) -> Iterator[str]:
    '''
    Returns image-relative paths to mountpoints.  Directories get a trailing
    /, while files do not.  See the `_protected_path_set` docblock if this
    convention proves onerous.
    '''
    mounts_path = subvol.path(META_MOUNTS_DIR)
    if not os.path.exists(mounts_path):
        return
    for path, _next_dirs, _files in os.walk(
        # We are not `chroot`ed, so following links could access outside the
        # image; `followlinks=False` is the default -- explicit for safety.
        mounts_path, onerror=_raise, followlinks=False,
    ):
        relpath = os.path.relpath(path, subvol.path(META_MOUNTS_DIR)).decode()
        if os.path.basename(relpath) == MOUNT_MARKER:
            mountpoint = os.path.dirname(relpath)
            assert not mountpoint.endswith('/'), mountpoint
            # It would be more technically correct to use `subvol.path()`
            # here (since that prevents us from following links outside the
            # image), but this is much more legible and probably safe.
            with open(os.path.join(path, b'is_directory')) as f:
                is_directory = json.load(f)
            yield mountpoint + ('/' if is_directory else '') 
Example 3
Project: incubator-spot   Author: apache   File: start_oa.py    Apache License 2.0 6 votes vote down vote up
def validate_parameters_values(args,logger):

    logger.info("Validating input parameter values")

    #date.
    is_date_ok = True if len(args.date) == 8 else False

    # type
    dirs = os.walk(script_path).next()[1]
    is_type_ok = True if args.type in dirs else False

    #limit
    try:
        int(args.limit)
        is_limit_ok = True
    except ValueError:
        is_limit_ok = False

    if not is_date_ok: logger.error("date parameter is not correct, please validate it")
    if not is_type_ok: logger.error("type parameter is not supported, please select a valid type")
    if not is_limit_ok: logger.error("limit parameter is not correct, please select a valid limit")
    if not is_date_ok or not is_type_ok or not is_limit_ok: sys.exit(1) 
Example 4
Project: malcode   Author: moonsea   File: filtersamemd5.py    GNU General Public License v3.0 6 votes vote down vote up
def traveseFile(path):
    for parent, dirnames, filenames in os.walk(path):
        print filenames
        log('Entering', parent, subpath='ndisasm')
        # normal file
        log('origin', str(len(filenames)), subpath='ndisasm')
        # unpack file
        # log('origin', str(countFile(parent, 'dump')))

        total = 0
        for filename in filenames:
            print filename
            filepath = os.path.join(parent, filename)
            log('asming', filename, subpath='ndisasm')

            filemd5 = getMd5(filepath)
            asmpath = os.path.join(asmPath, filemd5)

            total = genAsm(filepath, total, asmpath)

        log('genasm', str(countFile(asmPath)), subpath='ndisasm') 
Example 5
Project: malcode   Author: moonsea   File: getOpCode.py    GNU General Public License v3.0 6 votes vote down vote up
def traveseFile(path):
    for parent, dirnames, filenames in os.walk(path):
        log('Entering', parent, subpath='opcode')

        for filename in filenames:
            if(not checkFileType(filename, 'asm')):
                continue

            filepath = os.path.join(parent, filename)
            print filepath

            with open(filepath) as asmfile:
                lines = asmfile.readlines()

            if(isOpCodeFile(lines)):
                print 'opcode'
                log('OpCoding', filename, subpath='opcode')
                getOpCode(lines, filename)
            else:
                print 'Binary'
                log('Bytecoding', filename, subpath='opcode')
                getByteCode(parent, filename) 
Example 6
Project: malcode   Author: moonsea   File: virusgenasm.py    GNU General Public License v3.0 6 votes vote down vote up
def traveseFile(path):
    for parent, dirnames, filenames in os.walk(path):
        print filenames
        log('Entering', parent, subpath='ndisasm')
        # normal file
        log('origin', str(len(filenames)), subpath='ndisasm')
        # unpack file
        # log('origin', str(countFile(parent, 'dump')))

        total = 0
        for filename in filenames:
            print filename
            filepath = os.path.join(parent, filename)
            log('asming', filename, subpath='ndisasm')

            filemd5 = getMd5(filepath)
            asmpath = os.path.join(asmPath, filemd5)

            total = genAsm(filepath, total, asmpath)

        log('genasm', str(countFile(asmPath)), subpath='ndisasm') 
Example 7
Project: hydrus   Author: HTTP-APIs   File: test_pep8.py    MIT License 6 votes vote down vote up
def test_pep8(self):
        """Test method to check PEP8 compliance over the entire project."""
        self.file_structure = dirname(dirname(abspath(__file__)))
        print("Testing for PEP8 compliance of python files in {}".format(
            self.file_structure))
        style = pep8.StyleGuide()
        style.options.max_line_length = 100  # Set this to desired maximum line length
        filenames = []
        # Set this to desired folder location
        for root, _, files in os.walk(self.file_structure):
            python_files = [f for f in files if f.endswith(
                '.py') and "examples" not in root]
            for file in python_files:
                if len(root.split('samples')) != 2:     # Ignore samples directory
                    filename = '{0}/{1}'.format(root, file)
                    filenames.append(filename)
        check = style.check_files(filenames)
        self.assertEqual(check.total_errors, 0, 'PEP8 style errors: %d' %
                         check.total_errors) 
Example 8
Project: pyblish-win   Author: pyblish   File: refactor.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def refactor_dir(self, dir_name, write=False, doctests_only=False):
        """Descends down a directory and refactor every Python file found.

        Python files are assumed to have a .py extension.

        Files and subdirectories starting with '.' are skipped.
        """
        py_ext = os.extsep + "py"
        for dirpath, dirnames, filenames in os.walk(dir_name):
            self.log_debug("Descending into %s", dirpath)
            dirnames.sort()
            filenames.sort()
            for name in filenames:
                if (not name.startswith(".") and
                    os.path.splitext(name)[1] == py_ext):
                    fullname = os.path.join(dirpath, name)
                    self.refactor_file(fullname, write, doctests_only)
            # Modify dirnames in-place to remove subdirs with leading dots
            dirnames[:] = [dn for dn in dirnames if not dn.startswith(".")] 
Example 9
Project: pyblish-win   Author: pyblish   File: mailbox.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def remove_folder(self, folder):
        """Delete the named folder, which must be empty."""
        path = os.path.join(self._path, '.' + folder)
        for entry in os.listdir(os.path.join(path, 'new')) + \
                     os.listdir(os.path.join(path, 'cur')):
            if len(entry) < 1 or entry[0] != '.':
                raise NotEmptyError('Folder contains message(s): %s' % folder)
        for entry in os.listdir(path):
            if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
               os.path.isdir(os.path.join(path, entry)):
                raise NotEmptyError("Folder contains subdirectory '%s': %s" %
                                    (folder, entry))
        for root, dirs, files in os.walk(path, topdown=False):
            for entry in files:
                os.remove(os.path.join(root, entry))
            for entry in dirs:
                os.rmdir(os.path.join(root, entry))
        os.rmdir(path) 
Example 10
Project: pyblish-win   Author: pyblish   File: test_runpy.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def _del_pkg(self, top, depth, mod_name):
        for entry in list(sys.modules):
            if entry.startswith("__runpy_pkg__"):
                del sys.modules[entry]
        if verbose: print "  Removed sys.modules entries"
        del sys.path[0]
        if verbose: print "  Removed sys.path entry"
        for root, dirs, files in os.walk(top, topdown=False):
            for name in files:
                try:
                    os.remove(os.path.join(root, name))
                except OSError, ex:
                    if verbose: print ex # Persist with cleaning up
            for name in dirs:
                fullname = os.path.join(root, name)
                try:
                    os.rmdir(fullname)
                except OSError, ex:
                    if verbose: print ex # Persist with cleaning up 
Example 11
Project: pygis   Author: giswqs   File: zipfile.py    MIT License 6 votes vote down vote up
def get_all_file_paths(directory): 
  
    # initializing empty file paths list 
    file_paths = [] 
  
    # crawling through directory and subdirectories 
    for root, directories, files in os.walk(directory): 
        for filename in files: 
            # join the two strings in order to form the full filepath. 
            filepath = os.path.join(root, filename) 
            file_paths.append(filepath) 
  
    # returning all file paths 
    return file_paths      


# Writing to a zip file 
Example 12
Project: wikilinks   Author: trovdimi   File: redirectscandidatespostioninserter.py    MIT License 6 votes vote down vote up
def manageWork(self):
        #file = open("/home/ddimitrov/20160305_en_wikilinks/tmp/missing_article_ids.p",'r')
        file = open(SSD_HOME+"pickle/redirects_ids.obj",'r')
        object_file = pickle.load(file)
        #print object_file
        #print type(object_file)
        for root, dirs, files in os.walk(STATIC_HTML_DUMP_ARTICLES_DIR+self.path):
            for i, file_name in enumerate(files):
                if file_name.endswith(".zip"):
                    parts = file_name.split('_')
                    if long(parts[1]) in object_file:

                        try:
                            self.parse_article(file_name,root)
                        except  Exception as e:
                            print("FILENAME_FAIL:"+file_name)
                            print(type(e))    # the exception instance
                            print(e)
                            print (e.message) 
Example 13
Project: git-deployment-scripts   Author: codechick   File: deployment.py    MIT License 6 votes vote down vote up
def __unlink_shared_folders(self):
        self.__print('Linking shared directories...')

        #scanning the share directory. the direcoty to symlink in the share will replicate the subfolder tree and a symlink will be added
        #when a subfolder contains at least a file
        for dir_path, subdir_list, file_list in os.walk(self.shr_dir, True):
            if len(file_list) == 0:
                continue

            subdir_list[:] = []

            symlink_path = os.path.join(self.git_work_tree, os.path.relpath(dir_path, self.shr_dir))
            if os.path.exists(symlink_path):
                os.remove(symlink_path)

            symlink_path = os.path.join(self.bck_dir, os.path.relpath(dir_path, self.shr_dir))
            if os.path.exists(symlink_path):
                os.remove(symlink_path)

        self.__print('Unlinked.')
        return True 
Example 14
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: util.py    MIT License 6 votes vote down vote up
def zip_dir(directory):
    """zip a directory tree into a BytesIO object"""
    result = io.BytesIO()
    dlen = len(directory)
    with ZipFile(result, "w") as zf:
        for root, dirs, files in os.walk(directory):
            for name in files:
                full = os.path.join(root, name)
                rel = root[dlen:]
                dest = os.path.join(rel, name)
                zf.write(full, dest)
    return result

#
# Simple progress bar
# 
Example 15
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: locators.py    MIT License 6 votes vote down vote up
def get_distribution_names(self):
        """
        Return all the distribution names known to this locator.
        """
        result = set()
        for root, dirs, files in os.walk(self.base_dir):
            for fn in files:
                if self.should_include(fn, root):
                    fn = os.path.join(root, fn)
                    url = urlunparse(('file', '',
                                      pathname2url(os.path.abspath(fn)),
                                      '', '', ''))
                    info = self.convert_url_to_download_info(url, None)
                    if info:
                        result.add(info['name'])
            if not self.recursive:
                break
        return result 
Example 16
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: file.py    MIT License 6 votes vote down vote up
def recursive_set_attributes(module, b_path, follow, file_args):
    changed = False
    for b_root, b_dirs, b_files in os.walk(b_path):
        for b_fsobj in b_dirs + b_files:
            b_fsname = os.path.join(b_root, b_fsobj)
            if not os.path.islink(b_fsname):
                tmp_file_args = file_args.copy()
                tmp_file_args['path'] = to_native(b_fsname, errors='surrogate_or_strict')
                changed |= module.set_fs_attributes_if_different(tmp_file_args, changed, expand=False)
            else:
                tmp_file_args = file_args.copy()
                tmp_file_args['path'] = to_native(b_fsname, errors='surrogate_or_strict')
                changed |= module.set_fs_attributes_if_different(tmp_file_args, changed, expand=False)
                if follow:
                    b_fsname = os.path.join(b_root, os.readlink(b_fsname))
                    if os.path.isdir(b_fsname):
                        changed |= recursive_set_attributes(module, b_fsname, follow, file_args)
                    tmp_file_args = file_args.copy()
                    tmp_file_args['path'] = to_native(b_fsname, errors='surrogate_or_strict')
                    changed |= module.set_fs_attributes_if_different(tmp_file_args, changed, expand=False)
    return changed 
Example 17
Project: interact   Author: dongshengmu   File: util.py    MIT License 6 votes vote down vote up
def find_files(pattern, path=os.curdir, ignore_dirs=None, sort_key=None):
    """Recursively find all files matching supplied filename pattern in the specified path.

    - pattern: filename matching shell pattern as in 'ls' command, eg. '\*.txt'
    - path: a pathname, can be relative or absolute pathname.
    - ignore_dirs: a list of name of dirs under the specified path, to be skipped.
    """
    if ignore_dirs is None:
        ignore_dirs = []
    filenames = []
    for pathname, subdirs, files in os.walk(os.path.abspath(path)):
        # change the mutable subdirs, so os.walk won't walk in the ignored dir.
        for x in ignore_dirs:
            if x in subdirs:
                subdirs.remove(x)
        subdirs.sort()
        filenames += sorted([os.path.join(pathname, filename)
                             for filename in fnmatch.filter(files, pattern)],
                            key=sort_key)
    return filenames 
Example 18
Project: interact   Author: dongshengmu   File: util.py    MIT License 6 votes vote down vote up
def find_dirs(pattern, path=os.curdir, ignore_dirs=None, sort_key=None):
    """Recursively find all sub directories matching supplied filename pattern in the specified path.

    - pattern: directory name matching shell pattern as in 'ls' command
    - path: a pathname, can be relative or absolute pathname.
    - ignore_dirs: a list of name of dirs under the specified path, to be skipped.
    """
    if ignore_dirs is None:
        ignore_dirs = []
    dir_names = []
    for pathname, subdirs, files in os.walk(os.path.abspath(path)):
        # change the mutable subdirs, so os.walk won't walk in the ignored dir.
        for x in ignore_dirs:
            if x in subdirs:
                subdirs.remove(x)
        dir_names += sorted([os.path.join(pathname, filename)
                             for filename in fnmatch.filter(subdirs, pattern)],
                            key=sort_key)
    return dir_names 
Example 19
Project: multi-embedding-cws   Author: wangjksjtu   File: pre_slicing.py    MIT License 5 votes vote down vote up
def All2oneFile(rootDir, out_filename):
	with open(out_filename, "w") as opt:
		for dirName, subdirList, fileList in os.walk(rootDir):
			curDir = os.path.join(dirName)
			for file in fileList:
				line_raw_list = []
				line_list = []
				if file.endswith(".txt"):
					curFile = os.path.join(curDir, file)
					inp = open(curFile, 'r')
					for line in inp:
						ustr = line.decode("utf-8")
						ustr = ustr.strip()
						opt.write(ustr.encode("utf-8") + '\n')

# NE_Extracting("gold_raw.txt")

# with open("gold_no_NE.txt", "w") as opt:
#	NE_Removing("gold_raw.txt", opt)

# CleanSentence("gold_no_NE.txt", "gold_test.txt") 
Example 20
Project: leapp-repository   Author: oamg   File: install_actor_deps.py    Apache License 2.0 5 votes vote down vote up
def install_actor_deps(actor, directory):
    for root, dirs, files in os.walk(directory):
        if actor in dirs:
            makefile_path = os.path.join(root, actor, 'Makefile')
            if os.path.isfile(makefile_path):
                install(makefile_path)
            else:
                sys.stderr.write("Actor '{}' doesn't have Makefile!\n".format(actor))
            return
    error("Actor '{}' doesn't exist!\n".format(actor), 1) 
Example 21
Project: leapp-repository   Author: oamg   File: install_actor_deps.py    Apache License 2.0 5 votes vote down vote up
def install_all_deps(directory):
    for root, dirs, files in os.walk(directory):
        if 'Makefile' in files:
            install(os.path.join(root, 'Makefile')) 
Example 22
Project: incubator-spot   Author: apache   File: utils.py    Apache License 2.0 5 votes vote down vote up
def validate_data_source(cls,pipeline_type):
        dirs = os.walk("{0}/pipelines/".format(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))).next()[1]
        is_type_ok = True if pipeline_type in dirs else False
        return is_type_ok


#class NewFileEvent(FileSystemEventHandler):
#
#    pipeline_instance = None
#    def __init__(self,pipeline_instance):
#        self.pipeline_instance = pipeline_instance
#
#    def on_moved(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.dest_path)
#
#    def on_created(self,event):
#        if not event.is_directory:
#            self.pipeline_instance.new_file_detected(event.src_path) 
Example 23
Project: ubittool   Author: carlosperate   File: make.py    MIT License 5 votes vote down vote up
def _rm_folder_named(scan_path, folder_name):
    """Remove all folders named folder_name from the given directory tree.

    :param scan_path: Directory to scan for folders with specific name.
    """
    for root, dirs, files in os.walk(scan_path, topdown=False):
        for name in dirs:
            if name == folder_name:
                _rm_dir(os.path.join(root, name)) 
Example 24
Project: ubittool   Author: carlosperate   File: make.py    MIT License 5 votes vote down vote up
def _rm_file_extension(scan_path, file_extension):
    """Remove all files with an specific extension from a given directory.

    :param scan_path: Directory to scan for file removal.
    :param file_extension: File extension of the files to remove
    """
    for root, dirs, files in os.walk(scan_path, topdown=False):
        for file_ in files:
            if file_.endswith(".{}".format(file_extension)):
                file_path = os.path.join(root, file_)
                _rm_file(file_path) 
Example 25
Project: malcode   Author: moonsea   File: copyasm.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path, despath):
    for parent, dirnames, filenames in os.walk(path):

        for filename in filenames:
            filepath = os.path.join(parent, filename)
            filetype = filename.split('.')[-1]
            if (filetype == 'asm'):
                CopyFile(filepath, despath, filename) 
Example 26
Project: malcode   Author: moonsea   File: wingenasm.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path, initClean=False):
    for parent, dirnames, filenames in os.walk(path):

        if(initClean):
            log('Cleaning', '', '[-]')

            for filename in filenames:
                filepath = os.path.join(parent, filename)

                cleanFile(filename, filepath)

            continue

        log('Entering', parent)
        # normal file
        # log('origin', str(len(filenames)))
        # unpack file
        log('origin', str(countFile(parent, 'dump')))

        total = 0
        for filename in filenames:
            filepath = os.path.join(parent, filename)

            if (cleanFile(filename, filepath)):
                continue

            log('asming', filename)

            total = genAsm(filepath, total)

        log('genasm', str(countFile(parent, 'asm'))) 
Example 27
Project: malcode   Author: moonsea   File: getOpCodeNdisasm.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path, type):
    for parent, dirnames, filenames in os.walk(path):
        log('Entering', parent, subpath='ndisasm')

        for filename in filenames:
            filepath = os.path.join(parent, filename)
            print filepath

            with open(filepath) as asmfile:
                lines = asmfile.readlines()

            log('OpCoding', filename, subpath='ndisasm')
            opfiledir = os.path.join(BASEPATH, 'opcode', type)
            getOpCode(lines, filename, opfiledir) 
Example 28
Project: malcode   Author: moonsea   File: gengram.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path):
    for parent, dirnames, filenames in os.walk(path):
        log('Entering', parent, subpath='opcode')

        for filename in filenames:

            filepath = os.path.join(parent, filename)
            print filepath

            with open(filepath) as asmfile:
                lines = asmfile.readlines()
            # print lines
            log('Generating', filename, subpath='opcode')
            genGram(lines, filename) 
Example 29
Project: malcode   Author: moonsea   File: dosgenasm.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path, initClean=False):
    for parent, dirnames, filenames in os.walk(path):

        if(initClean):
            log('Cleaning', '', '[-]')

            for filename in filenames:
                filepath = os.path.join(parent, filename)

                cleanFile(filename, filepath)

            continue

        log('Entering', parent)
        # normal file
        # log('origin', str(len(filenames)))
        # unpack file
        log('origin', str(countFile(parent, 'dump')))

        total = 0
        for filename in filenames:
            filepath = os.path.join(parent, filename)

            if (cleanFile(filename, filepath)):
                continue

            log('asming', filename)

			# windows platform
            filepath = filepath.replace('\\', '\\\\')

            total = genAsm(filepath, total)

        log('genasm', str(countFile(parent, 'asm'))) 
Example 30
Project: malcode   Author: moonsea   File: gramfreq.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path):
    totaltf = dict()
    totaldf = dict()
    totalterm = 0
    maxterm = 0
    totaldocument = 0
    maxdocument = 0

    for parent, dirnames, filenames in os.walk(path):
        log('Entering', parent, subpath='classfier')

        totaldocument += len(filenames)
        for filename in filenames:

            filepath = os.path.join(parent, filename)
            print filepath

            with open(filepath) as asmfile:
                lines = asmfile.readlines()

            log('Generating', filename, subpath='classfier')
            genSingleTF(lines, filename)
            # totalterm += len(lines)
            getTotalTF(lines, totaltf, totaldf)

    # print totaltf
    desfilepath = os.path.join(BASEPATH, '2-gram-totaltf')
    maxterm = max(totaltf.values())
    maxdocument = max(totaldf.values())
    totalterm = len(totaltf)
    with open(desfilepath, 'w') as desfile:
        for key in totaltf.keys():
            # print key, totaltf[key]
            tmp = '----'.join([key, str(totaltf[key]), str(totalterm), str(totaltf[key] / maxterm), str(
                totaldf.get(key, 0)), str(totaldocument), str(totaldf.get(key, 0) / maxdocument), str(math.log(totaldocument / totaldf.get(key, 1)))])
            desfile.write(tmp + '\n') 
Example 31
Project: malcode   Author: moonsea   File: benignasm.py    GNU General Public License v3.0 5 votes vote down vote up
def traveseFile(path, initClean=False):
    for parent, dirnames, filenames in os.walk(path):

        if(initClean):
            log('Cleaning', '', '[-]')

            for filename in filenames:
                filepath = os.path.join(parent, filename)

                cleanFile(filename, filepath)

            continue

        log('Entering', parent)
        # normal file
        log('origin', str(len(filenames)))
        # unpack file
        # log('origin', str(countFile(parent, 'dump')))

        total = 0
        for filename in filenames:
            filepath = os.path.join(parent, filename)

            if (cleanFile(filename, filepath)):
                continue

            log('asming', filename)

            total = genAsm(filepath, total)

        log('genasm', str(countFile(parent, 'asm'))) 
Example 32
Project: cat-bbs   Author: aleju   File: create_dataset.py    MIT License 5 votes vote down vote up
def find_image_filepaths(dataset_dir):
    """Load image filepaths from the 10k cats dataset."""
    result = []
    for root, dirs, files in os.walk(dataset_dir):
        if "/CAT_" in root:
            for name in files:
                fp = os.path.join(root, name)
                if name.endswith(".jpg") and os.path.isfile("%s.cat" % (fp,)):
                    result.append(fp)
    return result 
Example 33
Project: pyblish-win   Author: pyblish   File: findnocoding.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def walk_python_files(self, paths, *args, **kwargs):
            for path in paths:
                if os.path.isfile(path):
                    yield path.endswith(".py")
                elif os.path.isdir(path):
                    for root, dirs, files in os.walk(path):
                        for filename in files:
                            if filename.endswith(".py"):
                                yield os.path.join(root, filename) 
Example 34
Project: pyblish-win   Author: pyblish   File: pysource.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def walk_python_files(paths, is_python=looks_like_python, exclude_dirs=None):
    """\
    Recursively yield all Python source files below the given paths.

    paths: a list of files and/or directories to be checked.
    is_python: a function that takes a file name and checks whether it is a
               Python source file
    exclude_dirs: a list of directory base names that should be excluded in
                  the search
    """
    if exclude_dirs is None:
        exclude_dirs=[]

    for path in paths:
        print_debug("testing: %s" % path)
        if os.path.isfile(path):
            if is_python(path):
                yield path
        elif os.path.isdir(path):
            print_debug("    it is a directory")
            for dirpath, dirnames, filenames in os.walk(path):
                for exclude in exclude_dirs:
                    if exclude in dirnames:
                        dirnames.remove(exclude)
                for filename in filenames:
                    fullpath = os.path.join(dirpath, filename)
                    print_debug("testing: %s" % fullpath)
                    if is_python(fullpath):
                        yield fullpath
        else:
            print_debug("    unknown type") 
Example 35
Project: pyblish-win   Author: pyblish   File: support.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def all_project_files():
    for dirpath, dirnames, filenames in os.walk(proj_dir):
        for filename in filenames:
            if filename.endswith(".py"):
                yield os.path.join(dirpath, filename) 
Example 36
Project: pyblish-win   Author: pyblish   File: runtktests.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_tests_modules(basepath=this_dir_path, gui=True, packages=None):
    """This will import and yield modules whose names start with test_
    and are inside packages found in the path starting at basepath.

    If packages is specified it should contain package names that want
    their tests collected.
    """
    py_ext = '.py'

    for dirpath, dirnames, filenames in os.walk(basepath):
        for dirname in list(dirnames):
            if dirname[0] == '.':
                dirnames.remove(dirname)

        if is_package(dirpath) and filenames:
            pkg_name = dirpath[len(basepath) + len(os.sep):].replace('/', '.')
            if packages and pkg_name not in packages:
                continue

            filenames = filter(
                    lambda x: x.startswith('test_') and x.endswith(py_ext),
                    filenames)

            for name in filenames:
                try:
                    yield importlib.import_module(
                            ".%s" % name[:-len(py_ext)], pkg_name)
                except test.test_support.ResourceDenied:
                    if gui:
                        raise 
Example 37
Project: pyblish-win   Author: pyblish   File: test_repr.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        actions = []
        for dirpath, dirnames, filenames in os.walk(self.pkgname):
            for name in dirnames + filenames:
                actions.append(os.path.join(dirpath, name))
        actions.append(self.pkgname)
        actions.sort()
        actions.reverse()
        for p in actions:
            if os.path.isdir(p):
                os.rmdir(p)
            else:
                os.remove(p)
        del sys.path[0] 
Example 38
Project: pyblish-win   Author: pyblish   File: test_os.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        # Tear everything down.  This is a decent use for bottom-up on
        # Windows, which doesn't have a recursive delete command.  The
        # (not so) subtlety is that rmdir will fail unless the dir's
        # kids are removed first, so bottom up is essential.
        for root, dirs, files in os.walk(test_support.TESTFN, topdown=False):
            for name in files:
                os.remove(os.path.join(root, name))
            for name in dirs:
                dirname = os.path.join(root, name)
                if not os.path.islink(dirname):
                    os.rmdir(dirname)
                else:
                    os.remove(dirname)
        os.rmdir(test_support.TESTFN) 
Example 39
Project: robust-atd   Author: shadowbq   File: __init__.py    MIT License 5 votes vote down vote up
def get_filepaths(self, directory):

        file_paths = []

        for root, directories, files in os.walk(directory):
            files = [f for f in files if not f[0] == '.']
            directories[:] = [d for d in directories if not d[0] == '.']
            for filename in files:
                filepath = os.path.join(root, filename)
                file_paths.append(filepath)

        return file_paths 
Example 40
Project: robust-atd   Author: shadowbq   File: __init__.py    MIT License 5 votes vote down vote up
def get_filepaths(self, directory):

        file_paths = []

        for root, directories, files in os.walk(directory):
            for filename in files:
                if filename.endswith('json'):
                    filepath = os.path.join(root, filename)
                    file_paths.append(filepath)

        return file_paths 
Example 41
Project: plantuml-parser   Author: pjcuadra   File: test_data.py    Apache License 2.0 5 votes vote down vote up
def test_class():

    parser = get_parser()

    diagrams_path = os.path.join(test_data_path, 'class_diagram')

    for root, dirs, files in os.walk(diagrams_path):
        for filename in files:
            f = open(os.path.join(diagrams_path, filename))
            parser.parse(f.read()) 
Example 42
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in os.walk(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 43
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: loaders.py    Apache License 2.0 5 votes vote down vote up
def list_templates(self):
        found = set()
        for searchpath in self.searchpath:
            for dirpath, dirnames, filenames in os.walk(searchpath):
                for filename in filenames:
                    template = os.path.join(dirpath, filename) \
                        [len(searchpath):].strip(os.path.sep) \
                                          .replace(os.path.sep, '/')
                    if template[:2] == './':
                        template = template[2:]
                    if template not in found:
                        found.add(template)
        return sorted(found) 
Example 44
Project: CFN-CR-PythonLambdaLayer   Author: kisst   File: lambda_function.py    GNU General Public License v3.0 5 votes vote down vote up
def zipit(src, dst):
    """
    Create a zip file from src into dst.zip
    """
    zipf = zipfile.ZipFile("%s.zip" % (dst), "w", zipfile.ZIP_DEFLATED)
    abs_src = os.path.abspath(src)
    for dirname, _, files in os.walk(src):
        for filename in files:
            absname = os.path.abspath(os.path.join(dirname, filename))
            arcname = absname[len(abs_src) + 1:]
            zipf.write(absname, arcname)
    zipf.close() 
Example 45
Project: wikilinks   Author: trovdimi   File: pickle_data.py    MIT License 5 votes vote down vote up
def merge_semsim():
    merge = {}
    for dirname, dirnames, filenames in os.walk("/home/psinger/WikiLinks/data/sem_sim"):
        for file_name in filenames:
            if file_name.endswith(".p"):
                sem_sim = pickle.load( open( "/home/psinger/WikiLinks/data/sem_sim/"+file_name, "rb" ) )
                merge = merge_two_dicts(merge, sem_sim)
                print len(merge)
    pickle.dump(merge, open("/ssd/ddimitrov/pickle/sem_sim", "wb"), protocol=pickle.HIGHEST_PROTOCOL)
    print "semsim" 
Example 46
Project: wikilinks   Author: trovdimi   File: linkpostioninserter.py    MIT License 5 votes vote down vote up
def manageWork(self):
        for root, dirs, files in os.walk(STATIC_HTML_DUMP_ARTICLES_DIR+self.path):
            for i, file_name in enumerate(files):
                if file_name.endswith(".zip"):
                    self.parse_article(file_name,root) 
Example 47
Project: wikilinks   Author: trovdimi   File: htmlstripper.py    MIT License 5 votes vote down vote up
def manageWork(self, nProcess):
        pool = multiprocessing.Pool(processes=nProcess)
        for dirname, dirnames, filenames in os.walk(STATIC_HTML_DUMP_ARTICLES_DIR):
            for subdirname in dirnames:
                for root, dirs, files in os.walk(os.path.join(STATIC_HTML_DUMP_ARTICLES_DIR, subdirname)):
                    for file_name in files:
                        if file_name.endswith(".zip"):
                            pool.apply_async(self.html_stripper, args=(file_name, root, subdirname,))
        pool.close()
        pool.join() 
Example 48
Project: AutoDL   Author: tanguofu   File: sync.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_files(rootDir, suffix, lastupdate=0):
    result = []
    #pwd = os.getcwd()
    for root, dirs, files in os.walk(rootDir):
         result += [root+"/"+f for f in files if f.split('.')[-1] in suffix]
         
    result = filter(lambda x : os.stat('.').st_mtime > lastupdate, result)
    
    return result 
Example 49
Project: thyme-tools   Author: jtourille   File: anafora.py    MIT License 5 votes vote down vote up
def brat_to_anafora(input_brat_dir: str = None,
                    output_anafora_dir: str = None) -> None:
    """
    Convert a THYME corpus part from brat to anafora

    Args:
        input_brat_dir (str): annotation path (brat format)
        output_anafora_dir (str): output path where anafora files will be created

    Returns:
        None
    """

    for root, dirs, files in os.walk(os.path.abspath(input_brat_dir)):
        for filename in files:
            if re.match("^.*\.ann$", filename):

                document_id = filename.split(".")[0]

                # Fetching entities and relations from files and converting to anafora format
                source_ann_file = os.path.join(root, filename)
                entities, relations = parse_ann_file(source_ann_file)
                ana_entities, ana_relations = convert_brat_payload_to_anafora_payload(entities, relations, document_id)

                # Building target directory
                target_dir = os.path.join(os.path.abspath(output_anafora_dir), document_id)
                ensure_dir(target_dir)

                # Creating xml payload
                target_file = os.path.join(target_dir, "{}.Temporal-Relation.system.completed.xml".format(document_id))
                xml_payload = generate_payload(ana_entities, ana_relations, document_id)

                # Writing payload to disk
                tree = etree.ElementTree(xml_payload)
                tree.write(target_file, pretty_print=True, xml_declaration=True, encoding='UTF-8') 
Example 50
Project: DBC-FederatedLearning-Client-VNX   Author: DeepBrainChain   File: transfer.py    Apache License 2.0 5 votes vote down vote up
def retrieve_file_paths(dirName):
 
    # setup file paths variable
    filePaths = []
   
    # Read all directory, subdirectories and file lists
    for root, directories, files in os.walk(dirName):
        for filename in files:
            # Create the full filepath by using os module.
            filePath = os.path.join(root, filename)
            filePaths.append(filePath)
         
    # return all paths
    return filePaths