Python shutil.rmtree() Examples

The following are code examples for showing how to use shutil.rmtree(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: leapp-repository   Author: oamg   File: initramgen.py    Apache License 2.0 7 votes vote down vote up
def copy_dracut_modules(context, modules):
    """
    Copies our dracut modules into the target userspace.
    """
    try:
        shutil.rmtree(context.full_path('/dracut'))
    except EnvironmentError:
        pass
    for module in modules:
        try:
            context.copytree_to(module.module_path, os.path.join('/dracut', os.path.basename(module.module_path)))
        except shutil.Error as e:
            api.current_logger().error('Failed to copy dracut module "{name}" from "{source}" to "{target}"'.format(
                name=module.name, source=module.module_path, target=context.full_path('/dracut')), exc_info=True)
            raise StopActorExecutionError(
                message='Failed to install dracut modules required in the initram. Error: {}'.format(str(e))
            ) 
Example 2
Project: fs_image   Author: facebookincubator   File: temp_repos.py    MIT License 6 votes vote down vote up
def temp_repos_steps(base_dir=None, arch: str = 'x86_64', *args, **kwargs):
    '''
    Given a history of changes to a set of RPM repos (as in `SAMPLE_STEPS`),
    generates a collection of RPM repos on disk by running:
      - `rpmbuild` to build the RPM files
      - `createrepo` to build the repo metadata

    Returns a temporary path, cleaned up once the context exits, containing
    a directory per time step (named 0, 1, 2, etc).  Each timestep directory
    contains a directory per repo, and each repo has this layout:
        repodata/{repomd.xml,other-repodata.{xml,sqlite}.bz2}
        reponame-pkgs/rpm-test-<name>-<version>-<release>.<arch>.rpm
    '''
    td = Path(tempfile.mkdtemp(dir=base_dir))
    try:
        make_repo_steps(out_dir=td, arch=arch, *args, **kwargs)
        yield td
    except BaseException:  # Clean up even on Ctrl-C
        shutil.rmtree(td)
        raise 
Example 3
Project: leapp-repository   Author: oamg   File: test_lib_backup.py    Apache License 2.0 6 votes vote down vote up
def test_backup_file():
    tmpdir = tempfile.mkdtemp()
    try:
        file_path = os.path.join(tmpdir, 'foo-bar')
        content = 'test content\n'
        with open(file_path, 'w') as f:
            f.write(content)

        backup_path = backup_file(file_path)

        assert os.path.basename(backup_path) == 'foo-bar.leapp-backup'
        assert os.path.dirname(backup_path) == tmpdir
        assert len(os.listdir(tmpdir)) == 2
        st = os.stat(backup_path)
        assert stat.S_IMODE(st.st_mode) == (stat.S_IRUSR | stat.S_IWUSR)
        with open(backup_path, 'r') as f:
            backup_content = f.read()
        assert backup_content == content
        with open(file_path, 'r') as f:
            orig_content = f.read()
        assert orig_content == content
    finally:
        shutil.rmtree(tmpdir) 
Example 4
Project: leapp-repository   Author: oamg   File: test_lib_backup.py    Apache License 2.0 6 votes vote down vote up
def test_backup_file_target_exists():
    tmpdir = tempfile.mkdtemp()
    try:
        file_path = os.path.join(tmpdir, 'foo-bar')
        primary_target_path = '%s.leapp-backup' % file_path
        primary_target_content = 'do not overwrite me'
        content = 'test_content\n'
        with open(file_path, 'w') as f:
            f.write(content)
        with open(primary_target_path, 'w') as f:
            f.write(primary_target_content)

        backup_path = backup_file(file_path)

        assert os.path.basename(backup_path).startswith('foo-bar.leapp-backup.')
        assert os.path.dirname(backup_path) == tmpdir
        assert len(os.listdir(tmpdir)) == 3
        st = os.stat(backup_path)
        assert stat.S_IMODE(st.st_mode) == (stat.S_IRUSR | stat.S_IWUSR)
        with open(backup_path, 'r') as f:
            assert f.read() == content
        with open(primary_target_path, 'r') as f:
            assert f.read() == primary_target_content
    finally:
        shutil.rmtree(tmpdir) 
Example 5
Project: alfred-yubikey-otp   Author: robertoriv   File: workflow.py    MIT License 6 votes vote down vote up
def _delete_directory_contents(self, dirpath, filter_func):
        """Delete all files in a directory.

        :param dirpath: path to directory to clear
        :type dirpath: ``unicode`` or ``str``
        :param filter_func function to determine whether a file shall be
            deleted or not.
        :type filter_func ``callable``

        """
        if os.path.exists(dirpath):
            for filename in os.listdir(dirpath):
                if not filter_func(filename):
                    continue
                path = os.path.join(dirpath, filename)
                if os.path.isdir(path):
                    shutil.rmtree(path)
                else:
                    os.unlink(path)
                self.logger.debug('deleted : %r', path) 
Example 6
Project: pyblish-win   Author: pyblish   File: test_filecmp.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def setUp(self):
        tmpdir = tempfile.gettempdir()
        self.dir = os.path.join(tmpdir, 'dir')
        self.dir_same = os.path.join(tmpdir, 'dir-same')
        self.dir_diff = os.path.join(tmpdir, 'dir-diff')
        self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a')
        data = 'Contents of file go here.\n'
        for dir in [self.dir, self.dir_same, self.dir_diff]:
            shutil.rmtree(dir, True)
            os.mkdir(dir)
            if self.caseinsensitive and dir is self.dir_same:
                fn = 'FiLe'     # Verify case-insensitive comparison
            else:
                fn = 'file'
            output = open(os.path.join(dir, fn), 'w')
            output.write(data)
            output.close()

        output = open(os.path.join(self.dir_diff, 'file2'), 'w')
        output.write('An extra file.\n')
        output.close() 
Example 7
Project: pyblish-win   Author: pyblish   File: test_import.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def setUp(self):
        if os.path.exists(self.tagged):
            shutil.rmtree(self.tagged)
        if os.path.exists(self.package_name):
            symlink_support.remove_symlink(self.package_name)
        self.orig_sys_path = sys.path[:]

        # create a sample package; imagine you have a package with a tag and
        #  you want to symbolically link it from its untagged name.
        os.mkdir(self.tagged)
        init_file = os.path.join(self.tagged, '__init__.py')
        open(init_file, 'w').close()
        assert os.path.exists(init_file)

        # now create a symlink to the tagged package
        # sample -> sample-tagged
        symlink_support.symlink(self.tagged, self.package_name)

        assert os.path.isdir(self.package_name)
        assert os.path.isfile(os.path.join(self.package_name, '__init__.py')) 
Example 8
Project: pyblish-win   Author: pyblish   File: test_tarfile.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_exclude(self):
        tempdir = os.path.join(TEMPDIR, "exclude")
        os.mkdir(tempdir)
        try:
            for name in ("foo", "bar", "baz"):
                name = os.path.join(tempdir, name)
                open(name, "wb").close()

            exclude = os.path.isfile

            tar = tarfile.open(tmpname, self.mode, encoding="iso8859-1")
            with test_support.check_warnings(("use the filter argument",
                                              DeprecationWarning)):
                tar.add(tempdir, arcname="empty_dir", exclude=exclude)
            tar.close()

            tar = tarfile.open(tmpname, "r")
            self.assertEqual(len(tar.getmembers()), 1)
            self.assertEqual(tar.getnames()[0], "empty_dir")
        finally:
            shutil.rmtree(tempdir) 
Example 9
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_on_error(self):
        self.errorState = 0
        os.mkdir(TESTFN)
        self.childpath = os.path.join(TESTFN, 'a')
        f = open(self.childpath, 'w')
        f.close()
        old_dir_mode = os.stat(TESTFN).st_mode
        old_child_mode = os.stat(self.childpath).st_mode
        # Make unwritable.
        os.chmod(self.childpath, stat.S_IREAD)
        os.chmod(TESTFN, stat.S_IREAD)

        shutil.rmtree(TESTFN, onerror=self.check_args_to_onerror)
        # Test whether onerror has actually been called.
        self.assertEqual(self.errorState, 2,
                            "Expected call to onerror function did not happen.")

        # Make writable again.
        os.chmod(TESTFN, old_dir_mode)
        os.chmod(self.childpath, old_child_mode)

        # Clean up.
        shutil.rmtree(TESTFN) 
Example 10
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def check_args_to_onerror(self, func, arg, exc):
        # test_rmtree_errors deliberately runs rmtree
        # on a directory that is chmod 400, which will fail.
        # This function is run when shutil.rmtree fails.
        # 99.9% of the time it initially fails to remove
        # a file in the directory, so the first time through
        # func is os.remove.
        # However, some Linux machines running ZFS on
        # FUSE experienced a failure earlier in the process
        # at os.listdir.  The first failure may legally
        # be either.
        if self.errorState == 0:
            if func is os.remove:
                self.assertEqual(arg, self.childpath)
            else:
                self.assertIs(func, os.listdir,
                              "func must be either os.remove or os.listdir")
                self.assertEqual(arg, TESTFN)
            self.assertTrue(issubclass(exc[0], OSError))
            self.errorState = 1
        else:
            self.assertEqual(func, os.rmdir)
            self.assertEqual(arg, TESTFN)
            self.assertTrue(issubclass(exc[0], OSError))
            self.errorState = 2 
Example 11
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_copytree_named_pipe(self):
        os.mkdir(TESTFN)
        try:
            subdir = os.path.join(TESTFN, "subdir")
            os.mkdir(subdir)
            pipe = os.path.join(subdir, "mypipe")
            os.mkfifo(pipe)
            try:
                shutil.copytree(TESTFN, TESTFN2)
            except shutil.Error as e:
                errors = e.args[0]
                self.assertEqual(len(errors), 1)
                src, dst, error_msg = errors[0]
                self.assertEqual("`%s` is a named pipe" % pipe, error_msg)
            else:
                self.fail("shutil.Error should have been raised")
        finally:
            shutil.rmtree(TESTFN, ignore_errors=True)
            shutil.rmtree(TESTFN2, ignore_errors=True) 
Example 12
Project: fs_image   Author: facebookincubator   File: test_volume_for_repo.py    MIT License 5 votes vote down vote up
def test_volume_repo(self):
        artifacts_dir = tempfile.mkdtemp(prefix='test_volume_repo')
        volume_dir = os.path.join(artifacts_dir, vfr.VOLUME_DIR)
        image_file = os.path.join(artifacts_dir, vfr.IMAGE_FILE)
        min_free_bytes = 250e6

        try:
            self.assertEqual(
                vfr.get_volume_for_current_repo(
                    min_free_bytes=min_free_bytes,
                    artifacts_dir=artifacts_dir,
                ),
                volume_dir,
            )
            self.assertGreaterEqual(
                os.stat(image_file).st_size, min_free_bytes,
            )

            fstype_and_avail = subprocess.check_output([
                'findmnt', '--noheadings', '--output', 'FSTYPE,AVAIL',
                '--bytes', volume_dir
            ])
            fstype, avail = fstype_and_avail.strip().split()
            self.assertEqual(b'btrfs', fstype)
            self.assertGreaterEqual(int(avail), min_free_bytes)
        finally:
            try:
                subprocess.call(['sudo', 'umount', volume_dir])
            except Exception:
                pass  # Might not have been mounted in case of an earlier error
            shutil.rmtree(artifacts_dir) 
Example 13
Project: leapp-repository   Author: oamg   File: initramgen.py    Apache License 2.0 5 votes vote down vote up
def install_multipath_files(context):
    # Include multipath related files (according to module-setup of multipath)
    if os.path.exists('/etc/xdrdevices.conf'):
        context.copy_to('/etc/xdrdevices.conf', '/etc/xdrdevices.conf')
    if os.path.exists('/etc/multipath.conf'):
        context.copy_to('/etc/multipath.conf', '/etc/multipath.conf')
        if os.path.isdir('/etc/multipath'):
            shutil.rmtree(context.full_path('/etc/multipath'))
            context.copytree_to('/etc/multipath', '/etc/multipath') 
Example 14
Project: leapp-repository   Author: oamg   File: actor.py    Apache License 2.0 5 votes vote down vote up
def process(self):
        # FIXME: we hitting issue now because the network is down and rhsm
        # # is trying to connect to the server. Commenting this out for now
        # # so people will not be affected in case they do not have set a
        # # release and we will have time to fix it properly.
        # Make sure Subscription Manager OS Release is unset
        # cmd = ['subscription-manager', 'release', '--unset']
        # run(cmd)

        src_rhsm_info = next(self.consume(SourceRHSMInfo), None)
        if src_rhsm_info:
            for prod_cert in src_rhsm_info.existing_product_certificates:
                run(['rm', '-f', prod_cert])

        used_repos = self.consume(UsedTargetRepositories)
        storage_info = next(self.consume(StorageInfo), None)
        tasks = next(self.consume(FilteredRpmTransactionTasks), FilteredRpmTransactionTasks())
        target_userspace_info = next(self.consume(TargetUserSpaceInfo), None)

        dnfplugin.perform_transaction_install(tasks=tasks, used_repos=used_repos, storage_info=storage_info,
                                              target_userspace_info=target_userspace_info)
        self.produce(TransactionCompleted())
        userspace = next(self.consume(TargetUserSpaceInfo), None)
        if userspace:
            try:
                shutil.rmtree(userspace.path)
            except EnvironmentError:
                self.log.info("Failed to remove temporary userspace - error ignored", exc_info=True) 
Example 15
Project: incubator-spot   Author: apache   File: collector.py    Apache License 2.0 5 votes vote down vote up
def __del__(self):
        '''
            Called when the instance is about to be destroyed.
        '''
        if hasattr(self, '_tmpdir'):
            self._logger.info('Clean up temporary directory "{0}".'.format(self._tmpdir))
            shutil.rmtree(self._tmpdir) 
Example 16
Project: ubittool   Author: carlosperate   File: make.py    MIT License 5 votes vote down vote up
def _rm_dir(dir_to_remove):
    """:param dir_to_remove: Directory to remove."""
    if os.path.isdir(dir_to_remove):
        print("Removing directory: {}".format(dir_to_remove))
        shutil.rmtree(dir_to_remove)
    else:
        print("Directory {} was not found.".format(dir_to_remove)) 
Example 17
Project: gog-galaxy-plugin-downloader   Author: Slashbunny   File: download.py    GNU General Public License v3.0 5 votes vote down vote up
def fix_plugin_directories(dest):
    """
    Loops through all folders in the output directory, reads the their manifest
    file, and renames the directory to the standard <platform>_<guid> format
    """
    # Loop through directories in the destination directory
    for existing_dir in os.listdir(dest):
        existing_path = os.path.join(dest, existing_dir)

        # Skip non-directories
        if not os.path.isdir(existing_path):
            continue

        try:
            with open(os.path.join(existing_path, 'manifest.json')) as m:
                data = json.load(m)
                platform = data['platform']
                guid = data['guid']

                # Close json file
                m.close()

                expected_dir = platform + '_' + guid
                expected_path = os.path.join(dest, expected_dir)

                if existing_path != expected_path:
                    print('NOTICE: Folder should be "{}", but it is named "{}"'
                          .format(expected_dir, existing_dir))

                    if os.path.isdir(expected_path):
                        print('NOTICE: Correct pathed plugin already exists,'
                              + ' deleting extra plugin')
                        shutil.rmtree(existing_path)
                    else:
                        print('NOTICE: Renaming folder to proper name')
                        shutil.move(existing_path, expected_path)
        except (FileNotFoundError, json.decoder.JSONDecodeError, KeyError):
            print('ERROR: Could not read plugin data from {} folder'
                  .format(existing_path)) 
Example 18
Project: gog-galaxy-plugin-downloader   Author: Slashbunny   File: download.py    GNU General Public License v3.0 5 votes vote down vote up
def delete_old_plugins(data, dest):
    """
    Deletes versions of plugins that don't match the yaml manifest. In theory
    this should only be older versions, but any version that doesn't match
    the yaml definition will be deleted

    This explicitly does not touch other directories that do not match the
    known plugin names.

    If the version doesn't match the yaml definition, the directory is removed
    """
    # Loop over each plugin
    for name, data in data.items():
        expected_plugin_dir = name + '_' + data['guid']

        # Loop through directories in the destination directory
        for item in os.listdir(dest):
            full_path = os.path.join(dest, item)

            # Skip non-directories
            if not os.path.isdir(full_path):
                continue

            # Skip directory names that are in the valid plugin directory array
            if item == expected_plugin_dir:
                continue

            # If any other directory begins with <plugin_name>_, delete it
            if item.startswith(name + '_'):
                print('Deleting wrong version "{}" from "{}"'
                      .format(item, dest))
                shutil.rmtree(full_path) 
Example 19
Project: webnull   Author: macrael   File: test_integration.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def tearDown(self):
        shutil.rmtree(self.temp_dir)

    # Run the given command
    # honetsly, best would be when the file changes, wait a fraction of a seonc and then send all new output and all new file
    # return when command exits. 
Example 20
Project: invenio-openaire   Author: inveniosoftware   File: conftest.py    MIT License 5 votes vote down vote up
def app(request):
    """Flask application fixture."""
    # Set temporary instance path for sqlite
    instance_path = tempfile.mkdtemp()
    app = Flask('testapp', instance_path=instance_path)
    app.config.update(
        SQLALCHEMY_DATABASE_URI=os.environ.get(
            'SQLALCHEMY_DATABASE_URI', 'sqlite:///test.db'),
        INDEXER_REPLACE_REFS=True,
        CELERY_ALWAYS_EAGER=True,
        CELERY_RESULT_BACKEND="cache",
        CELERY_CACHE_BACKEND="memory",
        CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
        JSONSCHEMAS_HOST='inveniosoftware.org',
        OPENAIRE_OAI_LOCAL_SOURCE='invenio_openaire/data/oaire_local.sqlite',
        TESTING=True,
    )

    app.url_map.converters['pid'] = PIDConverter
    app.url_map.converters['pidpath'] = PIDPathConverter

    LoginManager(app)
    InvenioDB(app)
    InvenioIndexer(app)
    InvenioRecords(app)
    InvenioCelery(app)
    InvenioPIDStore(app)
    InvenioOpenAIRE(app)
    InvenioSearch(app)
    InvenioJSONSchemas(app)

    with app.app_context():
        yield app

    shutil.rmtree(instance_path) 
Example 21
Project: phrydy   Author: Josef-Friedrich   File: _common.py    MIT License 5 votes vote down vote up
def remove_temp_dir(self):
        """Delete the temporary directory created by `create_temp_dir`.
        """
        if os.path.isdir(self.temp_dir):
            shutil.rmtree(self.temp_dir) 
Example 22
Project: pyblish-win   Author: pyblish   File: test_refactor.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS,
                               options=None, mock_log_debug=None,
                               actually_write=True):
        tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor")
        self.addCleanup(shutil.rmtree, tmpdir)
        # make a copy of the tested file that we can write to
        shutil.copy(test_file, tmpdir)
        test_file = os.path.join(tmpdir, os.path.basename(test_file))
        os.chmod(test_file, 0o644)

        def read_file():
            with open(test_file, "rb") as fp:
                return fp.read()

        old_contents = read_file()
        rt = self.rt(fixers=fixers, options=options)
        if mock_log_debug:
            rt.log_debug = mock_log_debug

        rt.refactor_file(test_file)
        self.assertEqual(old_contents, read_file())

        if not actually_write:
            return
        rt.refactor_file(test_file, True)
        new_contents = read_file()
        self.assertNotEqual(old_contents, new_contents)
        return new_contents 
Example 23
Project: pyblish-win   Author: pyblish   File: test_refactor.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_refactor_dir(self):
        def check(structure, expected):
            def mock_refactor_file(self, f, *args):
                got.append(f)
            save_func = refactor.RefactoringTool.refactor_file
            refactor.RefactoringTool.refactor_file = mock_refactor_file
            rt = self.rt()
            got = []
            dir = tempfile.mkdtemp(prefix="2to3-test_refactor")
            try:
                os.mkdir(os.path.join(dir, "a_dir"))
                for fn in structure:
                    open(os.path.join(dir, fn), "wb").close()
                rt.refactor_dir(dir)
            finally:
                refactor.RefactoringTool.refactor_file = save_func
                shutil.rmtree(dir)
            self.assertEqual(got,
                             [os.path.join(dir, path) for path in expected])
        check([], [])
        tree = ["nothing",
                "hi.py",
                ".dumb",
                ".after.py",
                "notpy.npy",
                "sappy"]
        expected = ["hi.py"]
        check(tree, expected)
        tree = ["hi.py",
                os.path.join("a_dir", "stuff.py")]
        check(tree, tree) 
Example 24
Project: pyblish-win   Author: pyblish   File: test_main.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        self.temp_dir = None  # tearDown() will rmtree this directory if set. 
Example 25
Project: pyblish-win   Author: pyblish   File: test_main.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        # Clean up logging configuration down by main.
        del logging.root.handlers[:]
        if self.temp_dir:
            shutil.rmtree(self.temp_dir) 
Example 26
Project: pyblish-win   Author: pyblish   File: test_sysconfig.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def cleanup_testfn(self):
        path = test.test_support.TESTFN
        if os.path.isfile(path):
            os.remove(path)
        elif os.path.isdir(path):
            shutil.rmtree(path) 
Example 27
Project: pyblish-win   Author: pyblish   File: support.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        # Restore working dir, for Solaris and derivatives, where rmdir()
        # on the current directory fails.
        os.chdir(self.old_cwd)
        super(TempdirManager, self).tearDown()
        while self.tempdirs:
            d = self.tempdirs.pop()
            shutil.rmtree(d, os.name in ('nt', 'cygwin')) 
Example 28
Project: pyblish-win   Author: pyblish   File: regrtest.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def restore_test_support_TESTFN(self, saved_value):
        if saved_value is None:
            if os.path.isfile(test_support.TESTFN):
                os.unlink(test_support.TESTFN)
            elif os.path.isdir(test_support.TESTFN):
                shutil.rmtree(test_support.TESTFN) 
Example 29
Project: pyblish-win   Author: pyblish   File: regrtest.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def cleanup_test_droppings(testname, verbose):
    import stat
    import gc

    # First kill any dangling references to open files etc.
    gc.collect()

    # Try to clean up junk commonly left behind.  While tests shouldn't leave
    # any files or directories behind, when a test fails that can be tedious
    # for it to arrange.  The consequences can be especially nasty on Windows,
    # since if a test leaves a file open, it cannot be deleted by name (while
    # there's nothing we can do about that here either, we can display the
    # name of the offending test, which is a real help).
    for name in (test_support.TESTFN,
                 "db_home",
                ):
        if not os.path.exists(name):
            continue

        if os.path.isdir(name):
            kind, nuker = "directory", shutil.rmtree
        elif os.path.isfile(name):
            kind, nuker = "file", os.unlink
        else:
            raise SystemError("os.path says %r exists but is neither "
                              "directory nor file" % name)

        if verbose:
            print "%r left behind %s %r" % (testname, kind, name)
        try:
            # if we have chmod, fix possible permissions problems
            # that might prevent cleanup
            if (hasattr(os, 'chmod')):
                os.chmod(name, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO)
            nuker(name)
        except Exception, msg:
            print >> sys.stderr, ("%r left behind %s %r and it couldn't be "
                "removed: %s" % (testname, kind, name, msg)) 
Example 30
Project: pyblish-win   Author: pyblish   File: test_bsddb185.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_anydbm_create(self):
        # Verify that anydbm.open does *not* create a bsddb185 file
        tmpdir = tempfile.mkdtemp()
        try:
            dbfile = os.path.join(tmpdir, "foo.db")
            anydbm.open(dbfile, "c").close()
            ftype = whichdb.whichdb(dbfile)
            self.assertNotEqual(ftype, "bsddb185")
        finally:
            shutil.rmtree(tmpdir) 
Example 31
Project: pyblish-win   Author: pyblish   File: test_support.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def rmtree(path):
    try:
        _rmtree(path)
    except OSError, e:
        # Unix returns ENOENT, Windows returns ESRCH.
        if e.errno not in (errno.ENOENT, errno.ESRCH):
            raise 
Example 32
Project: pyblish-win   Author: pyblish   File: test_support.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def temp_cwd(name='tempcwd', quiet=False):
    """
    Context manager that creates a temporary directory and set it as CWD.

    The new CWD is created in the current directory and it's named *name*.
    If *quiet* is False (default) and it's not possible to create or change
    the CWD, an error is raised.  If it's True, only a warning is raised
    and the original CWD is used.
    """
    if have_unicode and isinstance(name, unicode):
        try:
            name = name.encode(sys.getfilesystemencoding() or 'ascii')
        except UnicodeEncodeError:
            if not quiet:
                raise unittest.SkipTest('unable to encode the cwd name with '
                                        'the filesystem encoding.')
    saved_dir = os.getcwd()
    is_temporary = False
    try:
        os.mkdir(name)
        os.chdir(name)
        is_temporary = True
    except OSError:
        if not quiet:
            raise
        warnings.warn('tests may fail, unable to change the CWD to ' + name,
                      RuntimeWarning, stacklevel=3)
    try:
        yield os.getcwd()
    finally:
        os.chdir(saved_dir)
        if is_temporary:
            rmtree(name) 
Example 33
Project: pyblish-win   Author: pyblish   File: test_sysconfig.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _cleanup_testfn(self):
        path = TESTFN
        if os.path.isfile(path):
            os.remove(path)
        elif os.path.isdir(path):
            shutil.rmtree(path) 
Example 34
Project: pyblish-win   Author: pyblish   File: test_repr.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        longname = 'areallylongpackageandmodulenametotestreprtruncation'
        self.pkgname = os.path.join(longname)
        self.subpkgname = os.path.join(longname, longname)
        # Make the package and subpackage
        shutil.rmtree(self.pkgname, ignore_errors=True)
        os.mkdir(self.pkgname)
        touch(os.path.join(self.pkgname, '__init__'+os.extsep+'py'))
        shutil.rmtree(self.subpkgname, ignore_errors=True)
        os.mkdir(self.subpkgname)
        touch(os.path.join(self.subpkgname, '__init__'+os.extsep+'py'))
        # Remember where we are
        self.here = os.getcwd()
        sys.path.insert(0, self.here) 
Example 35
Project: pyblish-win   Author: pyblish   File: make_ssl_certs.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def unmake_ca():
    shutil.rmtree(TMP_CADIR) 
Example 36
Project: pyblish-win   Author: pyblish   File: test_tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_no_files_left_behind(self):
        # use a private empty directory
        our_temp_directory = tempfile.mkdtemp()
        try:
            # force _get_default_tempdir() to consider our empty directory
            def our_candidate_list():
                return [our_temp_directory]

            with support.swap_attr(tempfile, "_candidate_tempdir_list",
                                   our_candidate_list):
                # verify our directory is empty after _get_default_tempdir()
                tempfile._get_default_tempdir()
                self.assertEqual(os.listdir(our_temp_directory), [])

                def raise_OSError(*args, **kwargs):
                    raise OSError(-1)

                with support.swap_attr(io, "open", raise_OSError):
                    # test again with failing io.open()
                    with self.assertRaises(IOError) as cm:
                        tempfile._get_default_tempdir()
                    self.assertEqual(cm.exception.errno, errno.ENOENT)
                    self.assertEqual(os.listdir(our_temp_directory), [])

                open = io.open
                def bad_writer(*args, **kwargs):
                    fp = open(*args, **kwargs)
                    fp.write = raise_OSError
                    return fp

                with support.swap_attr(io, "open", bad_writer):
                    # test again with failing write()
                    with self.assertRaises(IOError) as cm:
                        tempfile._get_default_tempdir()
                    self.assertEqual(cm.exception.errno, errno.ENOENT)
                    self.assertEqual(os.listdir(our_temp_directory), [])
        finally:
            shutil.rmtree(our_temp_directory) 
Example 37
Project: pyblish-win   Author: pyblish   File: test_tempfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _inside_empty_temp_dir():
    dir = tempfile.mkdtemp()
    try:
        with support.swap_attr(tempfile, 'tempdir', dir):
            yield
    finally:
        support.rmtree(dir) 
Example 38
Project: pyblish-win   Author: pyblish   File: test_httpservers.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        try:
            os.chdir(self.cwd)
            try:
                shutil.rmtree(self.tempdir)
            except OSError:
                pass
        finally:
            BaseTestCase.tearDown(self) 
Example 39
Project: pyblish-win   Author: pyblish   File: test_import.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        sys.path[:] = self.sys_path
        if self.orig_module is not None:
            sys.modules[self.module_name] = self.orig_module
        else:
            unload(self.module_name)
        unlink(self.file_name)
        unlink(self.compiled_name)
        rmtree(self.dir_name) 
Example 40
Project: pyblish-win   Author: pyblish   File: test_import.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        # now cleanup
        if os.path.exists(self.package_name):
            symlink_support.remove_symlink(self.package_name)
        if os.path.exists(self.tagged):
            shutil.rmtree(self.tagged)
        sys.path[:] = self.orig_sys_path 
Example 41
Project: pyblish-win   Author: pyblish   File: test_tarfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_filter(self):
        tempdir = os.path.join(TEMPDIR, "filter")
        os.mkdir(tempdir)
        try:
            for name in ("foo", "bar", "baz"):
                name = os.path.join(tempdir, name)
                open(name, "wb").close()

            def filter(tarinfo):
                if os.path.basename(tarinfo.name) == "bar":
                    return
                tarinfo.uid = 123
                tarinfo.uname = "foo"
                return tarinfo

            tar = tarfile.open(tmpname, self.mode, encoding="iso8859-1")
            tar.add(tempdir, arcname="empty_dir", filter=filter)
            tar.close()

            tar = tarfile.open(tmpname, "r")
            for tarinfo in tar:
                self.assertEqual(tarinfo.uid, 123)
                self.assertEqual(tarinfo.uname, "foo")
            self.assertEqual(len(tar.getmembers()), 3)
            tar.close()
        finally:
            shutil.rmtree(tempdir)

    # Guarantee that stored pathnames are not modified. Don't
    # remove ./ or ../ or double slashes. Still make absolute
    # pathnames relative.
    # For details see bug #6054. 
Example 42
Project: pyblish-win   Author: pyblish   File: test_tarfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_extractall_symlinks(self):
        # Test if extractall works properly when tarfile contains symlinks
        tempdir = os.path.join(TEMPDIR, "testsymlinks")
        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
        os.mkdir(tempdir)
        try:
            source_file = os.path.join(tempdir,'source')
            target_file = os.path.join(tempdir,'symlink')
            with open(source_file,'w') as f:
                f.write('something\n')
            os.symlink(source_file, target_file)
            tar = tarfile.open(temparchive,'w')
            tar.add(source_file, arcname=os.path.basename(source_file))
            tar.add(target_file, arcname=os.path.basename(target_file))
            tar.close()
            # Let's extract it to the location which contains the symlink
            tar = tarfile.open(temparchive,'r')
            # this should not raise OSError: [Errno 17] File exists
            try:
                tar.extractall(path=tempdir)
            except OSError:
                self.fail("extractall failed with symlinked files")
            finally:
                tar.close()
        finally:
            os.unlink(temparchive)
            shutil.rmtree(tempdir) 
Example 43
Project: pyblish-win   Author: pyblish   File: test_tarfile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_extractall_hardlinks(self):
        # Test if extractall works properly when tarfile contains symlinks
        tempdir = os.path.join(TEMPDIR, "testsymlinks")
        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
        os.mkdir(tempdir)
        try:
            source_file = os.path.join(tempdir,'source')
            target_file = os.path.join(tempdir,'symlink')
            with open(source_file,'w') as f:
                f.write('something\n')
            os.link(source_file, target_file)
            tar = tarfile.open(temparchive,'w')
            tar.add(source_file, arcname=os.path.basename(source_file))
            tar.add(target_file, arcname=os.path.basename(target_file))
            tar.close()
            # Let's extract it to the location which contains the symlink
            tar = tarfile.open(temparchive,'r')
            # this should not raise OSError: [Errno 17] File exists
            try:
                tar.extractall(path=tempdir)
            except OSError:
                self.fail("extractall failed with linked files")
            finally:
                tar.close()
        finally:
            os.unlink(temparchive)
            shutil.rmtree(tempdir) 
Example 44
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        super(TestShutil, self).tearDown()
        while self.tempdirs:
            d = self.tempdirs.pop()
            shutil.rmtree(d, os.name in ('nt', 'cygwin')) 
Example 45
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_rmtree_errors(self):
        # filename is guaranteed not to exist
        filename = tempfile.mktemp()
        self.assertRaises(OSError, shutil.rmtree, filename) 
Example 46
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_dont_copy_file_onto_link_to_itself(self):
            # bug 851123.
            os.mkdir(TESTFN)
            src = os.path.join(TESTFN, 'cheese')
            dst = os.path.join(TESTFN, 'shop')
            try:
                f = open(src, 'w')
                f.write('cheddar')
                f.close()

                os.link(src, dst)
                self.assertRaises(shutil.Error, shutil.copyfile, src, dst)
                with open(src, 'r') as f:
                    self.assertEqual(f.read(), 'cheddar')
                os.remove(dst)

                # Using `src` here would mean we end up with a symlink pointing
                # to TESTFN/TESTFN/cheese, while it should point at
                # TESTFN/cheese.
                os.symlink('cheese', dst)
                self.assertRaises(shutil.Error, shutil.copyfile, src, dst)
                with open(src, 'r') as f:
                    self.assertEqual(f.read(), 'cheddar')
                os.remove(dst)
            finally:
                try:
                    shutil.rmtree(TESTFN)
                except OSError:
                    pass 
Example 47
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_rmtree_on_symlink(self):
            # bug 1669.
            os.mkdir(TESTFN)
            try:
                src = os.path.join(TESTFN, 'cheese')
                dst = os.path.join(TESTFN, 'shop')
                os.mkdir(src)
                os.symlink(src, dst)
                self.assertRaises(OSError, shutil.rmtree, dst)
            finally:
                shutil.rmtree(TESTFN, ignore_errors=True)

    # Issue #3002: copyfile and copytree block indefinitely on named pipes 
Example 48
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        for d in (self.src_dir, self.dst_dir, self.dir_other_fs):
            try:
                if d:
                    shutil.rmtree(d)
            except:
                pass 
Example 49
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_move_dir_other_fs(self):
        # Move a dir to another location on another filesystem.
        if not self.dir_other_fs:
            self.skipTest('dir on other filesystem not available')
        dst_dir = tempfile.mktemp(dir=self.dir_other_fs)
        try:
            self._check_move_dir(self.src_dir, dst_dir, dst_dir)
        finally:
            try:
                shutil.rmtree(dst_dir)
            except:
                pass 
Example 50
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_destinsrc_false_negative(self):
        os.mkdir(TESTFN)
        try:
            for src, dst in [('srcdir', 'srcdir/dest')]:
                src = os.path.join(TESTFN, src)
                dst = os.path.join(TESTFN, dst)
                self.assertTrue(shutil._destinsrc(src, dst),
                             msg='_destinsrc() wrongly concluded that '
                             'dst (%s) is not in src (%s)' % (dst, src))
        finally:
            shutil.rmtree(TESTFN, ignore_errors=True) 
Example 51
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_destinsrc_false_positive(self):
        os.mkdir(TESTFN)
        try:
            for src, dst in [('srcdir', 'src/dest'), ('srcdir', 'srcdir.new')]:
                src = os.path.join(TESTFN, src)
                dst = os.path.join(TESTFN, dst)
                self.assertFalse(shutil._destinsrc(src, dst),
                            msg='_destinsrc() wrongly concluded that '
                            'dst (%s) is in src (%s)' % (dst, src))
        finally:
            shutil.rmtree(TESTFN, ignore_errors=True) 
Example 52
Project: pyblish-win   Author: pyblish   File: test_gettext.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        self.env.__exit__()
        del self.env
        shutil.rmtree(os.path.split(LOCALEDIR)[0]) 
Example 53
Project: pyblish-win   Author: pyblish   File: test_py_compile.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        shutil.rmtree(self.directory)
        if self.cwd_drive:
            os.chdir(self.cwd_drive) 
Example 54
Project: pyblish-win   Author: pyblish   File: test_glob.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        shutil.rmtree(self.tempdir) 
Example 55
Project: pyblish-win   Author: pyblish   File: test_argparse.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def tearDown(self):
        os.chdir(self.old_dir)
        for root, dirs, files in os.walk(self.temp_dir, topdown=False):
            for name in files:
                os.chmod(os.path.join(self.temp_dir, name), stat.S_IWRITE)
        shutil.rmtree(self.temp_dir, True) 
Example 56
Project: pyblish-win   Author: pyblish   File: test_pkgutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        self.dirname = tempfile.mkdtemp()
        self.addCleanup(shutil.rmtree, self.dirname)
        sys.path.insert(0, self.dirname) 
Example 57
Project: pyblish-win   Author: pyblish   File: util.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def get_temp_dir():
    # get name of a temp directory which will be automatically cleaned up
    if current_process()._tempdir is None:
        import shutil, tempfile
        tempdir = tempfile.mkdtemp(prefix='pymp-')
        info('created temp directory %s', tempdir)
        Finalize(None, shutil.rmtree, args=[tempdir], exitpriority=-100)
        current_process()._tempdir = tempdir
    return current_process()._tempdir

#
# Support for reinitialization of objects when bootstrapping a child process
# 
Example 58
Project: pygis   Author: giswqs   File: path.py    MIT License 5 votes vote down vote up
def rmdir(dir_name, root_dir='.'):
    temp_dir = os.path.join(os.path.expanduser(root_dir), dir_name)
    if os.path.exists(temp_dir):
        shutil.rmtree(temp_dir) 
Example 59
Project: epub-highlighter   Author: LordAmit   File: epub_highlighter.py    MIT License 5 votes vote down vote up
def remove_extracted_directory(extract_root):
    import shutil
    shutil.rmtree(extract_root) 
Example 60
Project: arm_now   Author: nongiach   File: options.py    MIT License 5 votes vote down vote up
def clean(config):
    """ Clean the filesystem.
    """
    os.unlink(config.KERNEL)
    os.unlink(config.DTB)
    os.unlink(config.ROOTFS)
    shutil.rmtree(config.DIR, ignore_errors=True) 
Example 61
Project: aospy   Author: spencerahill   File: test_calc_basic.py    Apache License 2.0 5 votes vote down vote up
def _clean_test_direcs():
    for direc in [example_proj.direc_out, example_proj.tar_direc_out]:
        try:
            shutil.rmtree(direc)
        except OSError:
            pass 
Example 62
Project: aospy   Author: spencerahill   File: test_automate.py    Apache License 2.0 5 votes vote down vote up
def calcsuite_init_specs_single_calc(calcsuite_init_specs):
    specs = calcsuite_init_specs.copy()
    specs['variables'] = [condensation_rain]
    specs['regions'] = [None]
    specs['output_time_regional_reductions'] = ['av']
    yield specs
    # Teardown procedure
    for direc in [example_proj.direc_out, example_proj.tar_direc_out]:
        shutil.rmtree(direc, ignore_errors=True) 
Example 63
Project: aospy   Author: spencerahill   File: test_automate.py    Apache License 2.0 5 votes vote down vote up
def calcsuite_init_specs_two_calcs(calcsuite_init_specs):
    specs = calcsuite_init_specs.copy()
    specs['variables'] = [condensation_rain, convection_rain]
    specs['regions'] = [None]
    specs['output_time_regional_reductions'] = ['av']
    yield specs
    # Teardown procedure
    for direc in [example_proj.direc_out, example_proj.tar_direc_out]:
        shutil.rmtree(direc, ignore_errors=True) 
Example 64
Project: BASS   Author: Cisco-Talos   File: bindiff.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_pickle_export(self, sample, is_64_bit = True, timeout = None):
        """
        Load a sample into IDA Pro, perform autoanalysis and export a pickle file. 
        :param sample: The sample's path
        :param is_64_bit: If the sample needs to be analyzed by the 64 bit version of IDA
        :param timeout: Timeout for the analysis in seconds
        :return: The file name of the exported pickle database. The file needs
        to be deleted by the caller. Returns None on error.
        """

        data_to_send = {
            "timeout": timeout,
            "is_64_bit": is_64_bit}
        url = "%s/binexport_pickle" % next(self._urls)
        log.debug("curl -XPOST --data '%s' '%s'", json.dumps(data_to_send), url)
        response = requests.post(url, data = data_to_send, files = {os.path.basename(sample): open(sample, "rb")})
        if response.status_code == 200:
            handle_tar, path_tar = tempfile.mkstemp(suffix = ".tar.gz")
            with os.fdopen(handle_tar, "wb") as f:
                map(f.write, response.iter_content(1024))
            directory = tempfile.mkdtemp()
            subprocess.check_call(["tar", "xf", path_tar], cwd = directory)

            handle_bindiff, output_bindiff = tempfile.mkstemp(suffix = ".BinExport")
            with os.fdopen(handle_bindiff, "wb") as f:
                with open(os.path.join(directory, "output.BinExport"), "rb") as f2:
                    shutil.copyfileobj(f2, f)
            handle_pickle, output_pickle = tempfile.mkstemp(suffix = ".pickle")
            with os.fdopen(handle_pickle, "wb") as f:
                with open(os.path.join(directory, "output.pickle"), "rb") as f2:
                    shutil.copyfileobj(f2, f)
            os.unlink(path_tar)
            shutil.rmtree(directory)
            return output_bindiff, output_pickle
        else:
            log.error("Bindiff server responded with status code %d: %s", response.status_code, response.content)
            return None 
Example 65
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output = os.path.join(directory, "output.BinExport")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', True)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "binexport", output)
            logger.info("Command completed successfully")
            return send_file(open(output, "rb"), as_attachment = True, attachment_filename = "%s.BinExport" % filename, mimetype = "application/binary")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
            
    finally:
        if directory is not None:
            shutil.rmtree(directory) 
Example 66
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_pickle_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_pickle_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output_binexport = os.path.join(directory, "output.BinExport")
        output_pickle = os.path.join(directory, "output.pickle")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', True)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "binexport_pickle", output_binexport, output_pickle)
            logger.info("Command completed successfully")
            output_tar = os.path.join(directory, "output.tar.gz")
            subprocess.check_call(["tar", "czf", output_tar, os.path.relpath(output_binexport, directory), os.path.relpath(output_pickle, directory)], cwd = directory)
            return send_file(open(output_tar, "rb"), as_attachment = True, attachment_filename = "%s.tar.gz" % filename, mimetype = "application/gzip")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
    finally:
        if directory is not None:
            shutil.rmtree(directory) 
Example 67
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def pickle_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output = os.path.join(directory, "output.pickle")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', False)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "pickle", output)
            logger.info("Command completed successfully")
            return send_file(open(output, "rb"), as_attachment = True, attachment_filename = "%s.pickle" % filename, mimetype = "application/binary")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
    finally:
        if directory is not None:
            shutil.rmtree(directory) 
Example 68
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_compare():
    logger.info("bindiff_compare called")

    input_dir = tempfile.mkdtemp()
    output_dir = tempfile.mkdtemp()
    try:
        primary = os.path.join(input_dir, "primary")
        secondary = os.path.join(input_dir, "secondary")
        try:
            request.files["primary"].save(primary)
            request.files["secondary"].save(secondary)
        except KeyError:
            return make_response(jsonify(error="Missing parameter 'primary' or 'secondary'"), 422)

        timeout = request.form.get('timeout', None)

        cmd = (BINDIFF_DIFFER, "--primary", primary, "--secondary", secondary, "--output_dir", output_dir)
        logger.info("Executing %s", " ".join("'%s'" % x for x in cmd))
        check_call(cmd, cwd = output_dir, timeout = timeout)
        db_path = [os.path.join(output_dir, x) for x in os.listdir(output_dir)]
        if len(db_path) != 1:
            return make_response(jsonify(error = "BinDiff generated 0 or several output files"), 500)
        return send_file(open(db_path[0], "rb"), as_attachment = True, attachment_filename = "BinDiff.sqlite3", mimetype = "application/binary")
    except OSError as err:
        if err.errno == -9:
            return make_response(jsonify(error = "Program execution timed out"), 408)
        else:
            return make_response(jsonify(error = "Program execution failed with error %d" % err.errno), 500)
    finally:
        shutil.rmtree(input_dir)
        shutil.rmtree(output_dir) 
Example 69
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output = os.path.join(directory, "output.BinExport")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', True)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "binexport", output)
            logger.info("Command completed successfully")
            return send_file(open(output, "rb"), as_attachment = True, attachment_filename = "%s.BinExport" % filename, mimetype = "application/binary")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
            
    finally:
        if directory is not None:
            shutil.rmtree(directory) 
Example 70
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def pickle_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output = os.path.join(directory, "output.pickle")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', False)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "pickle", output)
            logger.info("Command completed successfully")
            return send_file(open(output, "rb"), as_attachment = True, attachment_filename = "%s.pickle" % filename, mimetype = "application/binary")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
    finally:
        if directory is not None:
            shutil.rmtree(directory) 
Example 71
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 5 votes vote down vote up
def bindiff_compare():
    logger.info("bindiff_compare called")

    input_dir = tempfile.mkdtemp()
    output_dir = tempfile.mkdtemp()
    try:
        primary = os.path.join(input_dir, "primary")
        secondary = os.path.join(input_dir, "secondary")
        try:
            request.files["primary"].save(primary)
            request.files["secondary"].save(secondary)
        except KeyError:
            return make_response(jsonify(error="Missing parameter 'primary' or 'secondary'"), 422)

        timeout = request.form.get('timeout', None)

        cmd = (BINDIFF_DIFFER, "--primary", primary, "--secondary", secondary, "--output_dir", output_dir)
        logger.info("Executing %s", " ".join("'%s'" % x for x in cmd))
        check_call(cmd, cwd = output_dir, timeout = timeout)
        db_path = [os.path.join(output_dir, x) for x in os.listdir(output_dir)]
        if len(db_path) != 1:
            return make_response(jsonify(error = "BinDiff generated 0 or several output files"), 500)
        return send_file(open(db_path[0], "rb"), as_attachment = True, attachment_filename = "BinDiff.sqlite3", mimetype = "application/binary")
    except OSError as err:
        if err.errno == -9:
            return make_response(jsonify(error = "Program execution timed out"), 408)
        else:
            return make_response(jsonify(error = "Program execution failed with error %d" % err.errno), 500)
    finally:
        shutil.rmtree(input_dir)
        shutil.rmtree(output_dir) 
Example 72
Project: claxon   Author: vanatteveldt   File: ml.py    GNU General Public License v3.0 5 votes vote down vote up
def delete_session_model(session: Session):
    model_dir = get_model_dir(session=session)
    logging.debug("Deleting model {model_dir}".format(**locals()))
    shutil.rmtree(model_dir) 
Example 73
Project: fs_image   Author: facebookincubator   File: fs_utils.py    MIT License 4 votes vote down vote up
def populate_temp_dir_and_rename(dest_path, overwrite=False) -> Path:
    '''
    Returns a Path to a temporary directory. The context block may populate
    this directory, which will then be renamed to `dest_path`, optionally
    deleting any preexisting directory (if `overwrite=True`).

    If the context block throws, the partially populated temporary directory
    is removed, while `dest_path` is left alone.

    By writing to a brand-new temporary directory before renaming, we avoid
    the problems of partially writing files, or overwriting some files but
    not others.  Moreover, populate-temporary-and-rename is robust to
    concurrent writers, and tends to work on broken NFSes unlike `flock`.
    '''
    dest_path = os.path.normpath(dest_path)  # Trailing / breaks `os.rename()`
    # Putting the temporary directory as a sibling minimizes permissions
    # issues, and maximizes the chance that we're on the same filesystem
    base_dir = os.path.dirname(dest_path)
    td = tempfile.mkdtemp(dir=base_dir)
    try:
        yield Path(td)

        # Delete+rename is racy, but EdenFS lacks RENAME_EXCHANGE (t34057927)
        # Retry if we raced with another writer -- i.e., last-to-run wins.
        while True:
            if overwrite and os.path.isdir(dest_path):
                with tempfile.TemporaryDirectory(dir=base_dir) as del_dir:
                    try:
                        os.rename(dest_path, del_dir)
                    except FileNotFoundError:  # pragma: no cover
                        continue  # retry, another writer deleted first?
            try:
                os.rename(td, dest_path)
            except OSError as ex:
                if not (overwrite and ex.errno in [
                    # Different kernels have different error codes when the
                    # target already exists and is a nonempty directory.
                    errno.ENOTEMPTY, errno.EEXIST,
                ]):
                    raise
                log.exception(  # pragma: no cover
                    f'Retrying deleting {dest_path}, another writer raced us'
                )
            break  # We won the race
    except BaseException:
        shutil.rmtree(td)
        raise 
Example 74
Project: gog-galaxy-plugin-downloader   Author: Slashbunny   File: download.py    GNU General Public License v3.0 4 votes vote down vote up
def download_plugins(data, dest):
    """
    Downloads and extracts plugins

    If the destination directory already exists, the plugin is skipped
    """
    for name, data in data.items():
        version = data['version']
        guid = data['guid']
        url = data['url']
        if 'archive_path' in data:
            archive_path = data['archive_path']
        else:
            archive_path = None

        # Destination directory
        dest_dir = os.path.join(dest, name + '_' + guid)

        if os.path.isdir(dest_dir):
            # If destination path exists, check the version that is installed
            with open(os.path.join(dest_dir, 'manifest.json')) as m:
                data = json.load(m)
                existing_version = data['version']

                # Close json file
                m.close()

                # Version already matches
                if version == existing_version:
                    print('NOTICE: Skipping "{}" download, "{}" already exists'
                          .format(name, version))
                    continue
                # Version does not match, delete old plugin and download
                else:
                    shutil.rmtree(dest_dir)
                    print('Upgrading {} plugin'.format(name))

        # Passed Pre-Download Checks
        print('Downloading "{}" version "{}" ({})'
              .format(name, version, guid))

        # Download zip file into memory
        plugin_url = urlopen(url)
        plugin_zip = zipfile.ZipFile(BytesIO(plugin_url.read()))

        # Extract zip directly into destination if there are no sub-folders
        #  to deal with
        if not archive_path:
            plugin_zip.extractall(path=dest_dir)
        else:
            # Create temporary directory for extraction
            tmp_dir = tempfile.TemporaryDirectory(prefix='galaxy-plugin')

            # Extract zip to temporary directory
            plugin_zip.extractall(path=tmp_dir.name)

            # Move sub-directory into final destination directory
            shutil.move(os.path.join(tmp_dir.name, archive_path), dest_dir)

            # Cleanup temporary directory
            tmp_dir.cleanup() 
Example 75
Project: alfred-yubikey-otp   Author: robertoriv   File: notify.py    MIT License 4 votes vote down vote up
def png_to_icns(png_path, icns_path):
    """Convert PNG file to ICNS using ``iconutil``.

    Create an iconset from the source PNG file. Generate PNG files
    in each size required by macOS, then call ``iconutil`` to turn
    them into a single ICNS file.

    Args:
        png_path (str): Path to source PNG file.
        icns_path (str): Path to destination ICNS file.

    Raises:
        RuntimeError: Raised if ``iconutil`` or ``sips`` fail.
    """
    tempdir = tempfile.mkdtemp(prefix='aw-', dir=wf().datadir)

    try:
        iconset = os.path.join(tempdir, 'Icon.iconset')

        assert not os.path.exists(iconset), \
            'iconset already exists: ' + iconset
        os.makedirs(iconset)

        # Copy source icon to icon set and generate all the other
        # sizes needed
        configs = []
        for i in (16, 32, 128, 256, 512):
            configs.append(('icon_{0}x{0}.png'.format(i), i))
            configs.append((('icon_{0}x{0}@2x.png'.format(i), i * 2)))

        shutil.copy(png_path, os.path.join(iconset, 'icon_256x256.png'))
        shutil.copy(png_path, os.path.join(iconset, '[email protected]'))

        for name, size in configs:
            outpath = os.path.join(iconset, name)
            if os.path.exists(outpath):
                continue
            convert_image(png_path, outpath, size)

        cmd = [
            b'iconutil',
            b'-c', b'icns',
            b'-o', icns_path,
            iconset]

        retcode = subprocess.call(cmd)
        if retcode != 0:
            raise RuntimeError('iconset exited with %d' % retcode)

        assert os.path.exists(icns_path), \
            'generated ICNS file not found: ' + repr(icns_path)
    finally:
        try:
            shutil.rmtree(tempdir)
        except OSError:  # pragma: no cover
            pass 
Example 76
Project: pyblish-win   Author: pyblish   File: __init__.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def bootstrap(root=None, upgrade=False, user=False,
              altinstall=False, default_pip=True,
              verbosity=0):
    """
    Bootstrap pip into the current Python installation (or the given root
    directory).

    Note that calling this function will alter both sys.path and os.environ.
    """
    if altinstall and default_pip:
        raise ValueError("Cannot use altinstall and default_pip together")

    _require_ssl_for_pip()
    _disable_pip_configuration_settings()

    # By default, installing pip and setuptools installs all of the
    # following scripts (X.Y == running Python version):
    #
    #   pip, pipX, pipX.Y, easy_install, easy_install-X.Y
    #
    # pip 1.5+ allows ensurepip to request that some of those be left out
    if altinstall:
        # omit pip, pipX and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "altinstall"
    elif not default_pip:
        # omit pip and easy_install
        os.environ["ENSUREPIP_OPTIONS"] = "install"

    tmpdir = tempfile.mkdtemp()
    try:
        # Put our bundled wheels into a temporary directory and construct the
        # additional paths that need added to sys.path
        additional_paths = []
        for project, version in _PROJECTS:
            wheel_name = "{}-{}-py2.py3-none-any.whl".format(project, version)
            whl = pkgutil.get_data(
                "ensurepip",
                "_bundled/{}".format(wheel_name),
            )
            with open(os.path.join(tmpdir, wheel_name), "wb") as fp:
                fp.write(whl)

            additional_paths.append(os.path.join(tmpdir, wheel_name))

        # Construct the arguments to be passed to the pip command
        args = ["install", "--no-index", "--find-links", tmpdir]
        if root:
            args += ["--root", root]
        if upgrade:
            args += ["--upgrade"]
        if user:
            args += ["--user"]
        if verbosity:
            args += ["-" + "v" * verbosity]

        _run_pip(args + [p[0] for p in _PROJECTS], additional_paths)
    finally:
        shutil.rmtree(tmpdir, ignore_errors=True) 
Example 77
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def test_copytree_simple(self):
        def write_data(path, data):
            f = open(path, "w")
            f.write(data)
            f.close()

        def read_data(path):
            f = open(path)
            data = f.read()
            f.close()
            return data

        src_dir = tempfile.mkdtemp()
        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')

        write_data(os.path.join(src_dir, 'test.txt'), '123')

        os.mkdir(os.path.join(src_dir, 'test_dir'))
        write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456')

        try:
            shutil.copytree(src_dir, dst_dir)
            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
            self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
                                                        'test.txt')))
            actual = read_data(os.path.join(dst_dir, 'test.txt'))
            self.assertEqual(actual, '123')
            actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt'))
            self.assertEqual(actual, '456')
        finally:
            for path in (
                    os.path.join(src_dir, 'test.txt'),
                    os.path.join(dst_dir, 'test.txt'),
                    os.path.join(src_dir, 'test_dir', 'test.txt'),
                    os.path.join(dst_dir, 'test_dir', 'test.txt'),
                ):
                if os.path.exists(path):
                    os.remove(path)
            for path in (src_dir,
                    os.path.dirname(dst_dir)
                ):
                if os.path.exists(path):
                    shutil.rmtree(path) 
Example 78
Project: pyblish-win   Author: pyblish   File: test_posix.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def test_getcwd_long_pathnames(self):
        dirname = 'getcwd-test-directory-0123456789abcdef-01234567890abcdef'
        curdir = os.getcwd()
        base_path = os.path.abspath(test_support.TESTFN) + '.getcwd'

        try:
            os.mkdir(base_path)
            os.chdir(base_path)
        except:
            self.skipTest("cannot create directory for testing")

        try:
            def _create_and_do_getcwd(dirname, current_path_length = 0):
                try:
                    os.mkdir(dirname)
                except:
                    self.skipTest("mkdir cannot create directory sufficiently "
                                  "deep for getcwd test")

                os.chdir(dirname)
                try:
                    os.getcwd()
                    if current_path_length < 4099:
                        _create_and_do_getcwd(dirname, current_path_length + len(dirname) + 1)
                except OSError as e:
                    expected_errno = errno.ENAMETOOLONG
                    # The following platforms have quirky getcwd()
                    # behaviour -- see issue 9185 and 15765 for
                    # more information.
                    quirky_platform = (
                        'sunos' in sys.platform or
                        'netbsd' in sys.platform or
                        'openbsd' in sys.platform
                    )
                    if quirky_platform:
                        expected_errno = errno.ERANGE
                    self.assertEqual(e.errno, expected_errno)
                finally:
                    os.chdir('..')
                    os.rmdir(dirname)

            _create_and_do_getcwd(dirname)

        finally:
            os.chdir(curdir)
            shutil.rmtree(base_path) 
Example 79
Project: aospy   Author: spencerahill   File: calc.py    Apache License 2.0 4 votes vote down vote up
def _write_to_tar(self, dtype_out_time):
        """Add the data to the tar file in tar_out_direc."""
        # When submitted in parallel and the directory does not exist yet
        # multiple processes may try to create a new directory; this leads
        # to an OSError for all processes that tried to make the
        # directory, but were later than the first.
        try:
            os.makedirs(self.dir_tar_out)
        except OSError:
            pass
        # tarfile 'append' mode won't overwrite the old file, which we want.
        # So open in 'read' mode, extract the file, and then delete it.
        # But 'read' mode throws OSError if file doesn't exist: make it first.
        utils.io.dmget([self.path_tar_out])
        with tarfile.open(self.path_tar_out, 'a') as tar:
            pass
        with tarfile.open(self.path_tar_out, 'r') as tar:
            old_data_path = os.path.join(self.dir_tar_out,
                                         self.file_name[dtype_out_time])
            try:
                tar.extract(self.file_name[dtype_out_time],
                            path=old_data_path)
            except KeyError:
                pass
            else:
                # The os module treats files on archive as non-empty
                # directories, so can't use os.remove or os.rmdir.
                shutil.rmtree(old_data_path)
                retcode = subprocess.call([
                    "tar", "--delete", "--file={}".format(self.path_tar_out),
                    self.file_name[dtype_out_time]
                ])
                if retcode:
                    msg = ("The 'tar' command to save your aospy output "
                           "exited with an error.  Most likely, this is due "
                           "to using an old version of 'tar' (especially if "
                           "you are on a Mac).  Consider installing a newer "
                           "version of 'tar' or disabling tar output by "
                           "setting `write_to_tar=False` in the "
                           "`calc_exec_options` argument of "
                           "`submit_mult_calcs`.")
                    logging.warn(msg)
        with tarfile.open(self.path_tar_out, 'a') as tar:
            tar.add(self.path_out[dtype_out_time],
                    arcname=self.file_name[dtype_out_time]) 
Example 80
Project: BASS   Author: Cisco-Talos   File: ida_service.py    GNU General Public License v2.0 4 votes vote down vote up
def bindiff_pickle_export():
    """
    Run the IDA Pro autoanalysis on the input file and export a BinExport database.
    :param input: The input file
    :return: Status code 200 and a JSON object containing the output database
        name in key 'output', or status code 422 on invalid parameters, 408 on
        timeout or 500 on other errors.
    """
    logger.info("bindiff_pickle_export called")

    directory = None
    try:
        directory = tempfile.mkdtemp()
        if len(request.files) != 1:
            return make_response(jsonify(error = "Missing file parameter"), 422)

        filename, file_ = request.files.items()[0]
        input_ = os.path.join(directory, sanitize_filename(filename))
        file_.save(input_)

        output_binexport = os.path.join(directory, "output.BinExport")
        output_pickle = os.path.join(directory, "output.pickle")

        timeout = request.form.get('timeout', None)
        is_64_bit = request.form.get('is_64_bit', True)
        try:
            run_ida(input_, is_64_bit, timeout, os.path.join(PREFIX, "export_binexport_pickle.py"), "binexport_pickle", output_binexport, output_pickle)
            logger.info("Command completed successfully")
            output_tar = os.path.join(directory, "output.tar.gz")
            subprocess.check_call(["tar", "czf", output_tar, os.path.relpath(output_binexport, directory), os.path.relpath(output_pickle, directory)], cwd = directory)
            return send_file(open(output_tar, "rb"), as_attachment = True, attachment_filename = "%s.tar.gz" % filename, mimetype = "application/gzip")
        except TimeoutError:
            return jsonify(error = "Program execution timed out"), 408
        except OSError as err:
            return jsonify(error = "Program execution failed with error %d" % err.errno), 500
    finally:
        if directory is not None:
            shutil.rmtree(directory)