Python os.mkdir() Examples

def main(arguments):
    parser = argparse.ArgumentParser()
    parser.add_argument('--data_dir', help='directory to save data to', type=str, default='glue_data')
    parser.add_argument('--tasks', help='tasks to download data for as a comma separated string',
                        type=str, default='all')
    parser.add_argument('--path_to_mrpc',
                        help='path to directory containing extracted MRPC data, msr_paraphrase_train.txt and msr_paraphrase_text.txt',
                        type=str, default='')
    args = parser.parse_args(arguments)

    if not os.path.isdir(args.data_dir):
        os.mkdir(args.data_dir)
    tasks = get_tasks(args.tasks)

    for task in tasks:
        if task == 'MRPC':
            format_mrpc(args.data_dir, args.path_to_mrpc)
        elif task == 'diagnostic':
            download_diagnostic(args.data_dir)
        else:
            download_and_extract(task, args.data_dir) 

def check_configs(self):
        """
        Checks if config directories and files exist and creates them if they do not
        """
        try:
            if not os.path.isdir(self.base_dir):
                os.mkdir(self.base_dir)
            if not os.path.isdir(self.config_path):
                os.mkdir(self.config_path)
            if not os.path.isdir(self.scripts_path):
                os.mkdir(self.scripts_path)
            if not os.path.isfile(self.conf_path):
                self.config['USER'] = {}
                self.config['SETTINGS'] = {'MAC_RANDOMIZER': 'False', 'KILL_SWITCH': 'False', 'AUTO_CONNECT': 'False'}
                self.write_conf()

        except PermissionError:
            self.statusbar.showMessage("Insufficient Permissions to create config folder", 2000) 

def snapshot_gpg_keys(
    *, key_urls: Iterable[str], whitelist_dir: Path, snapshot_dir: Path,
):
    os.mkdir(snapshot_dir / 'gpg_keys')
    for url in key_urls:
        with open_url(url) as key_file:
            key_content = key_file.read()

        # Check that the key is in our whitelist, and the content matches.
        filename = os.path.basename(urlparse(url).path)
        with open(whitelist_dir / filename, 'rb') as infile:
            whitelist_key = infile.read()
            assert whitelist_key == key_content, (whitelist_key, key_content)

        with create_ro(snapshot_dir / 'gpg_keys' / filename, 'wb') as outfile:
            outfile.write(whitelist_key) 

def test_list_subvolume_wrappers(self):
        with tempfile.TemporaryDirectory() as td:
            self.assertEqual([], sgc.list_subvolume_wrappers(td))

            self._touch(td, 'ba:nana')  # Not a directory
            self.assertEqual([], sgc.list_subvolume_wrappers(td))

            os.mkdir(os.path.join(td, 'apple'))  # No colon
            self.assertEqual([], sgc.list_subvolume_wrappers(td))

            os.mkdir(os.path.join(td, 'p:i'))
            os.mkdir(os.path.join(td, 'e:'))
            os.mkdir(os.path.join(td, ':x'))
            self.assertEqual(
                {'p:i', 'e:', ':x'}, set(sgc.list_subvolume_wrappers(td)),
            ) 

def ensure_per_repo_artifacts_dir_exists(path_in_repo: str) -> str:
    "See `find_repo_root`'s docblock to understand `path_in_repo`"
    repo_path = find_repo_root(path_in_repo)
    artifacts_dir = os.path.join(repo_path, 'buck-image-out')

    # On Facebook infra, the repo might be hosted on an Eden filesystem,
    # which is not intended as a backing store for a large sparse loop
    # device filesystem.  So, we will put our artifacts in a blessed scratch
    # space instead.
    #
    # The location in the scratch directory is a hardcoded path because
    # this really must be a per-repo singleton.
    real_dir = _maybe_make_symlink_to_scratch(
        artifacts_dir, 'buck-image-out', repo_path
    )

    try:
        os.mkdir(real_dir)
    except FileExistsError:
        pass  # May race with another mkdir from a concurrent artifacts_dir.py
    return artifacts_dir 

def process(self):
        leapp_home = "/root/tmp_leapp_py3"
        py3_leapp = os.path.join(leapp_home, "leapp3")
        os.mkdir(leapp_home)
        os.symlink(
            "/usr/lib/python2.7/site-packages/leapp",
            os.path.join(leapp_home, "leapp"))
        with open(py3_leapp, "w") as f:
            f_content = [
                "#!/usr/bin/python3",
                "import sys",
                "sys.path.append('{}')".format(leapp_home),
                "",
                "import leapp.cli",
                "sys.exit(leapp.cli.main())",
            ]
            f.write("{}\n\n".format("\n".join(f_content)))
        os.chmod(py3_leapp, 0o770) 

def makedirs(dir):
    if not dir:
        return
    if os.path.exists(dir):
        if not os.path.isdir(dir):
            try:
                os.rename(dir, dir + ".bak")
                os.mkdir(dir)
                os.rename(dir + ".bak", os.path.join(dir, "index.html"))
            except os.error:
                pass
        return
    head, tail = os.path.split(dir)
    if not tail:
        print "Huh?  Don't know how to make dir", dir
        return
    makedirs(head)
    os.mkdir(dir, 0777) 

def process(slave, master):
    cvsdir = os.path.join(master, "CVS")
    if not os.path.isdir(cvsdir):
        print "skipping master subdirectory", master
        print "-- not under CVS"
        return
    print "-"*40
    print "slave ", slave
    print "master", master
    if not os.path.isdir(slave):
        if not okay("create slave directory %s?" % slave,
                    answer=create_directories):
            print "skipping master subdirectory", master
            print "-- no corresponding slave", slave
            return
        print "creating slave directory", slave
        try:
            os.mkdir(slave)
        except os.error, msg:
            print "can't make slave directory", slave, ":", msg
            return
        else:
            print "made slave directory", slave 

def main():
    if not 3 <= len(sys.argv) <= 4:
        print 'usage:', sys.argv[0], 'oldtree newtree [linkto]'
        return 2
    oldtree, newtree = sys.argv[1], sys.argv[2]
    if len(sys.argv) > 3:
        link = sys.argv[3]
        link_may_fail = 1
    else:
        link = LINK
        link_may_fail = 0
    if not os.path.isdir(oldtree):
        print oldtree + ': not a directory'
        return 1
    try:
        os.mkdir(newtree, 0777)
    except os.error, msg:
        print newtree + ': cannot mkdir:', msg
        return 1 

def setup_test_source_trees(self):
        """Setup a test source tree and output destination tree."""
        self.temp_dir = tempfile.mkdtemp()  # tearDown() cleans this up.
        self.py2_src_dir = os.path.join(self.temp_dir, "python2_project")
        self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project")
        os.mkdir(self.py2_src_dir)
        os.mkdir(self.py3_dest_dir)
        # Turn it into a package with a few files.
        self.setup_files = []
        open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close()
        self.setup_files.append("__init__.py")
        shutil.copy(PY2_TEST_MODULE, self.py2_src_dir)
        self.setup_files.append(os.path.basename(PY2_TEST_MODULE))
        self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py")
        self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py")
        with open(self.trivial_py2_file, "w") as trivial:
            trivial.write("print 'I need a simple conversion.'")
        self.setup_files.append("trivial.py") 

def test_user_site(self):
        import site
        dist = Distribution({'name': 'xx'})
        cmd = build_ext(dist)

        # making sure the user option is there
        options = [name for name, short, label in
                   cmd.user_options]
        self.assertIn('user', options)

        # setting a value
        cmd.user = 1

        # setting user based lib and include
        lib = os.path.join(site.USER_BASE, 'lib')
        incl = os.path.join(site.USER_BASE, 'include')
        os.mkdir(lib)
        os.mkdir(incl)

        cmd.ensure_finalized()

        # see if include_dirs and library_dirs were set
        self.assertIn(lib, cmd.library_dirs)
        self.assertIn(lib, cmd.rpath)
        self.assertIn(incl, cmd.include_dirs) 

def test_newer_pairwise(self):
        tmpdir = self.mkdtemp()
        sources = os.path.join(tmpdir, 'sources')
        targets = os.path.join(tmpdir, 'targets')
        os.mkdir(sources)
        os.mkdir(targets)
        one = os.path.join(sources, 'one')
        two = os.path.join(sources, 'two')
        three = os.path.abspath(__file__)    # I am the old file
        four = os.path.join(targets, 'four')
        self.write_file(one)
        self.write_file(two)
        self.write_file(four)

        self.assertEqual(newer_pairwise([one, two], [three, four]),
                         ([one],[three])) 

def test_run_setup_uses_current_dir(self):
        # This tests that the setup script is run with the current directory
        # as its own current directory; this was temporarily broken by a
        # previous patch when TESTFN did not use the current directory.
        sys.stdout = StringIO.StringIO()
        cwd = os.getcwd()

        # Create a directory and write the setup.py file there:
        os.mkdir(test.test_support.TESTFN)
        setup_py = os.path.join(test.test_support.TESTFN, "setup.py")
        distutils.core.run_setup(
            self.write_setup(setup_prints_cwd, path=setup_py))

        output = sys.stdout.getvalue()
        if output.endswith("\n"):
            output = output[:-1]
        self.assertEqual(cwd, output) 

def __init__(self, dirname, factory=rfc822.Message, create=True):
        """Initialize a Maildir instance."""
        Mailbox.__init__(self, dirname, factory, create)
        self._paths = {
            'tmp': os.path.join(self._path, 'tmp'),
            'new': os.path.join(self._path, 'new'),
            'cur': os.path.join(self._path, 'cur'),
            }
        if not os.path.exists(self._path):
            if create:
                os.mkdir(self._path, 0700)
                for path in self._paths.values():
                    os.mkdir(path, 0o700)
            else:
                raise NoSuchMailboxError(self._path)
        self._toc = {}
        self._toc_mtimes = {'cur': 0, 'new': 0}
        self._last_read = 0         # Records last time we read cur/new
        self._skewfactor = 0.1      # Adjust if os/fs clocks are skewing 

def test_security(self):
        # This test is incomplete since we are normally not run as root and
        # therefore can't test the file ownership being wrong.
        d = test_support.TESTFN
        os.mkdir(d)
        self.addCleanup(test_support.rmtree, d)
        fn = os.path.join(d, '.netrc')
        with open(fn, 'wt') as f:
            f.write("""\
                machine foo.domain.com login bar password pass
                default login foo password pass
                """)
        with test_support.EnvironmentVarGuard() as environ:
            environ.set('HOME', d)
            os.chmod(fn, 0600)
            nrc = netrc.netrc()
            self.assertEqual(nrc.hosts['foo.domain.com'],
                             ('bar', None, 'pass'))
            os.chmod(fn, 0o622)
            self.assertRaises(netrc.NetrcParseError, netrc.netrc) 

def mkhier(self, descr):
        root = tempfile.mkdtemp()
        sys.path.insert(0, root)
        if not os.path.isdir(root):
            os.mkdir(root)
        for name, contents in descr:
            comps = name.split()
            fullname = root
            for c in comps:
                fullname = os.path.join(fullname, c)
            if contents is None:
                os.mkdir(fullname)
            else:
                f = open(fullname, "w")
                f.write(contents)
                if contents and contents[-1] != '\n':
                    f.write('\n')
                f.close()
        self.root = root
        # package name is the name of the first item
        self.pkgname = descr[0][0] 

def test_apropos_with_bad_package(self):
        # Issue 7425 - pydoc -k failed when bad package on path
        pkgdir = os.path.join(TESTFN, "syntaxerr")
        os.mkdir(pkgdir)
        badsyntax = os.path.join(pkgdir, "__init__") + os.extsep + "py"
        with open(badsyntax, 'w') as f:
            f.write("invalid python syntax = $1\n")
        with self.restrict_walk_packages(path=[TESTFN]):
            with captured_stdout() as out:
                with captured_stderr() as err:
                    pydoc.apropos('xyzzy')
            # No result, no error
            self.assertEqual(out.getvalue(), '')
            self.assertEqual(err.getvalue(), '')
            # The package name is still matched
            with captured_stdout() as out:
                with captured_stderr() as err:
                    pydoc.apropos('syntaxerr')
            self.assertEqual(out.getvalue().strip(), 'syntaxerr')
            self.assertEqual(err.getvalue(), '') 

def test_realpath_resolve_parents(self):
            # We also need to resolve any symlinks in the parents of a relative
            # path passed to realpath. E.g.: current working directory is
            # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call
            # realpath("a"). This should return /usr/share/doc/a/.
            try:
                old_path = abspath('.')
                os.mkdir(ABSTFN)
                os.mkdir(ABSTFN + "/y")
                os.symlink(ABSTFN + "/y", ABSTFN + "/k")

                os.chdir(ABSTFN + "/k")
                self.assertEqual(realpath("a"), ABSTFN + "/y/a")
            finally:
                os.chdir(old_path)
                test_support.unlink(ABSTFN + "/k")
                safe_rmdir(ABSTFN + "/y")
                safe_rmdir(ABSTFN) 

def test_realpath_resolve_first(self):
            # Bug #1213894: The first component of the path, if not absolute,
            # must be resolved too.

            try:
                old_path = abspath('.')
                os.mkdir(ABSTFN)
                os.mkdir(ABSTFN + "/k")
                os.symlink(ABSTFN, ABSTFN + "link")
                os.chdir(dirname(ABSTFN))

                base = basename(ABSTFN)
                self.assertEqual(realpath(base + "link"), ABSTFN)
                self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k")
            finally:
                os.chdir(old_path)
                test_support.unlink(ABSTFN + "link")
                safe_rmdir(ABSTFN + "/k")
                safe_rmdir(ABSTFN) 

def setUp(self):
        tmpdir = tempfile.gettempdir()
        self.dir = os.path.join(tmpdir, 'dir')
        self.dir_same = os.path.join(tmpdir, 'dir-same')
        self.dir_diff = os.path.join(tmpdir, 'dir-diff')
        self.caseinsensitive = os.path.normcase('A') == os.path.normcase('a')
        data = 'Contents of file go here.\n'
        for dir in [self.dir, self.dir_same, self.dir_diff]:
            shutil.rmtree(dir, True)
            os.mkdir(dir)
            if self.caseinsensitive and dir is self.dir_same:
                fn = 'FiLe'     # Verify case-insensitive comparison
            else:
                fn = 'file'
            output = open(os.path.join(dir, fn), 'w')
            output.write(data)
            output.close()

        output = open(os.path.join(self.dir_diff, 'file2'), 'w')
        output.write('An extra file.\n')
        output.close() 

def make_ca():
    os.mkdir(TMP_CADIR)
    with open(os.path.join('cadir','index.txt'),'a+') as f:
        pass # empty file
    with open(os.path.join('cadir','crl.txt'),'a+') as f:
        f.write("00")
    with open(os.path.join('cadir','index.txt.attr'),'w+') as f:
        f.write('unique_subject = no')

    with tempfile.NamedTemporaryFile("w") as t:
        t.write(req_template.format(hostname='our-ca-server'))
        t.flush()
        with tempfile.NamedTemporaryFile() as f:
            args = ['req', '-new', '-days', '3650', '-extensions', 'v3_ca', '-nodes',
                    '-newkey', 'rsa:2048', '-keyout', 'pycakey.pem',
                    '-out', f.name,
                    '-subj', '/C=XY/L=Castle Anthrax/O=Python Software Foundation CA/CN=our-ca-server']
            check_call(['openssl'] + args)
            args = ['ca', '-config', t.name, '-create_serial',
                    '-out', 'pycacert.pem', '-batch', '-outdir', TMP_CADIR,
                    '-keyfile', 'pycakey.pem', '-days', '3650',
                    '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name ]
            check_call(['openssl'] + args)
            args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl']
            check_call(['openssl'] + args) 

def test_write_python_directory(self):
        os.mkdir(TESTFN2)
        try:
            with open(os.path.join(TESTFN2, "mod1.py"), "w") as fp:
                fp.write("print(42)\n")

            with open(os.path.join(TESTFN2, "mod2.py"), "w") as fp:
                fp.write("print(42 * 42)\n")

            with open(os.path.join(TESTFN2, "mod2.txt"), "w") as fp:
                fp.write("bla bla bla\n")

            zipfp  = zipfile.PyZipFile(TemporaryFile(), "w")
            zipfp.writepy(TESTFN2)

            names = zipfp.namelist()
            self.assertTrue('mod1.pyc' in names or 'mod1.pyo' in names)
            self.assertTrue('mod2.pyc' in names or 'mod2.pyo' in names)
            self.assertNotIn('mod2.txt', names)

        finally:
            rmtree(TESTFN2) 

def test_write_dir(self):
        dirpath = os.path.join(TESTFN2, "x")
        os.mkdir(dirpath)
        mode = os.stat(dirpath).st_mode & 0xFFFF
        with zipfile.ZipFile(TESTFN, "w") as zipf:
            zipf.write(dirpath)
            zinfo = zipf.filelist[0]
            self.assertTrue(zinfo.filename.endswith("/x/"))
            self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10)
            zipf.write(dirpath, "y")
            zinfo = zipf.filelist[1]
            self.assertTrue(zinfo.filename, "y/")
            self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10)
        with zipfile.ZipFile(TESTFN, "r") as zipf:
            zinfo = zipf.filelist[0]
            self.assertTrue(zinfo.filename.endswith("/x/"))
            self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10)
            zinfo = zipf.filelist[1]
            self.assertTrue(zinfo.filename, "y/")
            self.assertEqual(zinfo.external_attr, (mode << 16) | 0x10)
            target = os.path.join(TESTFN2, "target")
            os.mkdir(target)
            zipf.extractall(target)
            self.assertTrue(os.path.isdir(os.path.join(target, "y")))
            self.assertEqual(len(os.listdir(target)), 2) 

def test_writestr_dir(self):
        os.mkdir(os.path.join(TESTFN2, "x"))
        with zipfile.ZipFile(TESTFN, "w") as zipf:
            zipf.writestr("x/", b'')
            zinfo = zipf.filelist[0]
            self.assertEqual(zinfo.filename, "x/")
            self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10)
        with zipfile.ZipFile(TESTFN, "r") as zipf:
            zinfo = zipf.filelist[0]
            self.assertTrue(zinfo.filename.endswith("x/"))
            self.assertEqual(zinfo.external_attr, (0o40775 << 16) | 0x10)
            target = os.path.join(TESTFN2, "target")
            os.mkdir(target)
            zipf.extractall(target)
            self.assertTrue(os.path.isdir(os.path.join(target, "x")))
            self.assertEqual(os.listdir(target), ["x"]) 

def setUp(self):
        if os.path.exists(self.tagged):
            shutil.rmtree(self.tagged)
        if os.path.exists(self.package_name):
            symlink_support.remove_symlink(self.package_name)
        self.orig_sys_path = sys.path[:]

        # create a sample package; imagine you have a package with a tag and
        #  you want to symbolically link it from its untagged name.
        os.mkdir(self.tagged)
        init_file = os.path.join(self.tagged, '__init__.py')
        open(init_file, 'w').close()
        assert os.path.exists(init_file)

        # now create a symlink to the tagged package
        # sample -> sample-tagged
        symlink_support.symlink(self.tagged, self.package_name)

        assert os.path.isdir(self.package_name)
        assert os.path.isfile(os.path.join(self.package_name, '__init__.py')) 

def _test_pathname(self, path, cmp_path=None, dir=False):
        # Create a tarfile with an empty member named path
        # and compare the stored name with the original.
        foo = os.path.join(TEMPDIR, "foo")
        if not dir:
            open(foo, "w").close()
        else:
            os.mkdir(foo)

        tar = tarfile.open(tmpname, self.mode)
        tar.add(foo, arcname=path)
        tar.close()

        tar = tarfile.open(tmpname, "r")
        t = tar.next()
        tar.close()

        if not dir:
            os.remove(foo)
        else:
            os.rmdir(foo)

        self.assertEqual(t.name, cmp_path or path.replace(os.sep, "/")) 

def save(filename):
    try:
        os.mkdir('saves/')
    except OSError:
        pass
    try:
        data = unicornhathd.get_pixels()
        data = repr(data)
        data = data.replace('array', 'list')
        print(filename, data)
        file = open('saves/' + filename + '.py', 'w')
        file.write("""#!/usr/bin/env python
import unicornhathd
import signal

unicornhathd.rotation(0)

pixels = {}

for x in range(unicornhathd.WIDTH):
    for y in range(unicornhathd.HEIGHT):
        r, g, b = pixels[x][y]
        unicornhathd.set_pixel(x, y, r, g, b)

unicornhathd.show()

print("\\nShowing: {}\\nPress Ctrl+C to exit!")

signal.pause()
""".format(data, filename))
        file.close()
        os.chmod('saves/' + filename + '.py', 0o777 | stat.S_IEXEC)

        return("ok" + str(unicornhathd.get_pixels()))
    except AttributeError:
        print("Unable to save, please update")
        print("unicornhathdhathd library!")
        return("fail") 

def download_diagnostic(data_dir):
    print("Downloading and extracting diagnostic...")
    if not os.path.isdir(os.path.join(data_dir, "diagnostic")):
        os.mkdir(os.path.join(data_dir, "diagnostic"))
    data_file = os.path.join(data_dir, "diagnostic", "diagnostic.tsv")
    urllib.request.urlretrieve(TASK2PATH["diagnostic"], data_file)
    print("\tCompleted!")
    return 

def safeInstall():
	FACTORIOPATH = getFactorioPath()

	try:
		if not os.path.isdir("%s" % (FACTORIOPATH) ):		

			if os.access("%s/.." % (FACTORIOPATH), os.W_OK):
				os.mkdir(FACTORIOPATH, 0o777)
			else:
				subprocess.call(['sudo', 'mkdir', '-p', FACTORIOPATH])
				subprocess.call(['sudo', 'chown', getpass.getuser(), FACTORIOPATH])
							

			os.mkdir(os.path.join(FACTORIOPATH, "saves"))
			os.mkdir(os.path.join(FACTORIOPATH, "config"))
			with open("%s/.bashrc" % (os.path.expanduser("~")), "r+") as bashrc:
				lines = bashrc.read()
				
				if lines.find("eval \"$(_FACTOTUM_COMPLETE=source factotum)\"\n") == -1:
					bashrc.write("eval \"$(_FACTOTUM_COMPLETE=source factotum)\"\n")
					print("You'll want to restart your shell for command autocompletion. Tab is your friend.")
		updateFactorio()
	except IOError as e:
		print("Cannot make %s. Please check permissions. Error %s" % (FACTORIOPATH, e))
		sys.exit(1) 

def _write_json_dir_db(db: PackageTagDb, path: Path, how_to_generate: str):
    with populate_temp_dir_and_rename(path, overwrite=True) as td:
        for package, tag_to_info in db.items():
            os.mkdir(td / package)
            for tag, info in tag_to_info.items():
                with open(td / package / (tag + _JSON), 'w') as outf:
                    outf.write(_with_generated_header(
                        json.dumps(info, sort_keys=True, indent=4) + '\n',
                        how_to_generate,
                    )) 

def make_temp_snapshot(
    repos, out_dir, gpg_key_path, gpg_key_whitelist_dir,
) -> Path:
    'Generates temporary RPM repo snapshots for tests to use as inputs.'
    snapshot_dir = out_dir / 'temp_snapshot_dir'
    os.mkdir(snapshot_dir)

    with temp_repos_steps(repo_change_steps=[repos]) as repos_root:
        snapshot_repos(
            dest=snapshot_dir,
            # Snapshot the 0th step only, since only that is defined
            yum_conf_content=_make_test_yum_dnf_conf(
                'yum', repos_root / '0', gpg_key_path,
            ),
            dnf_conf_content=_make_test_yum_dnf_conf(
                'dnf', repos_root / '0', gpg_key_path,
            ),
            repo_db_ctx=RepoDBContext(
                DBConnectionContext.make(
                    kind='sqlite', db_path=(out_dir / 'db.sqlite3').decode(),
                ),
                SQLDialect.SQLITE3,
            ),
            storage=Storage.make(
                key='test',
                kind='filesystem',
                base_dir=(out_dir / 'storage').decode(),
            ),
            rpm_shard=RpmShard(shard=0, modulo=1),
            gpg_key_whitelist_dir=no_gpg_keys_yet,
            retries=0,  # Nothing here should require retries, it's a bug.
        )

    # Merge the repo snapshot with the storage & RPM DB -- this makes our
    # test snapshot build target look very much like prod snapshots.
    for f in os.listdir(snapshot_dir):
        assert not os.path.exists(out_dir / f), f'Must not overwrite {f}'
        os.rename(snapshot_dir / f, out_dir / f)
    os.rmdir(snapshot_dir) 

def test_list_refcounts(self):
        with tempfile.TemporaryDirectory() as td:
            self.assertEqual({}, dict(sgc.list_refcounts(td)))

            self._touch(td, 'foo:bar')  # No .json
            self._touch(td, 'borf.json')  # No :
            self.assertEqual({}, dict(sgc.list_refcounts(td)))

            banana_json = os.path.join(td, 'ba:nana.json')
            os.mkdir(banana_json)  # Not a file
            with self.assertRaisesRegex(RuntimeError, 'not a regular file'):
                dict(sgc.list_refcounts(td))
            os.rmdir(banana_json)

            self._touch(banana_json)  # This is a real refcount file now
            self.assertEqual({'ba:nana': 1}, dict(sgc.list_refcounts(td)))

            # The linking is pathological, but it doesn't seem worth detecting.
            os.link(banana_json, os.path.join(td, 'ap:ple.json'))
            self.assertEqual(
                {'ba:nana': 2, 'ap:ple': 2}, dict(sgc.list_refcounts(td))
            )

            os.unlink(banana_json)
            self.assertEqual({'ap:ple': 1}, dict(sgc.list_refcounts(td)))

    # Not bothering with a direct test for `parse_args` because (a) it is
    # entirely argparse declarations, and that module has decent validation,
    # (b) we test it indirectly in `test_has_new_subvolume` and others. 

def test_has_new_subvolume(self):

        # Instead of creating a fake namespace, actually parse some args
        def dir_json(wrapper_dir, json):
            return sgc.parse_args([
                '--refcounts-dir', 'fake',
                '--subvolumes-dir', 'fake',
                '--new-subvolume-wrapper-dir', wrapper_dir,
                '--new-subvolume-json', json,
            ])

        self.assertFalse(sgc.has_new_subvolume(dir_json(*[None] * 2)))
        self.assertTrue(sgc.has_new_subvolume(dir_json('x:y', 'z')))

        for bad_example in [('x:y', None), (None, 'z')]:
            with self.assertRaisesRegex(
                RuntimeError, 'pass both .* or pass none',
            ):
                sgc.has_new_subvolume(dir_json(*bad_example))

        for bad_example in [('x/y', 'z'), ('no_colon', 'z')]:
            with self.assertRaisesRegex(
                RuntimeError, 'must contain : but not /',
            ):
                sgc.has_new_subvolume(dir_json(*bad_example))

        with tempfile.TemporaryDirectory() as td:
            os.mkdir(os.path.join(td, 'x:y'))
            with self.assertRaisesRegex(RuntimeError, 'wrapper-dir exists'):
                sgc.has_new_subvolume(sgc.parse_args([
                    '--refcounts-dir', 'fake',
                    '--subvolumes-dir', td,
                    '--new-subvolume-wrapper-dir', 'x:y',
                    '--new-subvolume-json', 'fake',
                ])) 

def __init__(self, path_in_repo=None):
        self.subvols = []
        # The 'tmp' subdirectory simplifies cleanup of leaked temp subvolumes
        volume_tmp_dir = os.path.join(volume_dir(path_in_repo), 'tmp')
        try:
            os.mkdir(volume_tmp_dir)
        except FileExistsError:
            pass
        # Our exit is written with exception-safety in mind, so this
        # `_temp_dir_ctx` **should** get `__exit__`ed when this class does.
        self._temp_dir_ctx = tempfile.TemporaryDirectory(  # noqa: P201
            dir=volume_tmp_dir,
            prefix=self.__class__.__name__ + '_',
        ) 

def test_conflict(self):
        # Future: de-duplicate this with TempSubvolumes, perhaps?
        tmp_parent = os.path.join(volume_dir(), 'tmp')
        try:
            os.mkdir(tmp_parent)
        except FileExistsError:
            pass
        # Removes get built before adds, so a conflict means nothing to remove
        with tempfile.TemporaryDirectory(dir=tmp_parent) as temp_subvol_dir, \
                self.assertRaisesRegex(AssertionError, 'Path does not exist'):
            try:
                # We cannot make this an `image.layer` target, since Buck
                # doesn't (yet) have a nice story for testing targets whose
                # builds are SUPPOSED to fail.
                build_image(parse_args([
                    '--subvolumes-dir', temp_subvol_dir,
                    '--subvolume-rel-path', 'SUBVOL',
                    '--child-layer-target', 'unused',
                    '--child-feature-json', self._resource_path('feature_both'),
                    '--child-dependencies',
                    _test_feature_target('feature_addremove_conflict_add'),
                    self._resource_path('feature_add'),
                    _test_feature_target('feature_addremove_conflict_remove'),
                    self._resource_path('feature_remove'),
                ]))
            finally:
                # Ignore error code in case something broke early in the test
                subprocess.run([
                    'sudo', 'btrfs', 'subvolume', 'delete',
                    os.path.join(temp_subvol_dir, 'SUBVOL'),
                ]) 

def mock_layer_dir_access(test_case, subvolume_path):
    '''
    `SubvolumeOnDisk` does a ton of validation, which makes it hard to
    use it to read or write subvols that are not actual target outputs.

    Instead, this yields a fake layer directory path, and mocks
    `SubvolumeOnDisk.from_json_file` **only** for calls querying the fake
    path.  For those calls, it returns a fake `SubvolumeOnDisk` pointing at
    the supplied `subvolume_path`.
    '''
    sigil_dirname = b'fake-parent-layer'
    orig_from_json_file = svod.SubvolumeOnDisk.from_json_file
    with unittest.mock.patch.object(
        svod.SubvolumeOnDisk, 'from_json_file'
    ) as from_json_file, temp_dir() as td:
        parent_layer_file = td / sigil_dirname / 'layer.json'
        os.mkdir(parent_layer_file.dirname())
        with open(parent_layer_file, 'w') as f:
            f.write('this will never be read')

        def check_call(infile, subvolumes_dir):
            if Path(infile.name).dirname().basename() != sigil_dirname:
                return orig_from_json_file(infile, subvolumes_dir)

            test_case.assertEqual(parent_layer_file, infile.name)
            test_case.assertEqual(_SUBVOLS_DIR, subvolumes_dir)

            class FakeSubvolumeOnDisk:
                def subvolume_path(self):
                    return subvolume_path.decode()

            return FakeSubvolumeOnDisk()

        from_json_file.side_effect = check_call
        yield parent_layer_file.dirname() 

def test_install_file(self):
        with tempfile.NamedTemporaryFile() as tf:
            os.chmod(tf.name, stat.S_IXUSR)
            exe_item = _install_file_item(
                from_target='t', source={'source': tf.name}, dest='d/c',
            )
        ep = _InstallablePath(Path(tf.name), ProvidesFile(path='d/c'), 'a+rx')
        self.assertEqual((ep,), exe_item.paths)
        self.assertEqual(tf.name.encode(), exe_item.source)
        self._check_item(exe_item, {ep.provides}, {require_directory('d')})

        # Checks `image.source(path=...)`
        with temp_dir() as td:
            os.mkdir(td / 'b')
            open(td / 'b/q', 'w').close()
            data_item = _install_file_item(
                from_target='t',
                source={'source': td, 'path': '/b/q'},
                dest='d',
            )
        dp = _InstallablePath(td / 'b/q', ProvidesFile(path='d'), 'a+r')
        self.assertEqual((dp,), data_item.paths)
        self.assertEqual(td / 'b/q', data_item.source)
        self._check_item(data_item, {dp.provides}, {require_directory('/')})

        # NB: We don't need to get coverage for this check on ALL the items
        # because the presence of the ProvidesDoNotAccess items it the real
        # safeguard -- e.g. that's what prevents TarballItem from writing
        # to /meta/ or other protected paths.
        with self.assertRaisesRegex(AssertionError, 'cannot start with meta/'):
            _install_file_item(
                from_target='t', source={'source': 'a/b/c'}, dest='/meta/foo',
            ) 

def test_install_file_command(self):
        with TempSubvolumes(sys.argv[0]) as temp_subvolumes, \
                tempfile.NamedTemporaryFile() as empty_tf:
            subvol = temp_subvolumes.create('tar-sv')
            subvol.run_as_root(['mkdir', subvol.path('d')])

            _install_file_item(
                from_target='t', source={'source': empty_tf.name},
                dest='/d/empty',
            ).build(subvol, DUMMY_LAYER_OPTS)
            self.assertEqual(
                ['(Dir)', {'d': ['(Dir)', {'empty': ['(File m444)']}]}],
                render_subvol(subvol),
            )

            # Fail to write to a nonexistent dir
            with self.assertRaises(subprocess.CalledProcessError):
                _install_file_item(
                    from_target='t', source={'source': empty_tf.name},
                    dest='/no_dir/empty',
                ).build(subvol, DUMMY_LAYER_OPTS)

            # Running a second copy to the same destination. This just
            # overwrites the previous file, because we have a build-time
            # check for this, and a run-time check would add overhead.
            _install_file_item(
                from_target='t', source={'source': empty_tf.name},
                dest='/d/empty',
                # A non-default mode & owner shows that the file was
                # overwritten, and also exercises HasStatOptions.
                mode='u+rw', user_group='12:34',
            ).build(subvol, DUMMY_LAYER_OPTS)
            self.assertEqual(
                ['(Dir)', {'d': ['(Dir)', {'empty': ['(File m600 o12:34)']}]}],
                render_subvol(subvol),
            ) 

def mkdir(self, path, mode=0o777):
        """
        Creates the given path as it would be on the real system.
        """
        os.mkdir(self.full_path(path), mode=mode) 

def _init_child(tmpdir):
    '''
        Initialize new process from multiprocessing module's Pool.

    :param tmpdir: Path of local staging area.
    '''
    signal.signal(signal.SIGINT, signal.SIG_IGN)

    # .................................for each process, create a isolated temp folder
    proc_dir = os.path.join(tmpdir, current_process().name)

    if not os.path.isdir(proc_dir):
        os.mkdir(proc_dir) 

def makedir(pathname):
    if os.path.isdir(pathname):
        return
    dirname = os.path.dirname(pathname)
    if dirname: makedir(dirname)
    os.mkdir(pathname, 0777)

# Write a dictionary to a file in a way that can be read back using
# rval() but is still somewhat readable (i.e. not a single long line).
# Also creates a backup file. 

def mkrealdir(name):
    st = os.stat(name) # Get the mode
    mode = S_IMODE(st[ST_MODE])
    linkto = os.readlink(name)
    files = os.listdir(name)
    os.unlink(name)
    os.mkdir(name, mode)
    os.chmod(name, mode)
    linkto = join(os.pardir, linkto)
    #
    for filename in files:
        if filename not in (os.curdir, os.pardir):
            os.symlink(join(linkto, filename), join(name, filename)) 

def test_refactor_dir(self):
        def check(structure, expected):
            def mock_refactor_file(self, f, *args):
                got.append(f)
            save_func = refactor.RefactoringTool.refactor_file
            refactor.RefactoringTool.refactor_file = mock_refactor_file
            rt = self.rt()
            got = []
            dir = tempfile.mkdtemp(prefix="2to3-test_refactor")
            try:
                os.mkdir(os.path.join(dir, "a_dir"))
                for fn in structure:
                    open(os.path.join(dir, fn), "wb").close()
                rt.refactor_dir(dir)
            finally:
                refactor.RefactoringTool.refactor_file = save_func
                shutil.rmtree(dir)
            self.assertEqual(got,
                             [os.path.join(dir, path) for path in expected])
        check([], [])
        tree = ["nothing",
                "hi.py",
                ".dumb",
                ".after.py",
                "notpy.npy",
                "sappy"]
        expected = ["hi.py"]
        check(tree, expected)
        tree = ["hi.py",
                os.path.join("a_dir", "stuff.py")]
        check(tree, tree) 

def test_simple_run(self):
        pkg_dir, dist = self.create_dist()
        cmd = clean(dist)

        # let's add some elements clean should remove
        dirs = [(d, os.path.join(pkg_dir, d))
                for d in ('build_temp', 'build_lib', 'bdist_base',
                'build_scripts', 'build_base')]

        for name, path in dirs:
            os.mkdir(path)
            setattr(cmd, name, path)
            if name == 'build_base':
                continue
            for f in ('one', 'two', 'three'):
                self.write_file(os.path.join(path, f))

        # let's run the command
        cmd.all = 1
        cmd.ensure_finalized()
        cmd.run()

        # make sure the files where removed
        for name, path in dirs:
            self.assertFalse(os.path.exists(path),
                         '%s was not removed' % path)

        # let's run the command again (should spit warnings but succeed)
        cmd.all = 1
        cmd.ensure_finalized()
        cmd.run() 

def setUp(self):
        # PyPIRCCommandTestCase creates a temp dir already
        # and put it in self.tmp_dir
        super(SDistTestCase, self).setUp()
        # setting up an environment
        self.old_path = os.getcwd()
        os.mkdir(join(self.tmp_dir, 'somecode'))
        os.mkdir(join(self.tmp_dir, 'dist'))
        # a package, and a README
        self.write_file((self.tmp_dir, 'README'), 'xxx')
        self.write_file((self.tmp_dir, 'somecode', '__init__.py'), '#')
        self.write_file((self.tmp_dir, 'setup.py'), SETUP_PY)
        os.chdir(self.tmp_dir) 

def test_prune_file_list(self):
        # this test creates a project with some VCS dirs and an NFS rename
        # file, then launches sdist to check they get pruned on all systems

        # creating VCS directories with some files in them
        os.mkdir(join(self.tmp_dir, 'somecode', '.svn'))
        self.write_file((self.tmp_dir, 'somecode', '.svn', 'ok.py'), 'xxx')

        os.mkdir(join(self.tmp_dir, 'somecode', '.hg'))
        self.write_file((self.tmp_dir, 'somecode', '.hg',
                         'ok'), 'xxx')

        os.mkdir(join(self.tmp_dir, 'somecode', '.git'))
        self.write_file((self.tmp_dir, 'somecode', '.git',
                         'ok'), 'xxx')

        self.write_file((self.tmp_dir, 'somecode', '.nfs0001'), 'xxx')

        # now building a sdist
        dist, cmd = self.get_cmd()

        # zip is available universally
        # (tar might not be installed under win32)
        cmd.formats = ['zip']

        cmd.ensure_finalized()
        cmd.run()

        # now let's check what we have
        dist_folder = join(self.tmp_dir, 'dist')
        files = os.listdir(dist_folder)
        self.assertEqual(files, ['fake-1.0.zip'])

        zip_file = zipfile.ZipFile(join(dist_folder, 'fake-1.0.zip'))
        try:
            content = zip_file.namelist()
        finally:
            zip_file.close()

        # making sure everything has been pruned correctly
        self.assertEqual(len(content), 4) 

def _make_tarball(self, target_name):
        # creating something to tar
        tmpdir = self.mkdtemp()
        self.write_file([tmpdir, 'file1'], 'xxx')
        self.write_file([tmpdir, 'file2'], 'xxx')
        os.mkdir(os.path.join(tmpdir, 'sub'))
        self.write_file([tmpdir, 'sub', 'file3'], 'xxx')

        tmpdir2 = self.mkdtemp()
        unittest.skipUnless(splitdrive(tmpdir)[0] == splitdrive(tmpdir2)[0],
                            "source and target should be on same drive")

        base_name = os.path.join(tmpdir2, target_name)

        # working with relative paths to avoid tar warnings
        old_dir = os.getcwd()
        os.chdir(tmpdir)
        try:
            make_tarball(splitdrive(base_name)[1], '.')
        finally:
            os.chdir(old_dir)

        # check if the compressed tarball was created
        tarball = base_name + '.tar.gz'
        self.assertTrue(os.path.exists(tarball))

        # trying an uncompressed one
        base_name = os.path.join(tmpdir2, target_name)
        old_dir = os.getcwd()
        os.chdir(tmpdir)
        try:
            make_tarball(splitdrive(base_name)[1], '.', compress=None)
        finally:
            os.chdir(old_dir)
        tarball = base_name + '.tar'
        self.assertTrue(os.path.exists(tarball)) 

def _create_files(self):
        # creating something to tar
        tmpdir = self.mkdtemp()
        dist = os.path.join(tmpdir, 'dist')
        os.mkdir(dist)
        self.write_file([dist, 'file1'], 'xxx')
        self.write_file([dist, 'file2'], 'xxx')
        os.mkdir(os.path.join(dist, 'sub'))
        self.write_file([dist, 'sub', 'file3'], 'xxx')
        os.mkdir(os.path.join(dist, 'sub2'))
        tmpdir2 = self.mkdtemp()
        base_name = os.path.join(tmpdir2, 'archive')
        return tmpdir, tmpdir2, base_name 

def test_empty_package_dir(self):
        # See SF 1668596/1720897.
        cwd = os.getcwd()

        # create the distribution files.
        sources = self.mkdtemp()
        open(os.path.join(sources, "__init__.py"), "w").close()

        testdir = os.path.join(sources, "doc")
        os.mkdir(testdir)
        open(os.path.join(testdir, "testfile"), "w").close()

        os.chdir(sources)
        old_stdout = sys.stdout
        sys.stdout = StringIO.StringIO()

        try:
            dist = Distribution({"packages": ["pkg"],
                                 "package_dir": {"pkg": ""},
                                 "package_data": {"pkg": ["doc/*"]}})
            # script_name need not exist, it just need to be initialized
            dist.script_name = os.path.join(sources, "setup.py")
            dist.script_args = ["build"]
            dist.parse_command_line()

            try:
                dist.run_commands()
            except DistutilsFileError:
                self.fail("failed package_data test when package_dir is ''")
        finally:
            # Restore state.
            os.chdir(cwd)
            sys.stdout = old_stdout 

def test_dir_in_package_data(self):
        """
        A directory in package_data should not be added to the filelist.
        """
        # See bug 19286
        sources = self.mkdtemp()
        pkg_dir = os.path.join(sources, "pkg")

        os.mkdir(pkg_dir)
        open(os.path.join(pkg_dir, "__init__.py"), "w").close()

        docdir = os.path.join(pkg_dir, "doc")
        os.mkdir(docdir)
        open(os.path.join(docdir, "testfile"), "w").close()

        # create the directory that could be incorrectly detected as a file
        os.mkdir(os.path.join(docdir, 'otherdir'))

        os.chdir(sources)
        dist = Distribution({"packages": ["pkg"],
                             "package_data": {"pkg": ["doc/*"]}})
        # script_name need not exist, it just need to be initialized
        dist.script_name = os.path.join(sources, "setup.py")
        dist.script_args = ["build"]
        dist.parse_command_line()

        try:
            dist.run_commands()
        except DistutilsFileError:
            self.fail("failed package_data when data dir includes a dir") 

def test_quiet(self):
        # let's create a package
        tmp_dir = self.mkdtemp()
        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
        pkg_dir = os.path.join(tmp_dir, 'foo')
        os.mkdir(pkg_dir)
        self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
        self.write_file((pkg_dir, 'foo.py'), '#')
        self.write_file((pkg_dir, 'MANIFEST.in'), 'include foo.py')
        self.write_file((pkg_dir, 'README'), '')

        dist = Distribution({'name': 'foo', 'version': '0.1',
                             'py_modules': ['foo'],
                             'url': 'xxx', 'author': 'xxx',
                             'author_email': 'xxx'})
        dist.script_name = 'setup.py'
        os.chdir(pkg_dir)

        sys.argv = ['setup.py']
        cmd = bdist_rpm(dist)
        cmd.fix_python = True

        # running in quiet mode
        cmd.quiet = 1
        cmd.ensure_finalized()
        cmd.run()

        dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
        self.assertIn('foo-0.1-1.noarch.rpm', dist_created)

        # bug #2945: upload ignores bdist_rpm files
        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files)
        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files)

    # XXX I am unable yet to make this test work without
    # spurious sdtout/stderr output under Mac OS X 

def test_no_optimize_flag(self):
        # let's create a package that brakes bdist_rpm
        tmp_dir = self.mkdtemp()
        os.environ['HOME'] = tmp_dir   # to confine dir '.rpmdb' creation
        pkg_dir = os.path.join(tmp_dir, 'foo')
        os.mkdir(pkg_dir)
        self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
        self.write_file((pkg_dir, 'foo.py'), '#')
        self.write_file((pkg_dir, 'MANIFEST.in'), 'include foo.py')
        self.write_file((pkg_dir, 'README'), '')

        dist = Distribution({'name': 'foo', 'version': '0.1',
                             'py_modules': ['foo'],
                             'url': 'xxx', 'author': 'xxx',
                             'author_email': 'xxx'})
        dist.script_name = 'setup.py'
        os.chdir(pkg_dir)

        sys.argv = ['setup.py']
        cmd = bdist_rpm(dist)
        cmd.fix_python = True

        cmd.quiet = 1
        cmd.ensure_finalized()
        cmd.run()

        dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
        self.assertIn('foo-0.1-1.noarch.rpm', dist_created)

        # bug #2945: upload ignores bdist_rpm files
        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.src.rpm'), dist.dist_files)
        self.assertIn(('bdist_rpm', 'any', 'dist/foo-0.1-1.noarch.rpm'), dist.dist_files)

        os.remove(os.path.join(pkg_dir, 'dist', 'foo-0.1-1.noarch.rpm')) 

def test_run(self):
        pkg_dir, dist = self.create_dist()
        cmd = build_clib(dist)

        foo_c = os.path.join(pkg_dir, 'foo.c')
        self.write_file(foo_c, 'int main(void) { return 1;}\n')
        cmd.libraries = [('foo', {'sources': [foo_c]})]

        build_temp = os.path.join(pkg_dir, 'build')
        os.mkdir(build_temp)
        cmd.build_temp = build_temp
        cmd.build_clib = build_temp

        # before we run the command, we want to make sure
        # all commands are present on the system
        # by creating a compiler and checking its executables
        from distutils.ccompiler import new_compiler
        from distutils.sysconfig import customize_compiler

        compiler = new_compiler()
        customize_compiler(compiler)
        for ccmd in compiler.executables.values():
            if ccmd is None:
                continue
            if find_executable(ccmd[0]) is None:
                self.skipTest('The %r command is not found' % ccmd[0])

        # this should work
        cmd.run()

        # let's check the result
        self.assertIn('libfoo.a', os.listdir(build_temp)) 

def makefolder(self, name):
        """Create a new folder (or raise os.error if it cannot be created)."""
        protect = pickline(self.profile, 'Folder-Protect')
        if protect and isnumeric(protect):
            mode = int(protect, 8)
        else:
            mode = FOLDER_PROTECT
        os.mkdir(os.path.join(self.getpath(), name), mode) 

def makedir(self, tarinfo, targetpath):
        """Make a directory called targetpath.
        """
        try:
            # Use a safe mode for the directory, the real mode is set
            # later in _extract_member().
            os.mkdir(targetpath, 0700)
        except EnvironmentError, e:
            if e.errno != errno.EEXIST:
                raise 

def test_directory(self):
        os.mkdir(TESTFN)
        os.chmod(TESTFN, 0o700)
        st_mode = self.get_mode()
        self.assertS_IS("DIR", st_mode) 

def make_pkg(pkg_dir):
    os.mkdir(pkg_dir)
    make_script(pkg_dir, '__init__', '') 

def setUp(self):
        longname = 'areallylongpackageandmodulenametotestreprtruncation'
        self.pkgname = os.path.join(longname)
        self.subpkgname = os.path.join(longname, longname)
        # Make the package and subpackage
        shutil.rmtree(self.pkgname, ignore_errors=True)
        os.mkdir(self.pkgname)
        touch(os.path.join(self.pkgname, '__init__'+os.extsep+'py'))
        shutil.rmtree(self.subpkgname, ignore_errors=True)
        os.mkdir(self.subpkgname)
        touch(os.path.join(self.subpkgname, '__init__'+os.extsep+'py'))
        # Remember where we are
        self.here = os.getcwd()
        sys.path.insert(0, self.here) 

def setUp(self):
        self.test_dir = os.mkdir(TESTFN)
        self.addCleanup(rmtree, TESTFN) 

def test_apropos_with_unreadable_dir(self):
        # Issue 7367 - pydoc -k failed when unreadable dir on path
        self.unreadable_dir = os.path.join(TESTFN, "unreadable")
        os.mkdir(self.unreadable_dir, 0)
        self.addCleanup(os.rmdir, self.unreadable_dir)
        # Note, on Windows the directory appears to be still
        #   readable so this is not really testing the issue there
        with self.restrict_walk_packages(path=[TESTFN]):
            with captured_stdout() as out:
                with captured_stderr() as err:
                    pydoc.apropos('SOMEKEY')
        # No result, no error
        self.assertEqual(out.getvalue(), '')
        self.assertEqual(err.getvalue(), '') 

def test_realpath_repeated_indirect_symlinks(self):
            # Issue #6975.
            try:
                os.mkdir(ABSTFN)
                os.symlink('../' + basename(ABSTFN), ABSTFN + '/self')
                os.symlink('self/self/self', ABSTFN + '/link')
                self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN)
            finally:
                test_support.unlink(ABSTFN + '/self')
                test_support.unlink(ABSTFN + '/link')
                safe_rmdir(ABSTFN) 

def test_realpath_resolve_before_normalizing(self):
            # Bug #990669: Symbolic links should be resolved before we
            # normalize the path. E.g.: if we have directories 'a', 'k' and 'y'
            # in the following hierarchy:
            # a/k/y
            #
            # and a symbolic link 'link-y' pointing to 'y' in directory 'a',
            # then realpath("link-y/..") should return 'k', not 'a'.
            try:
                old_path = abspath('.')
                os.mkdir(ABSTFN)
                os.mkdir(ABSTFN + "/k")
                os.mkdir(ABSTFN + "/k/y")
                os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y")

                # Absolute path.
                self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k")
                # Relative path.
                os.chdir(dirname(ABSTFN))
                self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."),
                                 ABSTFN + "/k")
            finally:
                os.chdir(old_path)
                test_support.unlink(ABSTFN + "/link-y")
                safe_rmdir(ABSTFN + "/k/y")
                safe_rmdir(ABSTFN + "/k")
                safe_rmdir(ABSTFN) 

def setUp(self):
        os.mkdir(TESTFN2) 

def test_bug_6050(self):
        # Extraction should succeed if directories already exist
        os.mkdir(os.path.join(TESTFN2, "a"))
        self.test_extract_dir() 

def test_bug7732(self):
        source = TESTFN + '.py'
        os.mkdir(source)
        try:
            self.assertRaises((ImportError, IOError),
                              imp.find_module, TESTFN, ["."])
        finally:
            os.rmdir(source) 

def setUp(self):
        self.sys_path = sys.path[:]
        self.orig_module = sys.modules.pop(self.module_name, None)
        os.mkdir(self.dir_name)
        with open(self.file_name, "w") as f:
            f.write(self.module_source)
        sys.path.insert(0, self.dir_name) 

def setUp(self):
        os.mkdir(self.path)
        self.syspath = sys.path[:] 

def test_directory_size(self):
        path = os.path.join(TEMPDIR, "directory")
        os.mkdir(path)
        try:
            tar = tarfile.open(tmpname, self.mode)
            tarinfo = tar.gettarinfo(path)
            self.assertEqual(tarinfo.size, 0)
        finally:
            os.rmdir(path) 

def test_filter(self):
        tempdir = os.path.join(TEMPDIR, "filter")
        os.mkdir(tempdir)
        try:
            for name in ("foo", "bar", "baz"):
                name = os.path.join(tempdir, name)
                open(name, "wb").close()

            def filter(tarinfo):
                if os.path.basename(tarinfo.name) == "bar":
                    return
                tarinfo.uid = 123
                tarinfo.uname = "foo"
                return tarinfo

            tar = tarfile.open(tmpname, self.mode, encoding="iso8859-1")
            tar.add(tempdir, arcname="empty_dir", filter=filter)
            tar.close()

            tar = tarfile.open(tmpname, "r")
            for tarinfo in tar:
                self.assertEqual(tarinfo.uid, 123)
                self.assertEqual(tarinfo.uname, "foo")
            self.assertEqual(len(tar.getmembers()), 3)
            tar.close()
        finally:
            shutil.rmtree(tempdir)

    # Guarantee that stored pathnames are not modified. Don't
    # remove ./ or ../ or double slashes. Still make absolute
    # pathnames relative.
    # For details see bug #6054. 

def test_extractall_symlinks(self):
        # Test if extractall works properly when tarfile contains symlinks
        tempdir = os.path.join(TEMPDIR, "testsymlinks")
        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
        os.mkdir(tempdir)
        try:
            source_file = os.path.join(tempdir,'source')
            target_file = os.path.join(tempdir,'symlink')
            with open(source_file,'w') as f:
                f.write('something\n')
            os.symlink(source_file, target_file)
            tar = tarfile.open(temparchive,'w')
            tar.add(source_file, arcname=os.path.basename(source_file))
            tar.add(target_file, arcname=os.path.basename(target_file))
            tar.close()
            # Let's extract it to the location which contains the symlink
            tar = tarfile.open(temparchive,'r')
            # this should not raise OSError: [Errno 17] File exists
            try:
                tar.extractall(path=tempdir)
            except OSError:
                self.fail("extractall failed with symlinked files")
            finally:
                tar.close()
        finally:
            os.unlink(temparchive)
            shutil.rmtree(tempdir) 

def test_extractall_broken_symlinks(self):
        # Test if extractall works properly when tarfile contains broken
        # symlinks
        tempdir = os.path.join(TEMPDIR, "testsymlinks")
        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
        os.mkdir(tempdir)
        try:
            source_file = os.path.join(tempdir,'source')
            target_file = os.path.join(tempdir,'symlink')
            with open(source_file,'w') as f:
                f.write('something\n')
            os.symlink(source_file, target_file)
            tar = tarfile.open(temparchive,'w')
            tar.add(target_file, arcname=os.path.basename(target_file))
            tar.close()
            # remove the real file
            os.unlink(source_file)
            # Let's extract it to the location which contains the symlink
            tar = tarfile.open(temparchive,'r')
            # this should not raise OSError: [Errno 17] File exists
            try:
                tar.extractall(path=tempdir)
            except OSError:
                self.fail("extractall failed with broken symlinked files")
            finally:
                tar.close()
        finally:
            os.unlink(temparchive)
            shutil.rmtree(tempdir) 

def test_extractall_hardlinks(self):
        # Test if extractall works properly when tarfile contains symlinks
        tempdir = os.path.join(TEMPDIR, "testsymlinks")
        temparchive = os.path.join(TEMPDIR, "testsymlinks.tar")
        os.mkdir(tempdir)
        try:
            source_file = os.path.join(tempdir,'source')
            target_file = os.path.join(tempdir,'symlink')
            with open(source_file,'w') as f:
                f.write('something\n')
            os.link(source_file, target_file)
            tar = tarfile.open(temparchive,'w')
            tar.add(source_file, arcname=os.path.basename(source_file))
            tar.add(target_file, arcname=os.path.basename(target_file))
            tar.close()
            # Let's extract it to the location which contains the symlink
            tar = tarfile.open(temparchive,'r')
            # this should not raise OSError: [Errno 17] File exists
            try:
                tar.extractall(path=tempdir)
            except OSError:
                self.fail("extractall failed with linked files")
            finally:
                tar.close()
        finally:
            os.unlink(temparchive)
            shutil.rmtree(tempdir) 

def mkdirs(fname):
    if os.path.exists(fname) or fname == '':
        return
    base, file = os.path.split(fname)
    mkdirs(base)
    os.mkdir(fname) 

def format_mrpc(data_dir, path_to_data):
    print("Processing MRPC...")
    mrpc_dir = os.path.join(data_dir, "MRPC")
    if not os.path.isdir(mrpc_dir):
        os.mkdir(mrpc_dir)
    if path_to_data:
        mrpc_train_file = os.path.join(path_to_data, "msr_paraphrase_train.txt")
        mrpc_test_file = os.path.join(path_to_data, "msr_paraphrase_test.txt")
    else:
        mrpc_train_file = os.path.join(mrpc_dir, "msr_paraphrase_train.txt")
        mrpc_test_file = os.path.join(mrpc_dir, "msr_paraphrase_test.txt")
        urllib.request.urlretrieve(MRPC_TRAIN, mrpc_train_file)
        urllib.request.urlretrieve(MRPC_TEST, mrpc_test_file)
    assert os.path.isfile(mrpc_train_file), "Train data not found at %s" % mrpc_train_file
    assert os.path.isfile(mrpc_test_file), "Test data not found at %s" % mrpc_test_file
    urllib.request.urlretrieve(TASK2PATH["MRPC"], os.path.join(mrpc_dir, "dev_ids.tsv"))

    dev_ids = []
    with open(os.path.join(mrpc_dir, "dev_ids.tsv")) as ids_fh:
        for row in ids_fh:
            dev_ids.append(row.strip().split('\t'))

    with open(mrpc_train_file, encoding="utf8") as data_fh, \
            open(os.path.join(mrpc_dir, "train.tsv"), 'w') as train_fh, \
            open(os.path.join(mrpc_dir, "dev.tsv"), 'w') as dev_fh:
        header = data_fh.readline()
        train_fh.write(header)
        dev_fh.write(header)
        for row in data_fh:
            label, id1, id2, s1, s2 = row.strip().split('\t')
            if [id1, id2] in dev_ids:
                dev_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2))
            else:
                train_fh.write("%s\t%s\t%s\t%s\t%s\n" % (label, id1, id2, s1, s2))

    with open(mrpc_test_file) as data_fh, \
            open(os.path.join(mrpc_dir, "test.tsv"), 'w') as test_fh:
        header = data_fh.readline()
        test_fh.write("index\t#1 ID\t#2 ID\t#1 String\t#2 String\n")
        for idx, row in enumerate(data_fh):
            label, id1, id2, s1, s2 = row.strip().split('\t')
            test_fh.write("%d\t%s\t%s\t%s\t%s\n" % (idx, id1, id2, s1, s2))
    print("\tCompleted!") 

def snapshot_repos(
    dest: Path, *,
    yum_conf_content: str,
    dnf_conf_content: str,
    repo_db_ctx: RepoDBContext,
    storage: Storage,
    rpm_shard: RpmShard,
    gpg_key_whitelist_dir: str,
    retries: int,
):
    declared_sizer = RepoSizer()
    saved_sizer = RepoSizer()
    repos = _write_confs_get_repos(dest, yum_conf_content, dnf_conf_content)
    os.mkdir(dest / 'repos')
    with RepoSnapshot.add_sqlite_to_storage(storage, dest) as db:
        # Randomize the order to reduce contention from concurrent writers
        for repo in shuffled(repos):
            log.info(f'Downloading repo {repo.name} from {repo.base_url}')
            with populate_temp_dir_and_rename(
                dest / 'repos' / repo.name, overwrite=True
            ) as td:
                # This is outside the retry_fn not to mask transient
                # verification failures.  I don't expect many infra failures.
                snapshot_gpg_keys(
                    key_urls=repo.gpg_key_urls,
                    whitelist_dir=gpg_key_whitelist_dir,
                    snapshot_dir=td,
                )
                retry_fn(
                    lambda: RepoDownloader(
                        repo.name, repo.base_url, repo_db_ctx, storage
                    ).download(
                        rpm_shard=rpm_shard, visitors=[declared_sizer]
                    ),
                    delays=[0] * retries,
                    what=f'Download failed: {repo.name} from {repo.base_url}',
                ).visit(saved_sizer).to_sqlite(repo.name, db)

    log.info(declared_sizer.get_report(
        f'According to their repodata, these {len(repos)} repos weigh'
    ))
    log.info(saved_sizer.get_report(f'This {rpm_shard} snapshot weighs')) 

def test_snapshot(self):
        with temp_repos.temp_repos_steps(repo_change_steps=[{
            'dog': temp_repos.SAMPLE_STEPS[0]['dog'],
        }]) as repos_root, temp_dir() as td:
            with open(td / 'fake_gpg_key', 'w'):
                pass

            whitelist_dir = td / 'gpg_whitelist'
            os.mkdir(whitelist_dir)
            shutil.copy(td / 'fake_gpg_key', whitelist_dir)

            storage_dict = {
                'key': 'test',
                'kind': 'filesystem',
                'base_dir': (td / 'storage').decode(),
            }
            snapshot_repo([
                '--repo-name', 'dog',
                '--repo-url', (repos_root / '0/dog').file_url(),
                '--gpg-key-whitelist-dir', whitelist_dir.decode(),
                '--gpg-url', (td / 'fake_gpg_key').file_url(),
                '--snapshot-dir', (td / 'snap').decode(),
                '--storage', json.dumps(storage_dict),
                '--db', json.dumps({
                    'kind': 'sqlite',
                    'db_path': (td / 'db.sqlite3').decode(),
                }),
            ])
            # This test simply checks the overall integration, so we don't
            # bother looking inside the DB or Storage, or inspecting the
            # details of the snapshot -- those should all be covered by
            # lower-level tests.
            with sqlite3.connect(RepoSnapshot.fetch_sqlite_from_storage(
                Storage.make(**storage_dict),
                td / 'snap',
                td / 'snapshot.sql3',
            )) as db:
                self.assertEqual({
                    'dog-pkgs/rpm-test-carrot-2-rc0.x86_64.rpm',
                    'dog-pkgs/rpm-test-mice-0.1-a.x86_64.rpm',
                    'dog-pkgs/rpm-test-milk-1.41-42.x86_64.rpm',
                }, {
                    path for path, in db.execute(
                        'SELECT "path" FROM "rpm";'
                    ).fetchall()
                }) 

def test_install_file_command_recursive(self):
        with TempSubvolumes(sys.argv[0]) as temp_subvolumes:
            subvol = temp_subvolumes.create('tar-sv')
            subvol.run_as_root(['mkdir', subvol.path('d')])

            with temp_dir() as td:
                with open(td / 'data.txt', 'w') as df:
                    print('Hello', file=df)
                os.mkdir(td / 'subdir')
                with open(td / 'subdir/exe.sh', 'w') as ef:
                    print('#!/bin/sh\necho "Hello"', file=ef)
                os.chmod(td / 'subdir/exe.sh', 0o100)

                dir_item = _install_file_item(
                    from_target='t', source={'source': td}, dest='/d/a',
                )

                ps = [
                    _InstallablePath(
                        td,
                        ProvidesDirectory(path='d/a'),
                        'u+rwx,og+rx',
                    ),
                    _InstallablePath(
                        td / 'data.txt',
                        ProvidesFile(path='d/a/data.txt'),
                        'a+r',
                    ),
                    _InstallablePath(
                        td / 'subdir',
                        ProvidesDirectory(path='d/a/subdir'),
                        'u+rwx,og+rx',
                    ),
                    _InstallablePath(
                        td / 'subdir/exe.sh',
                        ProvidesFile(path='d/a/subdir/exe.sh'),
                        'a+rx',
                    ),
                ]
                self.assertEqual(sorted(ps), sorted(dir_item.paths))
                self.assertEqual(td, dir_item.source)
                self._check_item(
                    dir_item, {p.provides for p in ps}, {require_directory('d')}
                )

                # This implicitly checks that `a` precedes its contents.
                dir_item.build(subvol, DUMMY_LAYER_OPTS)

            self.assertEqual(
                ['(Dir)', {'d': ['(Dir)', {'a': ['(Dir)', {
                    'data.txt': ['(File m444 d6)'],
                    'subdir': ['(Dir)', {
                        'exe.sh': ['(File m555 d23)'],
                    }],
                }]}]}],
                render_subvol(subvol),
            ) 

def test_simple_built(self):

        # let's create a simple package
        tmp_dir = self.mkdtemp()
        pkg_dir = os.path.join(tmp_dir, 'foo')
        os.mkdir(pkg_dir)
        self.write_file((pkg_dir, 'setup.py'), SETUP_PY)
        self.write_file((pkg_dir, 'foo.py'), '#')
        self.write_file((pkg_dir, 'MANIFEST.in'), 'include foo.py')
        self.write_file((pkg_dir, 'README'), '')

        dist = Distribution({'name': 'foo', 'version': '0.1',
                             'py_modules': ['foo'],
                             'url': 'xxx', 'author': 'xxx',
                             'author_email': 'xxx'})
        dist.script_name = 'setup.py'
        os.chdir(pkg_dir)

        sys.argv = ['setup.py']
        cmd = bdist_dumb(dist)

        # so the output is the same no matter
        # what is the platform
        cmd.format = 'zip'

        cmd.ensure_finalized()
        cmd.run()

        # see what we have
        dist_created = os.listdir(os.path.join(pkg_dir, 'dist'))
        base = "%s.%s.zip" % (dist.get_fullname(), cmd.plat_name)
        if os.name == 'os2':
            base = base.replace(':', '-')

        self.assertEqual(dist_created, [base])

        # now let's check what we have in the zip file
        fp = zipfile.ZipFile(os.path.join('dist', base))
        try:
            contents = fp.namelist()
        finally:
            fp.close()

        contents = sorted(os.path.basename(fn) for fn in contents)
        wanted = ['foo-0.1-py%s.%s.egg-info' % sys.version_info[:2], 'foo.py']
        if not sys.dont_write_bytecode:
            wanted.append('foo.pyc')
        self.assertEqual(contents, sorted(wanted)) 

def test02_WithSource(self):
        """
        A Recno file that is given a "backing source file" is essentially a
        simple ASCII file.  Normally each record is delimited by \n and so is
        just a line in the file, but you can set a different record delimiter
        if needed.
        """
        homeDir = get_new_environment_path()
        self.homeDir = homeDir
        source = os.path.join(homeDir, 'test_recno.txt')
        if not os.path.isdir(homeDir):
            os.mkdir(homeDir)
        f = open(source, 'w') # create the file
        f.close()

        d = db.DB()
        # This is the default value, just checking if both int
        d.set_re_delim(0x0A)
        d.set_re_delim('\n')  # and char can be used...
        d.set_re_source(source)
        d.open(self.filename, db.DB_RECNO, db.DB_CREATE)

        data = "The quick brown fox jumped over the lazy dog".split()
        for datum in data:
            d.append(datum)
        d.sync()
        d.close()

        # get the text from the backing source
        f = open(source, 'r')
        text = f.read()
        f.close()
        text = text.strip()
        if verbose:
            print text
            print data
            print text.split('\n')

        self.assertEqual(text.split('\n'), data)

        # open as a DB again
        d = db.DB()
        d.set_re_source(source)
        d.open(self.filename, db.DB_RECNO)

        d[3] = 'reddish-brown'
        d[8] = 'comatose'

        d.sync()
        d.close()

        f = open(source, 'r')
        text = f.read()
        f.close()
        text = text.strip()
        if verbose:
            print text
            print text.split('\n')

        self.assertEqual(text.split('\n'),
           "The quick reddish-brown fox jumped over the comatose dog".split()) 

def _extract_member(self, member, targetpath, pwd):
        """Extract the ZipInfo object 'member' to a physical
           file on the path targetpath.
        """
        # build the destination pathname, replacing
        # forward slashes to platform specific separators.
        arcname = member.filename.replace('/', os.path.sep)

        if os.path.altsep:
            arcname = arcname.replace(os.path.altsep, os.path.sep)
        # interpret absolute pathname as relative, remove drive letter or
        # UNC path, redundant separators, "." and ".." components.
        arcname = os.path.splitdrive(arcname)[1]
        arcname = os.path.sep.join(x for x in arcname.split(os.path.sep)
                    if x not in ('', os.path.curdir, os.path.pardir))
        if os.path.sep == '\\':
            # filter illegal characters on Windows
            illegal = ':<>|"?*'
            if isinstance(arcname, unicode):
                table = {ord(c): ord('_') for c in illegal}
            else:
                table = string.maketrans(illegal, '_' * len(illegal))
            arcname = arcname.translate(table)
            # remove trailing dots
            arcname = (x.rstrip('.') for x in arcname.split(os.path.sep))
            arcname = os.path.sep.join(x for x in arcname if x)

        targetpath = os.path.join(targetpath, arcname)
        targetpath = os.path.normpath(targetpath)

        # Create all upper directories if necessary.
        upperdirs = os.path.dirname(targetpath)
        if upperdirs and not os.path.exists(upperdirs):
            os.makedirs(upperdirs)

        if member.filename[-1] == '/':
            if not os.path.isdir(targetpath):
                os.mkdir(targetpath)
            return targetpath

        with self.open(member, pwd=pwd) as source, \
             file(targetpath, "wb") as target:
            shutil.copyfileobj(source, target)

        return targetpath