Python shutil.copytree() Examples

The following are code examples for showing how to use shutil.copytree(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_copytree_named_pipe(self):
        os.mkdir(TESTFN)
        try:
            subdir = os.path.join(TESTFN, "subdir")
            os.mkdir(subdir)
            pipe = os.path.join(subdir, "mypipe")
            os.mkfifo(pipe)
            try:
                shutil.copytree(TESTFN, TESTFN2)
            except shutil.Error as e:
                errors = e.args[0]
                self.assertEqual(len(errors), 1)
                src, dst, error_msg = errors[0]
                self.assertEqual("`%s` is a named pipe" % pipe, error_msg)
            else:
                self.fail("shutil.Error should have been raised")
        finally:
            shutil.rmtree(TESTFN, ignore_errors=True)
            shutil.rmtree(TESTFN2, ignore_errors=True) 
Example 2
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: user.py    MIT License 6 votes vote down vote up
def create_homedir(self, path):
        if not os.path.exists(path):
            if self.skeleton is not None:
                skeleton = self.skeleton
            else:
                skeleton = '/etc/skel'

            if os.path.exists(skeleton):
                try:
                    shutil.copytree(skeleton, path, symlinks=True)
                except OSError as e:
                    self.module.exit_json(failed=True, msg="%s" % to_native(e))
        else:
            try:
                os.makedirs(path)
            except OSError as e:
                self.module.exit_json(failed=True, msg="%s" % to_native(e)) 
Example 3
Project: pytorch_NER_BiLSTM_CNN_CRF   Author: bamtercelboo   File: mainHelp.py    Apache License 2.0 6 votes vote down vote up
def save_dictionary(config):
    """
    :param config: config
    :return:
    """
    if config.save_dict is True:
        if os.path.exists(config.dict_directory):
            shutil.rmtree(config.dict_directory)
        if not os.path.isdir(config.dict_directory):
            os.makedirs(config.dict_directory)

        config.word_dict_path = "/".join([config.dict_directory, config.word_dict])
        config.label_dict_path = "/".join([config.dict_directory, config.label_dict])
        print("word_dict_path : {}".format(config.word_dict_path))
        print("label_dict_path : {}".format(config.label_dict_path))
        save_dict2file(config.create_alphabet.word_alphabet.words2id, config.word_dict_path)
        save_dict2file(config.create_alphabet.label_alphabet.words2id, config.label_dict_path)
        # copy to mulu
        print("copy dictionary to {}".format(config.save_dir))
        shutil.copytree(config.dict_directory, "/".join([config.save_dir, config.dict_directory]))


# load data / create alphabet / create iterator 
Example 4
Project: fine-lm   Author: akzaidi   File: cloud_mlengine.py    MIT License 6 votes vote down vote up
def tar_and_copy_usr_dir(usr_dir, train_dir):
  """Package, tar, and copy usr_dir to GCS train_dir."""
  tf.logging.info("Tarring and pushing t2t_usr_dir.")
  usr_dir = os.path.abspath(os.path.expanduser(usr_dir))
  # Copy usr dir to a temp location
  top_dir = os.path.join(tempfile.gettempdir(), "t2t_usr_container")
  tmp_usr_dir = os.path.join(top_dir, usr_dir_lib.INTERNAL_USR_DIR_PACKAGE)
  shutil.rmtree(top_dir, ignore_errors=True)
  shutil.copytree(usr_dir, tmp_usr_dir)
  # Insert setup.py if one does not exist
  top_setup_fname = os.path.join(top_dir, "setup.py")
  setup_file_str = get_setup_file(
      name="DummyUsrDirPackage",
      packages=get_requirements(usr_dir)
  )
  with tf.gfile.Open(top_setup_fname, "w") as f:
    f.write(setup_file_str)
  usr_tar = _tar_and_copy(top_dir, train_dir)
  return usr_tar 
Example 5
Project: relay-bench   Author: uwsampl   File: gen_webpage.py    Apache License 2.0 6 votes vote down vote up
def set_up_out_dir(info, out_dir):
    idemp_mkdir(out_dir)

    web_graph_dir = os.path.join(out_dir, 'graph')
    web_data_dir = os.path.join(out_dir, 'data')
    shutil.rmtree(web_graph_dir, ignore_errors=True)
    shutil.rmtree(web_data_dir, ignore_errors=True)
    shutil.copytree(info.exp_graphs, web_graph_dir)
    if score_successful(info):
        score_graphs = os.path.join(info.subsys_output_dir('score'), 'graphs')
        for subdir in os.listdir(score_graphs):
            full_path = os.path.join(score_graphs, subdir)
            if not os.path.isdir(full_path):
                continue
            shutil.copytree(full_path, os.path.join(web_graph_dir, subdir))
    prepare_exp_data_pages(info, web_data_dir) 
Example 6
Project: calmjs   Author: calmjs   File: toolchain.py    GNU General Public License v2.0 6 votes vote down vote up
def compile_bundle_entry(self, spec, entry):
        """
        Handler for each entry for the bundle method of the compile
        process.  This copies the source file or directory into the
        build directory.
        """

        modname, source, target, modpath = entry
        bundled_modpath = {modname: modpath}
        bundled_target = {modname: target}
        export_module_name = []
        if isfile(source):
            export_module_name.append(modname)
            copy_target = join(spec[BUILD_DIR], target)
            if not exists(dirname(copy_target)):
                makedirs(dirname(copy_target))
            shutil.copy(source, copy_target)
        elif isdir(source):
            copy_target = join(spec[BUILD_DIR], modname)
            shutil.copytree(source, copy_target)

        return bundled_modpath, bundled_target, export_module_name 
Example 7
Project: glazier   Author: google   File: file_system.py    Apache License 2.0 6 votes vote down vote up
def Run(self):
    remove_existing = False
    try:
      src = self._args[0]
      dst = self._args[1]
      if len(self._args) > 2:
        remove_existing = self._args[2]
    except IndexError:
      raise ActionError('Unable to determine source and destination from %s.' %
                        str(self._args))
    try:
      if os.path.exists(dst) and remove_existing:
        logging.info('Deleting existing destination: %s', dst)
        shutil.rmtree(dst)
    except (shutil.Error, OSError) as e:
      raise ActionError('Unable to delete existing destination folder %s: %s' %
                        (dst, str(e)))
    try:
      logging.info('Copying directory: %s to %s', src, dst)
      shutil.copytree(src, dst)
    except (shutil.Error, OSError) as e:
      raise ActionError('Unable to copy %s to %s: %s' % (src, dst, str(e))) 
Example 8
Project: ssrspeed_backup   Author: mazhenting   File: exporter_wps.py    GNU General Public License v3.0 6 votes vote down vote up
def export(self):
		nowTime = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
		fileloc = "./results/" + nowTime
		res = "var res = " + json.dumps(self.__results,sort_keys=True,indent=4,separators=(',',':'))
		if (os.path.exists(fileloc)):
			shutil.rmtree(fileloc)
		shutil.copytree("./resources/template/", fileloc)
		filename = os.path.join(fileloc, "results.js")
		with open(filename,"w+",encoding="utf-8") as f:
			f.writelines(res)
			f.close()
		indexFilename = os.path.join(fileloc, "index.html")
		index = ""
		read = []
		with open(indexFilename, "r", encoding="utf-8") as f:
			read = f.readlines()
			f.close()
		for r in read:
			index += r
		index = index.replace(r"{{ $generatedTime }}", nowTime)
		with open(indexFilename, "w+", encoding="utf-8") as f:
			f.writelines(index)
			f.close()
		logger.info("Web page simulation result exported as %s" % fileloc) 
Example 9
Project: dcc   Author: amimo   File: dcc.py    Apache License 2.0 6 votes vote down vote up
def copy_compiled_libs(project_dir, decompiled_dir):
    compiled_libs_dir = os.path.join(project_dir, "libs")
    decompiled_libs_dir = os.path.join(decompiled_dir, "lib")
    if not os.path.exists(compiled_libs_dir):
        return
    if not os.path.exists(decompiled_libs_dir):
        shutil.copytree(compiled_libs_dir, decompiled_libs_dir)
        return

    for abi in os.listdir(decompiled_libs_dir):
        dst = os.path.join(decompiled_libs_dir, abi)
        src = os.path.join(compiled_libs_dir, abi)
        if not os.path.exists(src) and abi == 'armeabi':
            src = os.path.join(compiled_libs_dir, 'armeabi-v7a')
            logger.warning('Use armeabi-v7a for armeabi')

        if not os.path.exists(src):
            raise Exception("ABI %s is not supported!" % abi)

        libnc = os.path.join(src, LIBNATIVECODE)
        shutil.copy(libnc, dst) 
Example 10
Project: recipes-py   Author: luci   File: cmd.py    Apache License 2.0 6 votes vote down vote up
def export_protos(destination):
  """Exports the compiled protos for the bundle.

  The engine initialization process has already built all protos and made them
  importable as `PB`. We rely on `PB.__path__` because this allows the
  `--proto-override` flag to work.

  Args:
    * repo (RecipeRepo) - The repo to export.
    * destination (str) - The absolute path we're exporting to (we'll export to
      a subfolder `_pb/PB`).
  """
  shutil.copytree(
      PB_PATH[0], # root of generated PB folder.
      os.path.join(destination, '_pb', 'PB'),
      ignore=lambda _base, names: [n for n in names if n.endswith('.pyc')],
  ) 
Example 11
Project: pysploit-framework   Author: ahmadnourallah   File: files.py    GNU General Public License v2.0 6 votes vote down vote up
def copy(src,dst):
    if type(src) == list:
        if type(dst) == list:
            try:
                for i in range(0,10000000):
                    if os.path.isfile(src[i]):
                        shutil.copy(src[i], dst[i])
                    else:
                        shutil.copytree(src[i], dst[i]+'/'+src[i])
            except IndexError:
                pass
        else:
            for src_el in src:
                if os.path.isfile(src_el):
                    shutil.copy(src_el,dst)
                else:
                    shutil.copytree(src_el,dst+'/'+src_el)
    else:
        if os.path.isfile(src):
            shutil.copy(src,dst)
        else:
            shutil.copytree(src,dst+'/'+src) 
Example 12
Project: pysploit-framework   Author: ahmadnourallah   File: files.py    GNU General Public License v2.0 6 votes vote down vote up
def move(src,dst):
    if type(src) == list:
        if type(dst) == list:
            try:
                for i in range(0,10000000):
                    if os.path.isfile(src[i]):
                        shutil.move(src[i], dst[i])
                    else:
                        shutil.copytree(src[i], dst[i]+'/'+src[i])
                        shutil.rmtree(src[i])
            except IndexError:
                pass
        else:
            for src_el in src:
                if os.path.isfile(src_el):
                    shutil.move(src_el,dst)
                else:
                    shutil.copytree(src_el,dst+'/'+src_el)
                    shutil.rmtree(src_el)
    else:
        if os.path.isfile(src):
            shutil.move(src,dst)
        else:
            shutil.copytree(src,dst+'/'+src)
            shutil.rmtree(src) 
Example 13
Project: sagemaker-xgboost-container   Author: aws   File: local_mode.py    Apache License 2.0 6 votes vote down vote up
def create_hosting_dir(model_dir, customer_script, optml, image, additional_volumes,
                       additional_env_vars,
                       cluster_size=1, source_dir=None, entrypoint=None, use_gpu=False):
    tmpdir = os.path.abspath(optml)
    print('creating hosting dir in {}'.format(tmpdir))

    hosts = create_host_names(cluster_size)
    print('creating hosts: {}'.format(hosts))

    if model_dir:
        for h in hosts:
            host_dir = os.path.join(tmpdir, h)
            os.makedirs(host_dir)
            shutil.copytree(model_dir, os.path.join(tmpdir, h, 'model'))

    write_docker_file('serve', tmpdir, hosts, image, additional_volumes, additional_env_vars,
                      customer_script,
                      source_dir, entrypoint, use_gpu)

    print("hosting dir: \n{}".format(
        str(subprocess.check_output(['ls', '-lR', tmpdir]).decode('utf-8'))))

    return tmpdir 
Example 14
Project: NiujiaoDebugger   Author: MrSrc   File: test_shutil.py    GNU General Public License v3.0 6 votes vote down vote up
def test_copytree_simple(self):
        src_dir = tempfile.mkdtemp()
        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')
        self.addCleanup(shutil.rmtree, src_dir)
        self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir))
        write_file((src_dir, 'test.txt'), '123')
        os.mkdir(os.path.join(src_dir, 'test_dir'))
        write_file((src_dir, 'test_dir', 'test.txt'), '456')

        shutil.copytree(src_dir, dst_dir)
        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
        self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
        self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
                                                    'test.txt')))
        actual = read_file((dst_dir, 'test.txt'))
        self.assertEqual(actual, '123')
        actual = read_file((dst_dir, 'test_dir', 'test.txt'))
        self.assertEqual(actual, '456') 
Example 15
Project: NiujiaoDebugger   Author: MrSrc   File: test_shutil.py    GNU General Public License v3.0 6 votes vote down vote up
def test_copytree_retains_permissions(self):
        tmp_dir = tempfile.mkdtemp()
        src_dir = os.path.join(tmp_dir, 'source')
        os.mkdir(src_dir)
        dst_dir = os.path.join(tmp_dir, 'destination')
        self.addCleanup(shutil.rmtree, tmp_dir)

        os.chmod(src_dir, 0o777)
        write_file((src_dir, 'permissive.txt'), '123')
        os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777)
        write_file((src_dir, 'restrictive.txt'), '456')
        os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600)
        restrictive_subdir = tempfile.mkdtemp(dir=src_dir)
        os.chmod(restrictive_subdir, 0o600)

        shutil.copytree(src_dir, dst_dir)
        self.assertEqual(os.stat(src_dir).st_mode, os.stat(dst_dir).st_mode)
        self.assertEqual(os.stat(os.path.join(src_dir, 'permissive.txt')).st_mode,
                          os.stat(os.path.join(dst_dir, 'permissive.txt')).st_mode)
        self.assertEqual(os.stat(os.path.join(src_dir, 'restrictive.txt')).st_mode,
                          os.stat(os.path.join(dst_dir, 'restrictive.txt')).st_mode)
        restrictive_subdir_dst = os.path.join(dst_dir,
                                              os.path.split(restrictive_subdir)[1])
        self.assertEqual(os.stat(restrictive_subdir).st_mode,
                          os.stat(restrictive_subdir_dst).st_mode) 
Example 16
Project: leapp-repository   Author: oamg   File: mounting.py    Apache License 2.0 5 votes vote down vote up
def copytree_to(self, src, dst):
        """
        Recursively copy an entire directory tree rooted at src. The destination directory,
        named by dst, must not already exist; it will be created as well as missing parent directories.

        The destination directory is considered to be in the isolated environment.
        The source directory is considered to be on the current system root.
        """
        shutil.copytree(src, self.full_path(dst)) 
Example 17
Project: leapp-repository   Author: oamg   File: mounting.py    Apache License 2.0 5 votes vote down vote up
def copytree_from(self, src, dst):
        """
        Recursively copy an entire directory tree rooted at src. The destination directory,
        named by dst, must not already exist; it will be created as well as missing parent directories.

        The destination directory is considered to be on the current system root.
        The source directory is considered to be in the isolated environment.
        """
        shutil.copytree(self.full_path(src), dst) 
Example 18
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_rmtree_on_symlink(self):
            # bug 1669.
            os.mkdir(TESTFN)
            try:
                src = os.path.join(TESTFN, 'cheese')
                dst = os.path.join(TESTFN, 'shop')
                os.mkdir(src)
                os.symlink(src, dst)
                self.assertRaises(OSError, shutil.rmtree, dst)
            finally:
                shutil.rmtree(TESTFN, ignore_errors=True)

    # Issue #3002: copyfile and copytree block indefinitely on named pipes 
Example 19
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: download.py    MIT License 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 20
Project: pytorch_NER_BiLSTM_CNN_CRF   Author: bamtercelboo   File: mainHelp.py    Apache License 2.0 5 votes vote down vote up
def load_model(config):
    """
    :param config:  config
    :return:  nn model
    """
    print("***************************************")
    model = Sequence_Label(config)
    print("Copy models to {}".format(config.save_dir))
    shutil.copytree("models", "/".join([config.save_dir, "models"]))
    if config.device != cpu_device:
        model = model.cuda()
    if config.test is True:
        model = load_test_model(model, config)
    print(model)
    return model 
Example 21
Project: flasky   Author: RoseOu   File: req.py    MIT License 5 votes vote down vote up
def copy_to_build_dir(self, req_to_install):
        target_dir = req_to_install.editable and self.src_dir or self.build_dir
        logger.info("Copying %s to %s" % (req_to_install.name, target_dir))
        dest = os.path.join(target_dir, req_to_install.name)
        shutil.copytree(req_to_install.source_dir, dest)
        call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest,
                        command_desc='python setup.py clean') 
Example 22
Project: flasky   Author: RoseOu   File: firefox_profile.py    MIT License 5 votes vote down vote up
def __init__(self, profile_directory=None):
        """
        Initialises a new instance of a Firefox Profile

        :args:
         - profile_directory: Directory of profile that you want to use.
           This defaults to None and will create a new
           directory when object is created.
        """
        if not FirefoxProfile.DEFAULT_PREFERENCES:
            with open(os.path.join(os.path.dirname(__file__),
                                   WEBDRIVER_PREFERENCES)) as default_prefs:
                FirefoxProfile.DEFAULT_PREFERENCES = json.load(default_prefs)

        self.default_preferences = copy.deepcopy(
            FirefoxProfile.DEFAULT_PREFERENCES['mutable'])
        self.native_events_enabled = True
        self.profile_dir = profile_directory
        self.tempfolder = None
        if self.profile_dir is None:
            self.profile_dir = self._create_tempfolder()
        else:
            self.tempfolder = tempfile.mkdtemp()
            newprof = os.path.join(self.tempfolder, "webdriver-py-profilecopy")
            shutil.copytree(self.profile_dir, newprof,
                            ignore=shutil.ignore_patterns("parent.lock", "lock", ".parentlock"))
            self.profile_dir = newprof
            self._read_existing_userjs(os.path.join(self.profile_dir, "user.js"))
        self.extensionsDir = os.path.join(self.profile_dir, "extensions")
        self.userPrefs = os.path.join(self.profile_dir, "user.js")

    #Public Methods 
Example 23
Project: flasky   Author: RoseOu   File: easy_install.py    MIT License 5 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(self.install_dir,os.path.basename(egg_path))
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(os.unlink,(destination,),"Removing "+destination)
            uncache_zipdir(destination)
            if os.path.isdir(egg_path):
                if egg_path.startswith(tmpdir):
                    f,m = shutil.move, "Moving"
                else:
                    f,m = shutil.copytree, "Copying"
            elif self.should_unzip(dist):
                self.mkpath(destination)
                f,m = self.unpack_and_compile, "Extracting"
            elif egg_path.startswith(tmpdir):
                f,m = shutil.move, "Moving"
            else:
                f,m = shutil.copy2, "Copying"

            self.execute(f, (egg_path, destination),
                (m+" %s to %s") %
                (os.path.basename(egg_path),os.path.dirname(destination)))

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 24
Project: CodeDog   Author: BruceDLong   File: buildDog.py    GNU General Public License v2.0 5 votes vote down vote up
def copyTree(src, dst):
    if not(os.path.isdir(src)): makeDir(src)
    for item in os.listdir(src):
        #print "item: ", item
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, False, None)
        else:
            shutil.copy2(s, d) 
Example 25
Project: CodeDog   Author: BruceDLong   File: buildAndroid.py    GNU General Public License v2.0 5 votes vote down vote up
def copyTree(src, dst):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        if os.path.isdir(s):
            shutil.copytree(s, d, False, None)
        else:
            shutil.copy2(s, d) 
Example 26
Project: CodeDog   Author: BruceDLong   File: buildAndroid.py    GNU General Public License v2.0 5 votes vote down vote up
def copyFile(fileName, src, dst):
    s = os.path.join(src, fileName)
    d = os.path.join(dst, fileName)
    if os.path.isdir(s):
        shutil.copytree(s, d, False, None)
    else:
        shutil.copy2(s, d) 
Example 27
Project: Trusted-Platform-Module-nova   Author: BU-NU-CLOUD-SP16   File: configdrive.py    Apache License 2.0 5 votes vote down vote up
def _make_vfat(self, path, tmpdir):
        # NOTE(mikal): This is a little horrible, but I couldn't find an
        # equivalent to genisoimage for vfat filesystems.
        with open(path, 'wb') as f:
            f.truncate(CONFIGDRIVESIZE_BYTES)

        utils.mkfs('vfat', path, label='config-2')

        with utils.tempdir() as mountdir:
            mounted = False
            try:
                _, err = utils.trycmd(
                    'mount', '-o', 'loop,uid=%d,gid=%d' % (os.getuid(),
                                                           os.getgid()),
                    path,
                    mountdir,
                    run_as_root=True)
                if err:
                    raise exception.ConfigDriveMountFailed(operation='mount',
                                                           error=err)
                mounted = True

                # NOTE(mikal): I can't just use shutils.copytree here,
                # because the destination directory already
                # exists. This is annoying.
                for ent in os.listdir(tmpdir):
                    shutil.copytree(os.path.join(tmpdir, ent),
                                    os.path.join(mountdir, ent))

            finally:
                if mounted:
                    utils.execute('umount', mountdir, run_as_root=True) 
Example 28
Project: sic   Author: Yanixos   File: download.py    GNU General Public License v3.0 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 29
Project: sic   Author: Yanixos   File: egg2wheel.py    GNU General Public License v3.0 5 votes vote down vote up
def egg2wheel(egg_path, dest_dir):
    egg_info = egg_info_re.match(os.path.basename(egg_path)).groupdict()
    dir = tempfile.mkdtemp(suffix="_e2w")
    if os.path.isfile(egg_path):
        # assume we have a bdist_egg otherwise
        egg = zipfile.ZipFile(egg_path)
        egg.extractall(dir)
    else:
        # support buildout-style installed eggs directories
        for pth in os.listdir(egg_path):
            src = os.path.join(egg_path, pth)
            if os.path.isfile(src):
                shutil.copy2(src, dir)
            else:
                shutil.copytree(src, os.path.join(dir, pth))

    dist_info = "%s-%s" % (egg_info['name'], egg_info['ver'])
    abi = 'none'
    pyver = egg_info['pyver'].replace('.', '')
    arch = (egg_info['arch'] or 'any').replace('.', '_').replace('-', '_')
    if arch != 'any':
        # assume all binary eggs are for CPython
        pyver = 'cp' + pyver[2:]
    wheel_name = '-'.join((
                          dist_info,
                          pyver,
                          abi,
                          arch
                          ))
    bw = wheel.bdist_wheel.bdist_wheel(distutils.dist.Distribution())
    bw.root_is_purelib = egg_info['arch'] is None
    dist_info_dir = os.path.join(dir, '%s.dist-info' % dist_info)
    bw.egg2dist(os.path.join(dir, 'EGG-INFO'),
                dist_info_dir)
    bw.write_wheelfile(dist_info_dir, generator='egg2wheel')
    bw.write_record(dir, dist_info_dir)
    filename = make_archive(os.path.join(dest_dir, wheel_name), 'zip', root_dir=dir)
    os.rename(filename, filename[:-3] + 'whl')
    shutil.rmtree(dir) 
Example 30
Project: cards.py   Author: jhauberg   File: cards.py    MIT License 5 votes vote down vote up
def make_empty_project(in_path: str,
                       name: str=None) -> bool:
    """ Build an empty project that can be used as a starting point. """

    destination_path = in_path

    if name is not None and len(name) > 0:
        # make sure any whitespace is replaced with dashes
        name_components = name.split(' ')
        name = '-'.join(name_components).lower()

        destination_path = os.path.join(destination_path, name)

    empty_project_path = os.path.join(get_base_path(), 'templates/project')
    destination_path = os.path.join(destination_path, 'src')

    if os.path.isdir(destination_path):
        WarningDisplay.could_not_make_new_project_error(
            destination_path, already_exists=True)

        return False

    try:
        shutil.copytree(empty_project_path, destination_path)

        print('Made new project\n -> {0}\'{1}\'{2}'.format(
            WarningDisplay.apply_normal_color_underlined, destination_path,
            WarningDisplay.apply_normal_color))

        print()

        open_path(destination_path)

        return True
    except IOError as error:
        WarningDisplay.could_not_make_new_project_error(
            destination_path, reason=str(error))

    return False 
Example 31
Project: AshsSDK   Author: thehappydinoa   File: download.py    MIT License 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 32
Project: AshsSDK   Author: thehappydinoa   File: egg2wheel.py    MIT License 5 votes vote down vote up
def egg2wheel(egg_path, dest_dir):
    egg_info = egg_info_re.match(os.path.basename(egg_path)).groupdict()
    dir = tempfile.mkdtemp(suffix="_e2w")
    if os.path.isfile(egg_path):
        # assume we have a bdist_egg otherwise
        egg = zipfile.ZipFile(egg_path)
        egg.extractall(dir)
    else:
        # support buildout-style installed eggs directories
        for pth in os.listdir(egg_path):
            src = os.path.join(egg_path, pth)
            if os.path.isfile(src):
                shutil.copy2(src, dir)
            else:
                shutil.copytree(src, os.path.join(dir, pth))

    dist_info = "%s-%s" % (egg_info['name'], egg_info['ver'])
    abi = 'none'
    pyver = egg_info['pyver'].replace('.', '')
    arch = (egg_info['arch'] or 'any').replace('.', '_').replace('-', '_')
    if arch != 'any':
        # assume all binary eggs are for CPython
        pyver = 'cp' + pyver[2:]
    wheel_name = '-'.join((
                          dist_info,
                          pyver,
                          abi,
                          arch
                          ))
    bw = wheel.bdist_wheel.bdist_wheel(distutils.dist.Distribution())
    bw.root_is_purelib = egg_info['arch'] is None
    dist_info_dir = os.path.join(dir, '%s.dist-info' % dist_info)
    bw.egg2dist(os.path.join(dir, 'EGG-INFO'),
                dist_info_dir)
    bw.write_wheelfile(dist_info_dir, generator='egg2wheel')
    bw.write_record(dir, dist_info_dir)
    filename = make_archive(os.path.join(dest_dir, wheel_name), 'zip', root_dir=dir)
    os.rename(filename, filename[:-3] + 'whl')
    shutil.rmtree(dir) 
Example 33
Project: mx   Author: graalvm   File: mx.py    GNU General Public License v2.0 5 votes vote down vote up
def copytree(src, dst, symlinks=False, ignore=None):
    shutil.copytree(_safe_path(src), _safe_path(dst), symlinks, ignore) 
Example 34
Project: openhatch   Author: campbe13   File: py26.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def copytree(src, dst, symlinks=False, ignore=None):
        names = os.listdir(src)
        if ignore is not None:
            ignored_names = ignore(src, names)
        else:
            ignored_names = set()

        os.makedirs(dst)
        errors = []
        for name in names:
            if name in ignored_names:
                continue
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)
            try:
                if symlinks and os.path.islink(srcname):
                    linkto = os.readlink(srcname)
                    os.symlink(linkto, dstname)
                elif os.path.isdir(srcname):
                    copytree(srcname, dstname, symlinks, ignore)
                else:
                    copy2(srcname, dstname)
                # XXX What about devices, sockets etc.?
            except (IOError, os.error), why:
                errors.append((srcname, dstname, str(why)))
            # catch the Error from the recursive copytree so that we can
            # continue with other files
            except Error, err:
                errors.extend(err.args[0]) 
Example 35
Project: openhatch   Author: campbe13   File: __init__.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def setUp(self):
        orig_spiders_dir = os.path.join(module_dir, 'test_spiders')
        self.tmpdir = self.mktemp()
        os.mkdir(self.tmpdir)
        self.spiders_dir = os.path.join(self.tmpdir, 'test_spiders_xxx')
        shutil.copytree(orig_spiders_dir, self.spiders_dir)
        sys.path.append(self.tmpdir)
        self.spiderman = SpiderManager(['test_spiders_xxx']) 
Example 36
Project: Repobot   Author: Desgard   File: download.py    MIT License 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 37
Project: Repobot   Author: Desgard   File: egg2wheel.py    MIT License 5 votes vote down vote up
def egg2wheel(egg_path, dest_dir):
    egg_info = egg_info_re.match(os.path.basename(egg_path)).groupdict()
    dir = tempfile.mkdtemp(suffix="_e2w")
    if os.path.isfile(egg_path):
        # assume we have a bdist_egg otherwise
        egg = zipfile.ZipFile(egg_path)
        egg.extractall(dir)
    else:
        # support buildout-style installed eggs directories
        for pth in os.listdir(egg_path):
            src = os.path.join(egg_path, pth)
            if os.path.isfile(src):
                shutil.copy2(src, dir)
            else:
                shutil.copytree(src, os.path.join(dir, pth))

    dist_info = "%s-%s" % (egg_info['name'], egg_info['ver'])
    abi = 'none'
    pyver = egg_info['pyver'].replace('.', '')
    arch = (egg_info['arch'] or 'any').replace('.', '_').replace('-', '_')
    if arch != 'any':
        # assume all binary eggs are for CPython
        pyver = 'cp' + pyver[2:]
    wheel_name = '-'.join((
                          dist_info,
                          pyver,
                          abi,
                          arch
                          ))
    bw = wheel.bdist_wheel.bdist_wheel(distutils.dist.Distribution())
    bw.root_is_purelib = egg_info['arch'] is None
    dist_info_dir = os.path.join(dir, '%s.dist-info' % dist_info)
    bw.egg2dist(os.path.join(dir, 'EGG-INFO'),
                dist_info_dir)
    bw.write_wheelfile(dist_info_dir, generator='egg2wheel')
    bw.write_record(dir, dist_info_dir)
    filename = make_archive(os.path.join(dest_dir, wheel_name), 'zip', root_dir=dir)
    os.rename(filename, filename[:-3] + 'whl')
    shutil.rmtree(dir) 
Example 38
Project: vscode-mayapy   Author: FXTD-ODYSSEY   File: regression_check.py    MIT License 5 votes vote down vote up
def datadir(original_datadir, tmpdir):
    # Method from: https://github.com/gabrielcnr/pytest-datadir
    # License: MIT
    import shutil
    result = Path(str(tmpdir.join(original_datadir.stem)))
    if original_datadir.is_dir():
        shutil.copytree(str(original_datadir), str(result))
    else:
        result.mkdir()
    return result 
Example 39
Project: project-automation   Author: jarodburchill   File: project_types.py    MIT License 5 votes vote down vote up
def Html():
    scriptPath = os.path.dirname(os.path.realpath(__file__))
    src = "{}\\assets\\html\\".format(scriptPath)
    dst = "{}\\{}\\".format(directory, projectName)
    shutil.copytree(src, dst)
    os.chdir(projectName)
    subprocess.check_call("echo {} >> README.md".format(repoName), shell=True)


# process for react projects 
Example 40
Project: JukeBox   Author: gauravsarkar97   File: download.py    MIT License 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(SETUPTOOLS_SHIM % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 41
Project: dcc   Author: amimo   File: dcc.py    Apache License 2.0 5 votes vote down vote up
def dcc_main(apkfile, filtercfg, outapk, do_compile=True, project_dir=None, source_archive='project-source.zip'):
    if not os.path.exists(apkfile):
        logger.error("file %s is not exists", apkfile)
        return

    compiled_methods, errors = compile_dex(apkfile, filtercfg)

    if errors:
        logger.warning('================================')
        logger.warning('\n'.join(errors))
        logger.warning('================================')

    if len(compiled_methods) == 0:
        logger.info("no compiled methods")
        return

    if project_dir:
        if not os.path.exists(project_dir):
            shutil.copytree('project', project_dir)
        write_compiled_methods(project_dir, compiled_methods)
    else:
        project_dir = make_temp_dir('dcc-project-')
        shutil.rmtree(project_dir)
        shutil.copytree('project', project_dir)
        write_compiled_methods(project_dir, compiled_methods)
        src_zip = archive_compiled_code(project_dir)
        shutil.move(src_zip, source_archive)

    if do_compile:
        build_project(project_dir)

    if is_apk(apkfile) and outapk:
        decompiled_dir = ApkTool.decompile(apkfile)
        native_compiled_dexes(decompiled_dir, compiled_methods)
        copy_compiled_libs(project_dir, decompiled_dir)
        unsigned_apk = ApkTool.compile(decompiled_dir)
        sign(unsigned_apk, outapk) 
Example 42
Project: chattR   Author: patrickstocklin   File: download.py    GNU General Public License v2.0 5 votes vote down vote up
def unpack_file_url(link, location, download_dir=None):
    """Unpack link into location.
    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir."""

    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if os.path.isdir(link_path):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # if link has a hash, let's confirm it matches
    if link.hash:
        link_path_hash = _get_hash_from_file(link_path, link)
        _check_hash(link_path_hash, link)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link, download_dir)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, content_type, link) 
Example 43
Project: chattR   Author: patrickstocklin   File: download.py    GNU General Public License v2.0 5 votes vote down vote up
def _copy_dist_from_dir(link_path, location):
    """Copy distribution files in `link_path` to `location`.

    Invoked when user requests to install a local directory. E.g.:

        pip install .
        pip install ~/dev/git-repos/python-prompt-toolkit

    """

    # Note: This is currently VERY SLOW if you have a lot of data in the
    # directory, because it copies everything with `shutil.copytree`.
    # What it should really do is build an sdist and install that.
    # See https://github.com/pypa/pip/issues/2195

    if os.path.isdir(location):
        rmtree(location)

    # build an sdist
    setup_py = 'setup.py'
    sdist_args = [sys.executable]
    sdist_args.append('-c')
    sdist_args.append(
        "import setuptools, tokenize;__file__=%r;"
        "exec(compile(getattr(tokenize, 'open', open)(__file__).read()"
        ".replace('\\r\\n', '\\n'), __file__, 'exec'))" % setup_py)
    sdist_args.append('sdist')
    sdist_args += ['--dist-dir', location]
    logger.info('Running setup.py sdist for %s', link_path)

    with indent_log():
        call_subprocess(sdist_args, cwd=link_path, show_stdout=False)

    # unpack sdist into `location`
    sdist = os.path.join(location, os.listdir(location)[0])
    logger.info('Unpacking sdist %s into %s', sdist, location)
    unpack_file(sdist, location, content_type=None, link=None) 
Example 44
Project: d6tpipe   Author: d6t   File: utils.py    MIT License 5 votes vote down vote up
def copytree(src, dst, move=False, symlinks=False, ignore=None):
    for item in os.listdir(src):
        s = os.path.join(src, item)
        d = os.path.join(dst, item)
        print(s,d)
        if os.path.isdir(s):
            if move:
                shutil.move(s, d)
            else:
                shutil.copytree(s, d, symlinks, ignore)
        else:
            if move:
                shutil.move(s, d)
            else:
                shutil.copy2(s, d) 
Example 45
Project: kubeshift   Author: cdrage   File: test_config.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def setUp(self):
        try:
            # copy certs to /tmp directory
            shutil.copytree(
                os.path.join(FIXTURE_DIR, 'certs'),
                os.path.join(tempfile.gettempdir(), '.minikube')
            )
        except Exception:
            pass 
Example 46
Project: DRCOG_Urbansim   Author: apdjustino   File: misc.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def copytree(src, dst, skip_subdirectories=[]):
    """recursively copy the directory tree rooted at src to the destination directory dst.
    Skip any subdirectories in skip_subdirectories."""
    # This function is adapted from the shutil.copytree function.  It adds the optional 
    # skip_subdirectories parameter, and omits the symlinks parameter.
    names = os.listdir(src)
    os.mkdir(dst)
    for name in names:
        if name not in skip_subdirectories:
            srcname = os.path.join(src, name)
            dstname = os.path.join(dst, name)
            if os.path.isdir(srcname):
                copytree(srcname, dstname, skip_subdirectories=skip_subdirectories)
            else:
                shutil.copy2(srcname, dstname) 
Example 47
Project: DRCOG_Urbansim   Author: apdjustino   File: misc.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def test_copytree(self):
        dest = os.path.join(self.temp_dir, 'dest')
        os.mkdir(dest)
        dirs = [
            ['d1', 'd2', 'd3', 'CVS', 'sub'],
            ['d2', 'd3'],
            ['d4', 'CVS', 'd1'],
            ['d5'],
            ['d6', '.svn', 'd1', 'd2'],
            ]
        for t in dirs:
            path = self.temp_dir
            for n in t:
                path = os.path.join(path, n)
            os.makedirs(path)
            source = os.path.join(self.temp_dir, t[0])
            sub = os.path.join(dest, t[0])
            copytree(source, sub, skip_subdirectories=['CVS', '.svn'])
        for t in dirs:
            path = dest
            for n in t:
                path = os.path.join(path, n)
            if 'CVS' in t or '.svn' in t:
                self.assert_(not os.path.exists(path))
            else:
                self.assert_(os.path.exists(path)) 
Example 48
Project: DRCOG_Urbansim   Author: apdjustino   File: test_opus_project.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def setUp(self):
        testdatapath = self._get_data_path(self.TESTDATA)
        testdatapath_tmp = self._get_data_path(self.TESTDATA_TMP)
        if os.path.exists(testdatapath_tmp):
            shutil.rmtree(testdatapath_tmp, False)
        shutil.copytree(testdatapath, testdatapath_tmp)
        OpusProjectTestCase.setUp(self, testdatapath_tmp) 
Example 49
Project: conan-center-index   Author: conan-io   File: conanfile.py    MIT License 5 votes vote down vote up
def package(self):
        excludes = None
        if self.options.exclude_files:
            excludes = tuple(str(self.options.exclude_files).split(","))
        self.copy("*", dst="bin", src=self._msys_dir, excludes=excludes)
        shutil.copytree(os.path.join(self.package_folder, "bin", "usr", "share", "licenses"),
                        os.path.join(self.package_folder, "licenses")) 
Example 50
Project: word2vec-twitter   Author: loretoparisi   File: word2vecReaderUtils.py    MIT License 5 votes vote down vote up
def copytree_hardlink(source, dest):
    """
    Recursively copy a directory ala shutils.copytree, but hardlink files
    instead of copying. Available on UNIX systems only.
    """
    copy2 = shutil.copy2
    try:
        shutil.copy2 = os.link
        shutil.copytree(source, dest)
    finally:
        shutil.copy2 = copy2 
Example 51
Project: sagemaker-xgboost-container   Author: aws   File: local_mode.py    Apache License 2.0 5 votes vote down vote up
def copy_resource(resource_path, opt_ml_path, relative_src_path, relative_dst_path=None):
    if not relative_dst_path:
        relative_dst_path = relative_src_path

    shutil.copytree(os.path.join(resource_path, relative_src_path),
                    os.path.join(opt_ml_path, relative_dst_path)) 
Example 52
Project: Flask_Blog   Author: sugarguo   File: req.py    GNU General Public License v3.0 5 votes vote down vote up
def copy_to_build_dir(self, req_to_install):
        target_dir = req_to_install.editable and self.src_dir or self.build_dir
        logger.info("Copying %s to %s" % (req_to_install.name, target_dir))
        dest = os.path.join(target_dir, req_to_install.name)
        shutil.copytree(req_to_install.source_dir, dest)
        call_subprocess(["python", "%s/setup.py" % dest, "clean"], cwd=dest,
                        command_desc='python setup.py clean') 
Example 53
Project: Flask_Blog   Author: sugarguo   File: easy_install.py    GNU General Public License v3.0 5 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(self.install_dir,os.path.basename(egg_path))
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(os.unlink,(destination,),"Removing "+destination)
            uncache_zipdir(destination)
            if os.path.isdir(egg_path):
                if egg_path.startswith(tmpdir):
                    f,m = shutil.move, "Moving"
                else:
                    f,m = shutil.copytree, "Copying"
            elif self.should_unzip(dist):
                self.mkpath(destination)
                f,m = self.unpack_and_compile, "Extracting"
            elif egg_path.startswith(tmpdir):
                f,m = shutil.move, "Moving"
            else:
                f,m = shutil.copy2, "Copying"

            self.execute(f, (egg_path, destination),
                (m+" %s to %s") %
                (os.path.basename(egg_path),os.path.dirname(destination)))

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 54
Project: tablemate   Author: jhermann   File: tasks.py    Apache License 2.0 5 votes vote down vote up
def fresh_cookies(ctx, mold=''):
    """Refresh the project from the original cookiecutter template."""
    mold = mold or "https://github.com/Springerle/py-generic-project.git"  # TODO: URL from config
    tmpdir = os.path.join(tempfile.gettempdir(), "cc-upgrade-tablemate")

    if os.path.isdir('.git'):
        # TODO: Ensure there are no local unstashed changes
        pass

    # Make a copy of the new mold version
    if os.path.isdir(tmpdir):
        shutil.rmtree(tmpdir)
    if os.path.exists(mold):
        shutil.copytree(mold, tmpdir, ignore=shutil.ignore_patterns(
            ".git", ".svn", "*~",
        ))
    else:
        ctx.run("git clone {} {}".format(mold, tmpdir))

    # Copy recorded "cookiecutter.json" into mold
    shutil.copy2("project.d/cookiecutter.json", tmpdir)

    with pushd('..'):
        ctx.run("cookiecutter --no-input {}".format(tmpdir))
    if os.path.exists('.git'):
        ctx.run("git status") 
Example 55
Project: siphon-cli   Author: getsiphon   File: create.py    MIT License 5 votes vote down vote up
def copy_app_template(app_name, to_path):
    path = os.path.dirname(os.path.abspath(__file__))
    source_dir = os.path.join(path, '../../../app-template/')
    ignore_func = shutil.ignore_patterns('.DS_Store', 'Thumbs.db')
    shutil.copytree(source_dir, to_path, ignore=ignore_func)

    # Replace the app name placeholder
    index_path = os.path.join(to_path, 'index.js')
    s = open(index_path, 'r').read().replace('{{app_name}}', app_name)
    with open(index_path, 'w') as fp:
        fp.write(s) 
Example 56
Project: ltibootcamp   Author: claudevervoort   File: localprocess_jupyterhub_config.py    Apache License 2.0 5 votes vote down vote up
def copy_notebook(spawner):
    print('pre-spawning %s', (spawner.user.name))
    username = spawner.user.name # get the username
    
    notebooks_path = os.path.join(os.getcwd(),'jhnotebooks')
    volume_path = os.path.join(notebooks_path, username)
    if not os.path.exists(notebooks_path):
        os.mkdir(notebooks_path) #set proper version
    if not os.path.exists(volume_path):
        # create a directory with umask 0755 
        # hub and container user must have the same UID to be writeable
        # still readable by other users on the system
        print('making dir %s' % (volume_path))
        # os.mkdir(volume_path, 0o755)
        shutil.copytree(os.path.join(os.getcwd(), '../jupyter/notebooks/'), volume_path)
        print('copy dir %s' % (volume_path))
        with os.scandir(volume_path) as it:
            for entry in it:
                if entry.is_dir():
                    shutil.chown(entry.path, user=username, group=username)
                if entry.is_file():
                    shutil.chown(entry.path, user=username, group=username)

        # now do whatever you think your user needs
        # ...
        pass 
Example 57
Project: ltibootcamp   Author: claudevervoort   File: systemd_jupyterhub_config.py    Apache License 2.0 5 votes vote down vote up
def copy_notebook(spawner):
    print('pre-spawning %s', (spawner.user.name))
    username = spawner.user.name # get the username
    
    volume_path = os.path.join('/home/', username, 'ltibootcamp')
    #if not os.path.exists(notebooks_path):
    #    os.mkdir(notebooks_path) #set proper version
    if not os.path.exists(volume_path):
        # create a directory with umask 0755 
        # hub and container user must have the same UID to be writeable
        # still readable by other users on the system
        print('making dir %s' % (volume_path))
        # os.mkdir(volume_path, 0o755)
        shutil.copytree(os.path.join(os.getcwd(), '../jupyter/notebooks/'), volume_path)
        print('copy dir %s' % (volume_path))
        shutil.chown(volume_path, user=username, group=username)
        with os.scandir(volume_path) as it:
            for entry in it:
                if entry.is_dir():
                    shutil.chown(entry.path, user=username, group=username)
                if entry.is_file():
                    shutil.chown(entry.path, user=username, group=username)

        # now do whatever you think your user needs
        # ...
        pass 
Example 58
Project: NiujiaoDebugger   Author: MrSrc   File: test_shutil.py    GNU General Public License v3.0 5 votes vote down vote up
def test_copytree_symlinks(self):
        tmp_dir = self.mkdtemp()
        src_dir = os.path.join(tmp_dir, 'src')
        dst_dir = os.path.join(tmp_dir, 'dst')
        sub_dir = os.path.join(src_dir, 'sub')
        os.mkdir(src_dir)
        os.mkdir(sub_dir)
        write_file((src_dir, 'file.txt'), 'foo')
        src_link = os.path.join(sub_dir, 'link')
        dst_link = os.path.join(dst_dir, 'sub/link')
        os.symlink(os.path.join(src_dir, 'file.txt'),
                   src_link)
        if hasattr(os, 'lchmod'):
            os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO)
        if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'):
            os.lchflags(src_link, stat.UF_NODUMP)
        src_stat = os.lstat(src_link)
        shutil.copytree(src_dir, dst_dir, symlinks=True)
        self.assertTrue(os.path.islink(os.path.join(dst_dir, 'sub', 'link')))
        self.assertEqual(os.readlink(os.path.join(dst_dir, 'sub', 'link')),
                         os.path.join(src_dir, 'file.txt'))
        dst_stat = os.lstat(dst_link)
        if hasattr(os, 'lchmod'):
            self.assertEqual(dst_stat.st_mode, src_stat.st_mode)
        if hasattr(os, 'lchflags'):
            self.assertEqual(dst_stat.st_flags, src_stat.st_flags) 
Example 59
Project: fs_image   Author: facebookincubator   File: temp_repos.py    MIT License 4 votes vote down vote up
def make_repo_steps(
    out_dir: Path, repo_change_steps: List[Dict[str, Repo]], arch: str,
    avoid_symlinks: bool = False,
):
    # When an RPM occurs in two different repos, we want it to be
    # bit-identical (otherwise, the snapshot would see a `mutable_rpm`
    # error).  This means never rebuilding an RPM that was previously seen.
    # The paths are relative to `out_dir`.
    rpm_to_path = {}
    # The repos that exist at the current step.
    repos = {}
    for step, repo_changes in enumerate(repo_change_steps):
        step = Path(str(step))
        for repo_name, repo in repo_changes.items():
            if repo is None:
                del repos[repo_name]
            else:
                repos[repo_name] = repo
        step_dir = out_dir / step
        os.makedirs(step_dir)
        yum_dnf_conf = ConfigParser()
        yum_dnf_conf['main'] = {}
        for repo_name, repo in repos.items():
            repo_dir = step_dir / repo_name
            yum_dnf_conf[repo_name] = {'baseurl': repo_dir.file_url()}
            if isinstance(repo, str):  # Alias of another repo
                assert repo in repos
                if avoid_symlinks:
                    shutil.copytree(step_dir / repo, repo_dir)
                else:
                    os.symlink(repo, repo_dir)
                continue
            # Each repo's package dir is different to exercise the fact
            # that the same file's location may differ across repos.
            package_dir = repo_dir / f'{repo_name}-pkgs'
            os.makedirs(package_dir)
            for rpm in repo.rpms:
                prev_path = rpm_to_path.get(rpm)
                if prev_path and avoid_symlinks:
                    shutil.copy(
                        out_dir / prev_path,
                        package_dir / prev_path.basename(),
                    )
                elif prev_path:
                    os.symlink(
                        '../../..' / prev_path,
                        package_dir / prev_path.basename(),
                    )
                else:
                    rpm_to_path[rpm] = (
                        step / repo_name / package_dir.basename() /
                        build_rpm(package_dir, arch, rpm)
                    )
            # Now that all RPMs were built, we can generate the Yum metadata
            subprocess.run(['createrepo_c', repo_dir], check=True)
        for prog_name in ['dnf', 'yum']:
            with open(step_dir / f'{prog_name}.conf', 'w') as out_f:
                yum_dnf_conf.write(out_f) 
Example 60
Project: pyblish-win   Author: pyblish   File: test_shutil.py    GNU Lesser General Public License v3.0 4 votes vote down vote up
def test_copytree_simple(self):
        def write_data(path, data):
            f = open(path, "w")
            f.write(data)
            f.close()

        def read_data(path):
            f = open(path)
            data = f.read()
            f.close()
            return data

        src_dir = tempfile.mkdtemp()
        dst_dir = os.path.join(tempfile.mkdtemp(), 'destination')

        write_data(os.path.join(src_dir, 'test.txt'), '123')

        os.mkdir(os.path.join(src_dir, 'test_dir'))
        write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456')

        try:
            shutil.copytree(src_dir, dst_dir)
            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt')))
            self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir')))
            self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir',
                                                        'test.txt')))
            actual = read_data(os.path.join(dst_dir, 'test.txt'))
            self.assertEqual(actual, '123')
            actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt'))
            self.assertEqual(actual, '456')
        finally:
            for path in (
                    os.path.join(src_dir, 'test.txt'),
                    os.path.join(dst_dir, 'test.txt'),
                    os.path.join(src_dir, 'test_dir', 'test.txt'),
                    os.path.join(dst_dir, 'test_dir', 'test.txt'),
                ):
                if os.path.exists(path):
                    os.remove(path)
            for path in (src_dir,
                    os.path.dirname(dst_dir)
                ):
                if os.path.exists(path):
                    shutil.rmtree(path) 
Example 61
Project: DBC-FederatedLearning-Client-VNX   Author: DeepBrainChain   File: file_manipulator.py    Apache License 2.0 4 votes vote down vote up
def merge(input_paths, out_folder):	
	filelist=[]
	foldername=str(os.path.basename(input_paths[0]))
	tempin=str(input_paths[0])
	tempout=out_folder
	#copytree
	try:
		shutil.copytree(tempin, tempout)
	except OSError as e:
        # If the error was caused because the source wasn't a directory
		if e.errno == errno.ENOTDIR:
			shutil.copy(tempin, tempout)
		else:
			print('Directory not copied. Error: %s' % e)
			
	#clean folder		
	for dirpath, dirnames, filenames in os.walk(out_folder):
		
		for filename1 in filenames:
			os.remove(str(dirpath + '/' + filename1))
			
	#copy files by name		
	for dirpath, dirnames, filenames in os.walk(Path(tempin).parent):
		filelist=filelist+filenames
		for filename in filenames:
			commonpath=os.path.commonpath([dirpath, out_folder]) + '/in'
			p = pathlib.Path(dirpath.replace(commonpath,''))
			new_replacepath='/' + str(pathlib.Path(*p.parts[2:]))
			new_replacepath=new_replacepath.replace('/.','')
			#print(new_replacepath)
			
			
			dest=out_folder + new_replacepath + '/' + str(filename)

			#print(dest)
            #remove extension(folder)
					
			dest, file_extension = os.path.splitext(dest)
            
            #split filename
			repeated_times=filelist.count(filename)
            #print(repeated_times)
			 
			filenametemp=filename.replace(file_extension,'') + '(' + str(repeated_times) + ')' + file_extension
			dest=out_folder + new_replacepath
			
			copyfile(os.path.join(dirpath,filename), os.path.join(dest, filenametemp)) 
Example 62
Project: Ansible-Example-AB2018   Author: umit-ozturk   File: download.py    MIT License 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None, hashes=None):
    """Unpack link into location.

    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir.
    """
    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if is_dir_url(link):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # If --require-hashes is off, `hashes` is either empty, the
    # link's embedded hash, or MissingHashes; it is required to
    # match. If --require-hashes is on, we are satisfied by any
    # hash in `hashes` matching: a URL-based or an option-based
    # one; no internet-sourced hash will be in `hashes`.
    if hashes:
        hashes.check_against_path(link_path)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link,
                                                      download_dir,
                                                      hashes)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, link) 
Example 63
Project: flasky   Author: RoseOu   File: download.py    MIT License 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None):

    link_path = url_to_path(link.url_without_fragment)
    already_downloaded = False

    # If it's a url to a local directory
    if os.path.isdir(link_path):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        return

    # if link has a hash, let's confirm it matches
    if link.hash:
        link_path_hash = _get_hash_from_file(link_path, link)
        _check_hash(link_path_hash, link)

    # If a download dir is specified, is the file already there and valid?
    if download_dir:
        download_path = os.path.join(download_dir, link.filename)
        if os.path.exists(download_path):
            content_type = mimetypes.guess_type(download_path)[0]
            logger.notify('File was already downloaded %s' % download_path)
            if link.hash:
                download_hash = _get_hash_from_file(download_path, link)
                try:
                    _check_hash(download_hash, link)
                    already_downloaded = True
                except HashMismatch:
                    logger.warn(
                        'Previously-downloaded file %s has bad hash, '
                        're-downloading.' % link_path
                        )
                    os.unlink(download_path)
            else:
                already_downloaded = True

    if already_downloaded:
        from_path = download_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded:
        _copy_file(from_path, download_dir, content_type, link) 
Example 64
Project: flasky   Author: RoseOu   File: firefox_profile.py    MIT License 4 votes vote down vote up
def _install_extension(self, addon, unpack=True):
        """
            Installs addon from a filepath, url
            or directory of addons in the profile.
            - path: url, path to .xpi, or directory of addons
            - unpack: whether to unpack unless specified otherwise in the install.rdf
        """
        if addon == WEBDRIVER_EXT:
            addon = os.path.join(os.path.dirname(__file__), WEBDRIVER_EXT)

        tmpdir = None
        xpifile = None
        if addon.endswith('.xpi'):
            tmpdir = tempfile.mkdtemp(suffix='.' + os.path.split(addon)[-1])
            compressed_file = zipfile.ZipFile(addon, 'r')
            for name in compressed_file.namelist():
                if name.endswith('/'):
                    os.makedirs(os.path.join(tmpdir, name))
                else:
                    if not os.path.isdir(os.path.dirname(os.path.join(tmpdir, name))):
                        os.makedirs(os.path.dirname(os.path.join(tmpdir, name)))
                    data = compressed_file.read(name)
                    with open(os.path.join(tmpdir, name), 'wb') as f:
                        f.write(data)
            xpifile = addon
            addon = tmpdir

        # determine the addon id
        addon_details = self._addon_details(addon)
        addon_id = addon_details.get('id')
        assert addon_id, 'The addon id could not be found: %s' % addon

        # copy the addon to the profile
        extensions_path = os.path.join(self.profile_dir, 'extensions')
        addon_path = os.path.join(extensions_path, addon_id)
        if not unpack and not addon_details['unpack'] and xpifile:
            if not os.path.exists(extensions_path):
                os.makedirs(extensions_path)
            shutil.copy(xpifile, addon_path + '.xpi')
        else:
            shutil.copytree(addon, addon_path, symlinks=True)

        # remove the temporary directory, if any
        if tmpdir:
            shutil.rmtree(tmpdir) 
Example 65
Project: sic   Author: Yanixos   File: download.py    GNU General Public License v3.0 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None, hashes=None):
    """Unpack link into location.

    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir.
    """
    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if is_dir_url(link):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # If --require-hashes is off, `hashes` is either empty, the
    # link's embedded hash, or MissingHashes; it is required to
    # match. If --require-hashes is on, we are satisfied by any
    # hash in `hashes` matching: a URL-based or an option-based
    # one; no internet-sourced hash will be in `hashes`.
    if hashes:
        hashes.check_against_path(link_path)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link,
                                                      download_dir,
                                                      hashes)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, link) 
Example 66
Project: sic   Author: Yanixos   File: easy_install.py    GNU General Public License v3.0 4 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 67
Project: bigquerylayers   Author: smandaric   File: easy_install.py    GNU General Public License v3.0 4 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 68
Project: AshsSDK   Author: thehappydinoa   File: download.py    MIT License 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None, hashes=None):
    """Unpack link into location.

    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir.
    """
    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if is_dir_url(link):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # If --require-hashes is off, `hashes` is either empty, the
    # link's embedded hash, or MissingHashes; it is required to
    # match. If --require-hashes is on, we are satisfied by any
    # hash in `hashes` matching: a URL-based or an option-based
    # one; no internet-sourced hash will be in `hashes`.
    if hashes:
        hashes.check_against_path(link_path)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link,
                                                      download_dir,
                                                      hashes)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, link) 
Example 69
Project: AshsSDK   Author: thehappydinoa   File: easy_install.py    MIT License 4 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 70
Project: Parallel.GAMIT   Author: demiangomez   File: pyGamitTask.py    GNU General Public License v3.0 4 votes vote down vote up
def finish(self):

        try:
            # delete everything inside the processing dir
            shutil.rmtree(self.pwd_brdc)
            shutil.rmtree(self.pwd_igs)

            # remove files in tables
            for ftype in ['*.grid', '*.dat', '*.apr']:
                for ff in glob.glob(os.path.join(self.pwd_tables, ftype)):
                    os.remove(ff)

            # remove processing files
            for ftype in ['b*', 'cfmrg*', 'DPH.*', 'eq_rename.*', 'g*', 'k*', 'p*', 'rcvant.*', 'y*']:
                for ff in glob.glob(os.path.join(os.path.join(self.pwd, self.date.ddd()), ftype)):
                    os.remove(ff)

            try:
                if not os.path.exists(os.path.dirname(self.solution_pwd)):
                    os.makedirs(os.path.dirname(self.solution_pwd))
            except OSError:
                # racing condition having several processes trying to create the same folder
                # if OSError occurs, ignore and continue
                pass

            # the solution folder exists because it was created by GamitSession to start the processing.
            # erase it to upload the result
            if os.path.exists(self.solution_pwd):
                shutil.rmtree(self.solution_pwd)

            # execute final step: copy to self.solution_pwd
            shutil.copytree(self.pwd, self.solution_pwd, symlinks=True)
            # remove the remote pwd
            shutil.rmtree(self.pwd)

        except Exception:

            msg = traceback.format_exc() + '\nProcessing %s date %s on node %s' \
                  % (self.params['NetName'], self.date.yyyyddd(), platform.node())

            with open(os.path.join(self.pwd, 'monitor.log'), 'a') as monitor:
                monitor.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') +
                              ' -> ERROR in pyGamitTask.finish()\n%s' % msg)

        return 
Example 71
Project: garleek   Author: insilichem   File: test_connectors.py    MIT License 4 votes vote down vote up
def test_gaussian_tinker(directory):
    if directory.endswith('UFF'):
        pytest.skip()
    with temporary_directory() as tmp:
        # Original data
        data_original = os.path.join(data, directory)
        outfile_original = os.path.join(data_original, directory + '.out')
        infile_original = os.path.join(data_original, directory + '.in')
        # Copied tmp paths
        data_copy = os.path.join(tmp, directory)
        infile_copy = os.path.join(data_copy, directory + '.in')
        shutil.copytree(data_original, data_copy)
        os.chdir(data_copy)
        # We are now in /tmp/garleek*****/A_1 or similar, which
        # contains copies of the original Gaussian files and types
        # guess forcefield specified in atom.types
        ff = 'qmmm3.prm'
        types = 'uff_to_mm3'
        if os.path.isfile(directory + '.key'):
            ff = directory + '.key'
        elif os.path.isfile('atom.types'):
            types = 'atom.types'
            with open(types) as f:
                for line in f:
                    if line.startswith('# forcefield:'):
                        ff = line.split(':', 1)[1].strip()
        # patch inputfile
        garleek_in = frontend_garleek(infile_copy, qm='gaussian_'+gaussian_version, mm='tinker', ff=ff, types=types)

        call([gaussian_exe, garleek_in])
        garleek_out = os.path.splitext(garleek_in)[0] + '.log'

        # Save output in working dir
        if not os.path.exists(os.path.join(WORKING_DIR, 'outputs-'+gaussian_version)):
            os.mkdir(os.path.join(WORKING_DIR, 'outputs-'+gaussian_version))
        shutil.copy(garleek_in, os.path.join(WORKING_DIR, 'outputs-'+gaussian_version, os.path.basename(garleek_in)))
        assert os.path.isfile(garleek_out)
        shutil.copy(garleek_out, os.path.join(WORKING_DIR, 'outputs-'+gaussian_version, os.path.basename(garleek_out)))
        assert no_errors(garleek_out)
        # Check values
        assert isclose(oniom_energy(outfile_original), oniom_energy(garleek_out))
        if HAS_CCLIB:
            cc_original = cclib.ccopen(outfile_original).parse()
            cc_calculated = cclib.ccopen(garleek_out).parse()
            assert isclose(cc_original.scfenergies[-1], cc_calculated.scfenergies[-1])
            assert np.sqrt(np.mean(np.square(cc_original.atomcoords[-1]-cc_calculated.atomcoords[-1]))) < 0.001 
Example 72
Project: arma-project-template   Author: acemod   File: make.py    MIT License 4 votes vote down vote up
def copy_optionals_for_building(mod,pbos):
    src_directories = os.listdir(optionals_root)
    current_dir = os.getcwd()

    print_blue("\nChecking optionals folder...")
    try:
        #special server.pbo processing
        files = glob.glob(os.path.join(release_dir, project, "optionals", "*.pbo"))
        for file in files:
            file_name = os.path.basename(file)
            #print ("Adding the following file: {}".format(file_name))
            pbos.append(file_name)
            pbo_path = os.path.join(release_dir, project, "optionals", file_name)
            sigFile_name = file_name +"."+ key_name + ".bisign"
            sig_path = os.path.join(release_dir, project, "optionals", sigFile_name)
            if (os.path.isfile(pbo_path)):
                print("Moving {} for processing.".format(pbo_path))
                shutil.move(pbo_path, os.path.join(release_dir, project, "addons", file_name))

            if (os.path.isfile(sig_path)):
                #print("Moving {} for processing.".format(sig_path))
                shutil.move(sig_path, os.path.join(release_dir, project, "addons", sigFile_name))

    except:
        print_error("Error in moving")
        raise
    finally:
        os.chdir(current_dir)

    try:
        for dir_name in src_directories:
            mod.append(dir_name)
            #userconfig requires special handling since it is not a PBO source folder.
            #CfgConvert fails to build server.pbo if userconfig is not found in P:\
            if (dir_name == "userconfig"):
                if (os.path.exists(os.path.join(release_dir, project, "optionals", dir_name))):
                    shutil.rmtree(os.path.join(release_dir, project, "optionals", dir_name), True)
                shutil.copytree(os.path.join(optionals_root,dir_name), os.path.join(release_dir, project, "optionals", dir_name))
                destination = os.path.join(work_drive,dir_name)
            else:
                destination = os.path.join(module_root,dir_name)

            print("Temporarily copying {} => {} for building.".format(os.path.join(optionals_root,dir_name),destination))
            if (os.path.exists(destination)):
                shutil.rmtree(destination, True)
            shutil.copytree(os.path.join(optionals_root,dir_name), destination)
    except:
        print_error("Copy Optionals Failed")
        raise
    finally:
        os.chdir(current_dir) 
Example 73
Project: Repobot   Author: Desgard   File: download.py    MIT License 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None, hashes=None):
    """Unpack link into location.

    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir.
    """
    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if is_dir_url(link):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # If --require-hashes is off, `hashes` is either empty, the
    # link's embedded hash, or MissingHashes; it is required to
    # match. If --require-hashes is on, we are satisfied by any
    # hash in `hashes` matching: a URL-based or an option-based
    # one; no internet-sourced hash will be in `hashes`.
    if hashes:
        hashes.check_against_path(link_path)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link,
                                                      download_dir,
                                                      hashes)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, link) 
Example 74
Project: Repobot   Author: Desgard   File: easy_install.py    MIT License 4 votes vote down vote up
def install_egg(self, egg_path, tmpdir):
        destination = os.path.join(
            self.install_dir,
            os.path.basename(egg_path),
        )
        destination = os.path.abspath(destination)
        if not self.dry_run:
            ensure_directory(destination)

        dist = self.egg_distribution(egg_path)
        if not samefile(egg_path, destination):
            if os.path.isdir(destination) and not os.path.islink(destination):
                dir_util.remove_tree(destination, dry_run=self.dry_run)
            elif os.path.exists(destination):
                self.execute(
                    os.unlink,
                    (destination,),
                    "Removing " + destination,
                )
            try:
                new_dist_is_zipped = False
                if os.path.isdir(egg_path):
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copytree, "Copying"
                elif self.should_unzip(dist):
                    self.mkpath(destination)
                    f, m = self.unpack_and_compile, "Extracting"
                else:
                    new_dist_is_zipped = True
                    if egg_path.startswith(tmpdir):
                        f, m = shutil.move, "Moving"
                    else:
                        f, m = shutil.copy2, "Copying"
                self.execute(
                    f,
                    (egg_path, destination),
                    (m + " %s to %s") % (
                        os.path.basename(egg_path),
                        os.path.dirname(destination)
                    ),
                )
                update_dist_caches(
                    destination,
                    fix_zipimporter_caches=new_dist_is_zipped,
                )
            except Exception:
                update_dist_caches(destination, fix_zipimporter_caches=False)
                raise

        self.add_output(destination)
        return self.egg_distribution(destination) 
Example 75
Project: inmanta   Author: inmanta   File: dump_tool.py    Apache License 2.0 4 votes vote down vote up
def test_dump_db(server, client, postgres_db, database_name):
    """
        Note: remove following line from the dump: SELECT pg_catalog.set_config('search_path', '', false);

    """

    if False:
        # trick autocomplete to have autocomplete on client
        client = methods

    result = await client.create_project("project-test-a")
    assert result.code == 200
    project_id = result.result["project"]["id"]

    result = await client.create_environment(project_id=project_id, name="dev-1")
    assert result.code == 200
    env_id_1 = result.result["environment"]["id"]

    result = await client.create_environment(project_id=project_id, name="dev-2")
    assert result.code == 200

    project_dir = os.path.join(server.get_slice(SLICE_SERVER)._server_storage["environments"], str(env_id_1))
    project_source = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "simple_project")
    outname = "dbdump"
    print("Project at: ", project_dir)

    shutil.copytree(project_source, project_dir)

    check_result(await client.set_setting(env_id_1, "autostart_agent_deploy_splay_time", 0))
    check_result(await client.set_setting(env_id_1, "autostart_agent_deploy_interval", 0))
    check_result(await client.set_setting(env_id_1, "autostart_agent_repair_splay_time", 0))
    check_result(await client.set_setting(env_id_1, "autostart_agent_repair_interval", 600))

    await client.notify_change(id=env_id_1)

    versions = await wait_for_version(client, env_id_1, 1)
    v1 = versions["versions"][0]["version"]

    await client.release_version(env_id_1, v1, push=True, agent_trigger_method=const.AgentTriggerMethod.push_full_deploy)

    await _wait_until_deployment_finishes(client, env_id_1, v1, 20)

    await client.notify_change(id=env_id_1)

    versions = await wait_for_version(client, env_id_1, 2)
    v2 = versions["versions"][0]["version"]

    await client.release_version(env_id_1, v2, push=True, agent_trigger_method=const.AgentTriggerMethod.push_full_deploy)

    await _wait_until_deployment_finishes(client, env_id_1, v2, 20)

    proc = await asyncio.create_subprocess_exec(
        "pg_dump", "-h", "127.0.0.1", "-p", str(postgres_db.port), "-f", outname, "-O", "-U", postgres_db.user, database_name
    )
    await proc.wait() 
Example 76
Project: JukeBox   Author: gauravsarkar97   File: download.py    MIT License 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None, hashes=None):
    """Unpack link into location.

    If download_dir is provided and link points to a file, make a copy
    of the link file inside download_dir.
    """
    link_path = url_to_path(link.url_without_fragment)

    # If it's a url to a local directory
    if is_dir_url(link):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        if download_dir:
            logger.info('Link is a directory, ignoring download_dir')
        return

    # If --require-hashes is off, `hashes` is either empty, the
    # link's embedded hash, or MissingHashes; it is required to
    # match. If --require-hashes is on, we are satisfied by any
    # hash in `hashes` matching: a URL-based or an option-based
    # one; no internet-sourced hash will be in `hashes`.
    if hashes:
        hashes.check_against_path(link_path)

    # If a download dir is specified, is the file already there and valid?
    already_downloaded_path = None
    if download_dir:
        already_downloaded_path = _check_download_dir(link,
                                                      download_dir,
                                                      hashes)

    if already_downloaded_path:
        from_path = already_downloaded_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded_path:
        _copy_file(from_path, download_dir, link) 
Example 77
Project: recipes-py   Author: luci   File: cmd.py    Apache License 2.0 4 votes vote down vote up
def export_repo(repo, destination):
  """Copies all the recipe-relevant files for the repo to the given
  destination.

  Args:
    * repo (RecipeRepo) - The repo to export.
    * destination (str) - The absolute path we're exporting to (we'll export to
      a subfolder equal to `repo.name`).
  """
  _check(repo, RecipeRepo)
  _check(destination, str)

  bundle_dst = os.path.join(destination, repo.name)

  reldir = repo.simple_cfg.recipes_path
  if reldir:
    reldir += '/'

  args = [
    GIT, '-C', repo.path, 'ls-files', '--',
    ':(attr:recipes)',                       # anything tagged for recipes
    simple_cfg.RECIPES_CFG_LOCATION_REL,     # always grab recipes.cfg
    '%srecipes/**' % reldir,                 # all the recipes stuff
    '%srecipe_modules/**' % reldir,          # all the recipe_modules stuff
    '%srecipe_proto/**.proto' % reldir,      # all the protos in recipe_proto

    # And exclude all the json expectations
    ':(exclude)%s**/*.expected/*.json' % reldir,
  ]
  LOGGER.info('enumerating all recipe files: %r', args)
  to_copy = subprocess.check_output(args).splitlines()
  copy_map = defaultdict(set)
  for i in to_copy:
    if posixpath.sep != os.path.sep:
      i = i.replace(posixpath.sep, os.path.sep)
    while i:
      i, tail = os.path.split(i)
      base = os.path.join(repo.path, i) if i else repo.path
      copy_map[base].add(tail)

  def _ignore_fn(base, items):
    return set(items) - copy_map[base]

  shutil.copytree(repo.path, bundle_dst, ignore=_ignore_fn) 
Example 78
Project: sagemaker-xgboost-container   Author: aws   File: local_mode.py    Apache License 2.0 4 votes vote down vote up
def create_training(data_dir, customer_script, optml, image, additional_volumes,
                    additional_env_vars, additional_hps=None, cluster_size=1, source_dir=None,
                    entrypoint=None, use_gpu=False):

    additional_hps = additional_hps or None

    session = boto3.Session()
    tmpdir = os.path.abspath(optml)

    hosts = create_host_names(cluster_size)
    print('creating hosts: {}'.format(hosts))

    config = create_input_data_config(data_dir)
    hyperparameters = read_hyperparameters(additional_hps)

    if customer_script:
        timestamp = utils.sagemaker_timestamp()
        s3_script_path = fw_utils.tar_and_upload_dir(session=session,
                                                     bucket=default_bucket(session),
                                                     s3_key_prefix='test-{}'.format(timestamp),
                                                     script=customer_script,
                                                     directory=source_dir)[0]
        hyperparameters.update({
            'sagemaker_submit_directory': s3_script_path,
            'sagemaker_program': os.path.basename(customer_script)
        })

    for host in hosts:
        for d in ['input', 'input/config', 'output', 'model']:
            os.makedirs(os.path.join(tmpdir, host, d))

        write_hyperparameters(tmpdir, host, hyperparameters)
        write_resource_config(tmpdir, hosts, host)
        write_inputdataconfig(tmpdir, host, config)

        shutil.copytree(data_dir, os.path.join(tmpdir, host, 'input', 'data'))

    write_docker_file('train', tmpdir, hosts, image, additional_volumes, additional_env_vars,
                      customer_script, source_dir, entrypoint, use_gpu)

    print("training dir: \n{}".format(
        str(subprocess.check_output(['ls', '-l', tmpdir]).decode('utf-8'))))

    return tmpdir 
Example 79
Project: Flask_Blog   Author: sugarguo   File: download.py    GNU General Public License v3.0 4 votes vote down vote up
def unpack_file_url(link, location, download_dir=None):

    link_path = url_to_path(link.url_without_fragment)
    already_downloaded = False

    # If it's a url to a local directory
    if os.path.isdir(link_path):
        if os.path.isdir(location):
            rmtree(location)
        shutil.copytree(link_path, location, symlinks=True)
        return

    # if link has a hash, let's confirm it matches
    if link.hash:
        link_path_hash = _get_hash_from_file(link_path, link)
        _check_hash(link_path_hash, link)

    # If a download dir is specified, is the file already there and valid?
    if download_dir:
        download_path = os.path.join(download_dir, link.filename)
        if os.path.exists(download_path):
            content_type = mimetypes.guess_type(download_path)[0]
            logger.notify('File was already downloaded %s' % download_path)
            if link.hash:
                download_hash = _get_hash_from_file(download_path, link)
                try:
                    _check_hash(download_hash, link)
                    already_downloaded = True
                except HashMismatch:
                    logger.warn(
                        'Previously-downloaded file %s has bad hash, '
                        're-downloading.' % link_path
                        )
                    os.unlink(download_path)
            else:
                already_downloaded = True

    if already_downloaded:
        from_path = download_path
    else:
        from_path = link_path

    content_type = mimetypes.guess_type(from_path)[0]

    # unpack the archive to the build dir location. even when only downloading
    # archives, they have to be unpacked to parse dependencies
    unpack_file(from_path, location, content_type, link)

    # a download dir is specified and not already downloaded
    if download_dir and not already_downloaded:
        _copy_file(from_path, download_dir, content_type, link) 
Example 80
Project: fastclass   Author: cwerner   File: fc_download.py    Apache License 2.0 4 votes vote down vote up
def main(infile: str, size: int, crawler: List[str], keep: bool, maxnum:int, outpath: str):
    SIZE=(size,size)
    classes = []

    if 'ALL' in crawler:
        crawler = ['GOOGLE', 'BING']

    if os.path.isdir(outpath):
        print(f'Directory "{outpath}" exists. Would you like to overwrite the directory? [y/n]')
        choice = input().lower()
        while (not (choice is 'y' or 'n')):
            print("Please reply with 'y' or 'n'")
            choice = input().lower()
        if (choice == 'y'):
            shutil.rmtree(outpath)
            if os.path.isdir(outpath+'.raw'):
                shutil.rmtree(outpath+'.raw')
        else:
            exit(-1)

    os.makedirs(outpath)
    print(f'INFO: final dataset will be located in {outpath}')

    with tempfile.TemporaryDirectory() as tmp:
        for lcnt, line in enumerate(infile):
            if lcnt > 0:
                no_cols = line[:-1].count(',')+1
                if no_cols>1:
                    search_term, remove_terms = line[:-1].split(',')
                else:
                    search_term = line[:-1]
                    remove_terms = None
                classes.append((search_term, remove_terms))
                
        for i, (search_term, remove_terms) in enumerate(classes):
            print(f'[{i+1}/{len(classes)}] Searching: >> {search_term} <<')
            out_name = sanitize_searchstring(search_term, rstring=remove_terms)
            raw_folder = os.path.join(tmp, out_name)

            source_urls = crawl(raw_folder, search_term, maxnum, crawlers=crawler)
            remove_dups(raw_folder)

            # resize
            out_resized = os.path.join(outpath, out_name)
            os.makedirs(out_resized, exist_ok=True)

            files = sorted(glob.glob(raw_folder+'/*'))

            source_urls = resize(files, outpath=out_resized, size=SIZE, urls=source_urls)

            # write report file
            with open(out_resized + '.log', 'w', encoding="utf-8") as log:
                log.write('image,source\n')
                for item in source_urls:
                    log.write(','.join([item, source_urls[item]]) + '\n')

        if keep:
            shutil.copytree(tmp, outpath+'.raw')