Python shutil.unpack_archive() Examples

def train(self, save_data, opt):
        save_data_archive = save_temp_bin(save_data)

        save_data_dir = temp_dir()
        shutil.unpack_archive(filename=save_data_archive, extract_dir=save_data_dir, format="gztar")

        save_model = temp_dir()

        opt["data"] = save_data_dir + "data"
        opt["save_model"] = save_model
        if is_cuda:
            opt["world_size"] = 1
            opt["gpu_ranks"] = 0

        run_param('train.py', opt)

        return save_model 

def get_mathlib_olean(self) -> None:
        """Get precompiled mathlib oleans for this project."""
        # Just in case the user broke the workflow (for instance git clone
        # mathlib by hand and then run `leanproject get-cache`)
        if not (self.directory/'leanpkg.path').exists():
            self.run(['leanpkg', 'configure'])
        try:
            archive = get_mathlib_archive(self.mathlib_rev, self.cache_url,
                                           self.force_download, self.repo)
        except (EOFError, shutil.ReadError):
            log.info('Something wrong happened with the olean archive. '
                     'I will now retry downloading.')
            archive = get_mathlib_archive(self.mathlib_rev, self.cache_url,
                                          True, self.repo)
        self.clean_mathlib()
        self.mathlib_folder.mkdir(parents=True, exist_ok=True)
        unpack_archive(archive, self.mathlib_folder)
        # Let's now touch oleans, just in case
        touch_oleans(self.mathlib_folder) 

def currentdir(self, directory: Optional[str]) -> None:
        if directory is None:
            self._currentdir = None
        else:
            dirpath = os.path.join(self.basepath, directory)
            zippath = f'{dirpath}.zip'
            if os.path.exists(zippath):
                shutil.unpack_archive(
                    filename=zippath,
                    extract_dir=os.path.join(self.basepath, directory),
                    format='zip',
                )
                os.remove(zippath)
            elif not os.path.exists(dirpath):
                os.makedirs(dirpath)
            self._currentdir = str(directory) 

def untar(path, fname, deleteTar=True):
    """
    Unpack the given archive file to the same directory.

    :param str path:
        The folder containing the archive. Will contain the contents.

    :param str fname:
        The filename of the archive file.

    :param bool deleteTar:
        If true, the archive will be deleted after extraction.
    """
    logging.debug(f'unpacking {fname}')
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath) 

def _restore_from_backup(self, src_ver: str):
        logger.info('Restoring from backup for {}'.format(src_ver))
        for file_path in self.files_to_preserve:
            try:
                shutil.copy2(os.path.join(self.backup_target, file_path),
                             os.path.join(self.tmp_dir, file_path))
            except IOError as e:
                logger.warning('Copying {} failed due to {}'
                               .format(file_path, e))
        shutil.unpack_archive(self._backup_name_ext(
            src_ver), self.backup_target, self.backup_format)
        for file_path in self.files_to_preserve:
            try:
                shutil.copy2(os.path.join(self.tmp_dir, file_path),
                             os.path.join(self.backup_target, file_path))
            except IOError as e:
                logger.warning('Copying {} failed due to {}'
                               .format(file_path, e))
        shutil.rmtree(self.tmp_dir, ignore_errors=True) 

def test_cached_download(self, ts=httptest.NoServer()):
        with tempfile.TemporaryDirectory() as tempdir:

            @cached_download(
                ts.url() + "/archive.tar.gz",
                pathlib.Path(tempdir) / "archive.tar.gz",
                ARCHIVE_HASH,
                protocol_allowlist=["http://"],
            )
            async def func(filename):
                return filename

            # Directory to extract to
            extracted = pathlib.Path(tempdir, "extracted")

            # Unpack the archive
            shutil.unpack_archive(await func(), extracted)

            self.verify_extracted_contents(extracted) 

def untar(path, fname, deleteTar=True):
    """
    Unpack the given archive file to the same directory.

    :param str path:
        The folder containing the archive. Will contain the contents.

    :param str fname:
        The filename of the archive file.

    :param bool deleteTar:
        If true, the archive will be deleted after extraction.
    """
    print('unpacking ' + fname)
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath) 

def test_register_unpack_archive(tmp_path):
    shutil.register_unpack_format('7zip', ['.7z'], unpack_7zarchive)
    shutil.unpack_archive(str(testdata_path.joinpath('test_1.7z')), str(tmp_path))
    target = tmp_path.joinpath("setup.cfg")
    expected_mode = 33188
    expected_mtime = 1552522033
    if os.name == 'posix':
        assert target.stat().st_mode == expected_mode
    assert target.stat().st_mtime == expected_mtime
    m = hashlib.sha256()
    m.update(target.open('rb').read())
    assert m.digest() == binascii.unhexlify('ff77878e070c4ba52732b0c847b5a055a7c454731939c3217db4a7fb4a1e7240')
    m = hashlib.sha256()
    m.update(tmp_path.joinpath('setup.py').open('rb').read())
    assert m.digest() == binascii.unhexlify('b916eed2a4ee4e48c51a2b51d07d450de0be4dbb83d20e67f6fd166ff7921e49')
    m = hashlib.sha256()
    m.update(tmp_path.joinpath('scripts/py7zr').open('rb').read())
    assert m.digest() == binascii.unhexlify('b0385e71d6a07eb692f5fb9798e9d33aaf87be7dfff936fd2473eab2a593d4fd') 

def _download_and_extract_model_tar_gz(self, model_id):
        """
        This function first gets the s3 location from dynamo db,
        downloads the model, extracts it and then
        returns a tuple (str, str) of metadata string and model weights URL on disk
        """
        deployable_model_id_record = self.model_ddb_wrapper.get_model_record(experiment_id=self.experiment_id,
                                                                             model_id=model_id)
        s3_uri = deployable_model_id_record.get("s3_model_output_path", "")
        if s3_uri:
            try:
                tmp_dir = Path(f"/opt/ml/downloads/{gen_random_string()}")
                tmp_dir.mkdir(parents=True, exist_ok=True)
                tmp_model_tar_gz = os.path.join(tmp_dir.as_posix(), "model.tar.gz")
                bucket, key = parse_s3_url(s3_uri)
                self.s3_resource.Bucket(bucket).download_file(key, tmp_model_tar_gz)
                shutil.unpack_archive(filename=tmp_model_tar_gz, extract_dir=tmp_dir.as_posix())
                return self.get_model(tmp_dir.as_posix())
            except Exception as e:
                logger.exception(f"Could not parse or download {model_id} from {s3_uri} due to {e}")
                return None
        else:
            logger.exception(f"Could not s3 location of {model_id}")
            return None 

def download(_, target, droot, __):
    url = target["url"]
    fname = target.get("target", url.split("/")[-1])

    r = requests.get(
        url,
        stream=True,
        headers={
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_4) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/12.1 Safari/605.1.15"
        },
    )
    chars = "-\\|/"
    with open(f"{droot}/{fname}", "wb") as f:
        for i, chunk in enumerate(r.iter_content(chunk_size=1024)):
            arrow(f"Downloading... {chars[i%len(chars)]}", end="\r")
            if chunk:
                f.write(chunk)

    if fname.endswith(".zip") or fname.endswith(".tar.gz"):
        arrow(f"Unpacking {fname}...")
        shutil.unpack_archive(f"{droot}/{fname}", droot) 

def untar(path, fname, deleteTar=True):
    """
    Unpacks the given archive file to the same directory, then (by default)
    deletes the archive file.
    """
    print('unpacking ' + fname)
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath) 

def update_pdfjs(target_version=None):
    """Download and extract the latest pdf.js version.

    If target_version is not None, download the given version instead.

    Args:
        target_version: None or version string ('x.y.z')
    """
    if target_version is None:
        version, url = get_latest_pdfjs_url()
    else:
        # We need target_version as x.y.z, without the 'v' prefix, though the
        # user might give it on the command line
        if target_version.startswith('v'):
            target_version = target_version[1:]
        # version should have the prefix to be consistent with the return value
        # of get_latest_pdfjs_url()
        version = 'v' + target_version
        url = ('https://github.com/mozilla/pdf.js/releases/download/'
               'v{0}/pdfjs-{0}-dist.zip').format(target_version)

    os.chdir(os.path.join(os.path.dirname(os.path.abspath(__file__)),
                          '..', '..'))
    target_path = os.path.join('qutebrowser', '3rdparty', 'pdfjs')
    print("=> Downloading pdf.js {}".format(version))
    try:
        (archive_path, _headers) = urllib.request.urlretrieve(url)
    except urllib.error.HTTPError as error:
        print("Could not retrieve pdfjs {}: {}".format(version, error))
        return
    if os.path.isdir(target_path):
        print("Removing old version in {}".format(target_path))
        shutil.rmtree(target_path)
    os.makedirs(target_path)
    print("Extracting new version")
    shutil.unpack_archive(archive_path, target_path, 'zip')
    urllib.request.urlcleanup() 

def _extract_modules(self, dep, archive_path: Path, output_path: Path) -> bool:
        # say to shutils that wheel can be parsed as zip
        if 'wheel' not in shutil._UNPACK_FORMATS:  # type: ignore
            shutil.register_unpack_format(
                name='wheel',
                extensions=['.whl'],
                function=shutil._unpack_zipfile,  # type: ignore
            )

        with TemporaryDirectory(suffix=dep.name) as package_path:  # type: Path # type: ignore
            package_path = Path(package_path)
            shutil.unpack_archive(str(archive_path), str(package_path))
            if len(list(package_path.iterdir())) == 1:
                package_path = next(package_path.iterdir())

            # find modules
            root = PackageRoot(name=dep.name, path=package_path)
            if not root.packages:
                self.logger.error('cannot find modules', extra=dict(
                    dependency=dep.name,
                    version=dep.group.best_release.version,
                ))
                return False

            # copy modules
            module_path = root.packages[0].path
            module_name = root.packages[0].module
            self.logger.info('copying module...', extra=dict(
                path=str(module_path.relative_to(package_path)),
                dependency=dep.name,
            ))
            shutil.copytree(
                src=str(module_path),
                dst=str(output_path.joinpath(*module_name.split('.'))),
            )
            return True 

def load(filename: str):
        """
        Load model from NodeEmbedding model zip file.
        
        filename : str
            full filename of file to load (including extensions)
            The file should be the result of a `save()` call
            
        Loading checks for metadata and raises warnings if pkg versions
        are different than they were when saving the model.
        """
        with tempfile.TemporaryDirectory() as temp_dir:
            shutil.unpack_archive(filename, temp_dir, 'zip')
            model = joblib.load(os.path.join(temp_dir, BaseNodeEmbedder.f_model))
            with open(os.path.join(temp_dir, BaseNodeEmbedder.f_mdata)) as f:
                meta_data = json.load(f)
            # Validate the metadata
            sysverinfo = sys.version_info
            pyver = "{0}.{1}".format(sysverinfo[0], sysverinfo[1])
            if meta_data["python_"] != pyver:
                raise UserWarning(
                    "Invalid python version; {0}, required: {1}".format(
                        pyver, meta_data["python_"]))
            sklver = sklearn.__version__[:-2]
            if meta_data["skl_"] != sklver:
                raise UserWarning(
                    "Invalid sklearn version; {0}, required: {1}".format(
                        sklver, meta_data["skl_"]))
            pdver = pd.__version__[:-2]
            if meta_data["pd_"] != pdver:
                raise UserWarning(
                    "Invalid pandas version; {0}, required: {1}".format(
                        pdver, meta_data["pd_"]))
            csrv = cg.__version__[:-2]
            if meta_data["csrg_"] != csrv:
                raise UserWarning(
                    "Invalid csrgraph version; {0}, required: {1}".format(
                        csrv, meta_data["csrg_"]))
        return model 

def extract(cls, download_path, extract_dir):
        extracted_files = [
            join(extract_dir, f) for f in cls.files]

        if any(not exists(p) for p in extracted_files):
            shutil.unpack_archive(
                download_path, extract_dir)

        return extracted_files 

def unpack_file(file_path, target_path, logger=None):
    if logger is None:
        printf = print
    else:
        printf = logger.info

    file_path = Path(file_path)
    assert file_path.exists() and file_path.is_file()
    assert os.path.exists(target_path) and os.path.isdir(target_path)
    printf('Begin extract file from ' + str(file_path) + ' to ' + str(target_path))
    shutil.unpack_archive(str(file_path), target_path)
    printf('Extract Finish') 

def download(args):
    base_dir = os.path.dirname(os.path.abspath(__file__))
    temp_dir = args.temp_directory
    if not args.temp_directory:
        temp_dir = base_dir
    os.makedirs(base_dir, exist_ok=True)
    if sys.platform == 'darwin':
        plat = 'macosx'
    elif sys.platform == 'win32':
        plat = 'win64'
    else:
        plat = 'linux'
    print('Downloading precompiled binaries for {}...'.format(plat))

    download_link = 'http://mlmlab.org/mfa/precompiled_binaries/mfa_thirdparty_{}.zip'.format(
        plat)
    path = os.path.join(temp_dir, '{}.zip'.format(plat))
    if args.redownload or not os.path.exists(path):
        with tqdm(unit='B', unit_scale=True, miniters=1) as t:
            filename, headers = urlretrieve(download_link, path, reporthook=tqdm_hook(t), data=None)
    shutil.unpack_archive(path, base_dir)
    if not args.keep:
        os.remove(path)
    if plat != 'win':
        import stat
        bin_dir = os.path.join(base_dir, 'bin')
        for f in os.listdir(bin_dir):
            if '.' in f:
                continue
            os.chmod(os.path.join(bin_dir, f), stat.S_IEXEC | stat.S_IWUSR | stat.S_IRUSR)
    return True 

def unpack_archive(fname: Union[str, Path], tgt_dir: Union[str, Path]) -> None:
    """Unpack archive. This is needed for python < 3.7."""
    shutil.unpack_archive(str(fname), str(tgt_dir)) 

def get_cache(self, force: bool = False, url:str = '') -> None:
        """Tries to get olean cache.

        Will raise LeanDownloadError or FileNotFoundError if no archive exists.
        """
        if self.is_dirty and not force:
            raise LeanDirtyRepo
        if self.is_mathlib:
            self.get_mathlib_olean()
        else:
            unpack_archive(self.directory/'_cache'/(str(self.rev)+'.tar.bz2'),
                           self.directory) 

def unarchive(local_path, base_dir):
    """
    Unarchive zipped file
    """
    shutil.unpack_archive(local_path, base_dir) 

def _unpack_feed(path: str, view: View, config: nx.DiGraph) -> Feed:
    tmpdir = tempfile.mkdtemp()
    shutil.unpack_archive(path, tmpdir)
    feed: Feed = _load_feed(tmpdir, view, config)

    # Eager cleanup
    feed._delete_after_reading = True

    def finalize() -> None:
        shutil.rmtree(tmpdir)

    # Lazy cleanup
    weakref.finalize(feed, finalize)

    return feed 

def unpack_model(self, archive_file_path: str):
        if os.path.exists(self.model_path):
            raise Exception("Model {} {} local cache already existed".format(self.model_id, self.model_version))
        shutil.unpack_archive(archive_file_path, self.model_path)
        stat_logger.info("Unpack model archive to {}".format(self.model_path)) 

def pypi_package_contents(self, url: str) -> str:
    """
    Download a source code release and extract it to a temporary directory.
    """
    package_src_dir = tempfile.mkdtemp(prefix="pypi-")
    async with self.parent.session.get(url) as resp:
        # Create a temporary file to extract to
        with tempfile.NamedTemporaryFile(
            prefix="pypi-", suffix=".tar.gz"
        ) as package_src_file:
            package_src_file.write(await resp.read())
            shutil.unpack_archive(package_src_file.name, package_src_dir)
            return {"directory": package_src_dir} 

def load(self, path):
        file_path = os.path.join(path, self.model_dir_name + self.ext)

        with tempfile.TemporaryDirectory(prefix='ebonite_tf_v2') as tmpdir:
            shutil.unpack_archive(file_path, tmpdir)
            return tf.keras.models.load_model(tmpdir) 

def untar(path, fname, deleteTar=True):
    """
    Unpacks the given archive file to the same directory, then (by default)
    deletes the archive file.
    """
    print('unpacking ' + fname)
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath) 

def download(self, dest_root):
        """
        Downloads archive and extracts audio files.

        Parameters
        ----------
        dest_root : Path
            the root path where directories
            that contains audio files are placed.
        """
        with TemporaryDirectory() as working_dir:
            working_dir = Path(working_dir) # convert from str

            # download archive and extract files in the working directory.
            if self.user_option.verbose:
                print("Downloading", self.name, "from", self.src_url, "...")
            archive_path = DataArchive._download_file(
                self.src_url, working_dir
            )
            if self.user_option.verbose:
                print("Unpack:", archive_path)
            shutil.unpack_archive(str(archive_path), str(working_dir))

            # move audio files to the destination directory.
            self._move_all_audio(
                working_dir / self.audio_root_relative, dest_root
            ) 

def unzip(self,zip_path,extract_dir):
		import shutil
		shutil.unpack_archive(zip_path,extract_dir)
		os.remove(zip_path)
		self.status='Unzip Finished' 

def download_and_extract(buildpath, packagedir, pkg, args):
    srcpath = buildpath / packagedir

    if "source" not in pkg:
        return srcpath

    if "url" in pkg["source"]:
        tarballpath = buildpath / Path(pkg["source"]["url"]).name
        if not tarballpath.is_file():
            try:
                subprocess.run(
                    ["wget", "-q", "-O", str(tarballpath), pkg["source"]["url"]],
                    check=True,
                )
                check_checksum(tarballpath, pkg)
            except Exception:
                tarballpath.unlink()
                raise

        if not srcpath.is_dir():
            shutil.unpack_archive(str(tarballpath), str(buildpath))

    elif "path" in pkg["source"]:
        srcdir = Path(pkg["source"]["path"])

        if not srcdir.is_dir():
            raise ValueError("'path' must point to a path")

        if not srcpath.is_dir():
            shutil.copytree(srcdir, srcpath)
    else:
        raise ValueError("Incorrect source provided")

    return srcpath 

def untar(path, fname, deleteTar=True):
    """
    Unpacks the given archive file to the same directory, then (by default)
    deletes the archive file.
    """
    print('unpacking ' + fname)
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath) 

def untar(path, fname, deleteTar=True):
    """
    Unpacks the given archive file to the same directory, then (by default)
    deletes the archive file.
    """
    print('unpacking ' + fname)
    fullpath = os.path.join(path, fname)
    shutil.unpack_archive(fullpath, path)
    if deleteTar:
        os.remove(fullpath)