Python shutil.copyfileobj() Examples

The following are 30 code examples for showing how to use shutil.copyfileobj(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may check out the related API usage on the sidebar.

You may also want to check out all available functions/classes of the module shutil , or try the search function .

Example 1
Project: CyberTK-Self   Author: CyberTKR   File: LineApi.py    License: GNU General Public License v2.0 7 votes vote down vote up
def sendImageWithUrl(self, to_, url):
        """Send a image with given image url

        :param url: image url to send
        """
        path = '%s/pythonLine-%1.data' % (tempfile.gettempdir(), randint(0, 9))


        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'w') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download image failure.')

        try:
            self.sendImage(to_, path)
        except Exception as e:
            raise e 
Example 2
Project: macops   Author: google   File: can_haz_image.py    License: Apache License 2.0 7 votes vote down vote up
def DownloadFile(self, fileurl, dlfile):
    """Downloads a given file to a given path/filename.

    Args:
      fileurl: String with URL of file to download.
      dlfile: String with path of file to be written to.
    Raises:
      OSError: If file cannot be opened/written to, function raises OSError.
      URLError: If URL cannot be opened, fucntion raises URLError.
    """
    if not os.path.isfile(dlfile) or dlfile == TMPINDEX:
      print 'Downloading %s ...' % fileurl
      file_to_dl = urllib2.urlopen(fileurl)
      tmpfile = open(dlfile, 'wb')
      shutil.copyfileobj(file_to_dl, tmpfile)
    else:
      print '%s exists' % dlfile 
Example 3
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: text.py    License: Apache License 2.0 6 votes vote down vote up
def _get_data(self):
        archive_file_name, archive_hash = self._archive_file
        data_file_name, data_hash = self._data_file[self._segment]
        path = os.path.join(self._root, data_file_name)
        if not os.path.exists(path) or not check_sha1(path, data_hash):
            namespace = 'gluon/dataset/'+self._namespace
            downloaded_file_path = download(_get_repo_file_url(namespace, archive_file_name),
                                            path=self._root,
                                            sha1_hash=archive_hash)

            with zipfile.ZipFile(downloaded_file_path, 'r') as zf:
                for member in zf.namelist():
                    filename = os.path.basename(member)
                    if filename:
                        dest = os.path.join(self._root, filename)
                        with zf.open(member) as source, \
                             open(dest, "wb") as target:
                            shutil.copyfileobj(source, target)

        data, label = self._read_batch(path)

        self._data = nd.array(data, dtype=data.dtype).reshape((-1, self._seq_len))
        self._label = nd.array(label, dtype=label.dtype).reshape((-1, self._seq_len)) 
Example 4
Project: dockerfiles   Author: floydhub   File: dataset.py    License: Apache License 2.0 6 votes vote down vote up
def download(directory, filename):
  """Download (and unzip) a file from the MNIST dataset if not already done."""
  filepath = os.path.join(directory, filename)
  if tf.gfile.Exists(filepath):
    return filepath
  if not tf.gfile.Exists(directory):
    tf.gfile.MakeDirs(directory)
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
  _, zipped_filepath = tempfile.mkstemp(suffix='.gz')
  print('Downloading %s to %s' % (url, zipped_filepath))
  urllib.request.urlretrieve(url, zipped_filepath)
  with gzip.open(zipped_filepath, 'rb') as f_in, \
      tf.gfile.Open(filepath, 'wb') as f_out:
    shutil.copyfileobj(f_in, f_out)
  os.remove(zipped_filepath)
  return filepath 
Example 5
Project: dockerfiles   Author: floydhub   File: dataset.py    License: Apache License 2.0 6 votes vote down vote up
def download(directory, filename):
  """Download (and unzip) a file from the MNIST dataset if not already done."""
  filepath = os.path.join(directory, filename)
  if tf.gfile.Exists(filepath):
    return filepath
  if not tf.gfile.Exists(directory):
    tf.gfile.MakeDirs(directory)
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
  _, zipped_filepath = tempfile.mkstemp(suffix='.gz')
  print('Downloading %s to %s' % (url, zipped_filepath))
  urllib.request.urlretrieve(url, zipped_filepath)
  with gzip.open(zipped_filepath, 'rb') as f_in, \
      tf.gfile.Open(filepath, 'wb') as f_out:
    shutil.copyfileobj(f_in, f_out)
  os.remove(zipped_filepath)
  return filepath 
Example 6
Project: JJMumbleBot   Author: DuckBoss   File: image_helper.py    License: GNU General Public License v3.0 6 votes vote down vote up
def download_image_requests_to_dir(img_url, dir_name):
    dir_utils.clear_directory(f'{dir_utils.get_temp_med_dir()}/{dir_name}')
    img_ext = img_url.rsplit('.', 1)[1]
    s = requests.Session()
    r = s.get(img_url)
    if r.status_code == 200:
        with open(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.{img_ext}", 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)
        dprint(f"Downloaded image from: {img_url}")
    else:
        dprint(f"{r.status_code} Error! - {img_url}")
    if img_ext == 'png':
        dprint(f"Fixing image to force jpg conversion: {img_url}")
        img_fix = Image.open(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.{img_ext}")
        img_fix.convert('RGB').save(f"{dir_utils.get_temp_med_dir()}/{dir_name}/_image.jpg")
        dir_utils.remove_file("_image.png", f'{dir_utils.get_temp_med_dir()}/{dir_name}')
    dprint(f"Downloaded image from: {img_url}") 
Example 7
Project: jawfish   Author: war-and-code   File: tarfile.py    License: MIT License 6 votes vote down vote up
def copyfileobj(src, dst, length=None):
    """Copy length bytes from fileobj src to fileobj dst.
       If length is None, copy the entire content.
    """
    if length == 0:
        return
    if length is None:
        shutil.copyfileobj(src, dst)
        return

    BUFSIZE = 16 * 1024
    blocks, remainder = divmod(length, BUFSIZE)
    for b in range(blocks):
        buf = src.read(BUFSIZE)
        if len(buf) < BUFSIZE:
            raise IOError("end of file reached")
        dst.write(buf)

    if remainder != 0:
        buf = src.read(remainder)
        if len(buf) < remainder:
            raise IOError("end of file reached")
        dst.write(buf)
    return 
Example 8
Project: jawfish   Author: war-and-code   File: tarfile.py    License: MIT License 6 votes vote down vote up
def addfile(self, tarinfo, fileobj=None):
        """Add the TarInfo object `tarinfo' to the archive. If `fileobj' is
           given, tarinfo.size bytes are read from it and added to the archive.
           You can create TarInfo objects using gettarinfo().
           On Windows platforms, `fileobj' should always be opened with mode
           'rb' to avoid irritation about the file size.
        """
        self._check("aw")

        tarinfo = copy.copy(tarinfo)

        buf = tarinfo.tobuf(self.format, self.encoding, self.errors)
        self.fileobj.write(buf)
        self.offset += len(buf)

        # If there's data to follow, append it.
        if fileobj is not None:
            copyfileobj(fileobj, self.fileobj, tarinfo.size)
            blocks, remainder = divmod(tarinfo.size, BLOCKSIZE)
            if remainder > 0:
                self.fileobj.write(NUL * (BLOCKSIZE - remainder))
                blocks += 1
            self.offset += blocks * BLOCKSIZE

        self.members.append(tarinfo) 
Example 9
Project: fetchLandsatSentinelFromGoogleCloud   Author: vascobnunes   File: fels.py    License: MIT License 6 votes vote down vote up
def download_metadata_file(url, outputdir, program):
    """Download and unzip the catalogue files."""
    zipped_index_path = os.path.join(outputdir, 'index_' + program + '.csv.gz')
    if not os.path.isfile(zipped_index_path):
        if not os.path.exists(os.path.dirname(zipped_index_path)):
            os.makedirs(os.path.dirname(zipped_index_path))
        print("Downloading Metadata file...")
        content = urlopen(url)
        with open(zipped_index_path, 'wb') as f:
            shutil.copyfileobj(content, f)
    index_path = os.path.join(outputdir, 'index_' + program + '.csv')
    if not os.path.isfile(index_path):
        print("Unzipping Metadata file...")
        with gzip.open(zipped_index_path) as gzip_index, open(index_path, 'wb') as f:
            shutil.copyfileobj(gzip_index, f)
    return index_path 
Example 10
Project: snowflake-connector-python   Author: snowflakedb   File: file_util.py    License: Apache License 2.0 6 votes vote down vote up
def compress_file_with_gzip(file_name, tmp_dir):
        """Compresses a file with GZIP.

        Args:
            file_name: Local path to file to be compressed.
            tmp_dir: Temporary directory where an GZIP file will be created.

        Returns:
            A tuple of gzip file name and size.
        """
        logger = getLogger(__name__)
        base_name = os.path.basename(file_name)
        gzip_file_name = os.path.join(tmp_dir, base_name + '_c.gz')
        logger.debug('gzip file: %s, original file: %s', gzip_file_name,
                     file_name)
        fr = open(file_name, 'rb')
        fw = gzip.GzipFile(gzip_file_name, 'wb')
        shutil.copyfileobj(fr, fw)
        fw.close()
        fr.close()
        SnowflakeFileUtil.normalize_gzip_header(gzip_file_name)

        statinfo = os.stat(gzip_file_name)
        return gzip_file_name, statinfo.st_size 
Example 11
Project: pulseaudio-dlna   Author: masmu   File: chromecast-beam.py    License: GNU General Public License v3.0 6 votes vote down vote up
def do_GET(self):
        client_address = self.client_address[0]
        logger.info('Serving transcoded media file to {} ...'.format(
            client_address))

        self.send_head()
        path = self.translate_path(self.path)
        command = VLCEncoderSettings.command(path)
        logger.info('Launching {}'.format(command))

        try:
            with open(os.devnull, 'w') as dev_null:
                encoder_process = subprocess.Popen(
                    command, stdout=subprocess.PIPE, stderr=dev_null)
                shutil.copyfileobj(encoder_process.stdout, self.wfile)
        except:
            logger.info('Connection from {} closed.'.format(client_address))
            logger.debug(traceback.format_exc())
        finally:
            pid = encoder_process.pid
            logger.info('Terminating process {}'.format(pid))
            try:
                os.kill(pid, signal.SIGKILL)
            except:
                pass 
Example 12
Project: dpk   Author: arifistifik   File: talk.py    License: GNU General Public License v3.0 6 votes vote down vote up
def sendImageWithURL2(self, to, url):
        """Send a image with given image url

        :param url: image url to send
        """
        path = 'tmp/pythonLine.data'

        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download image failure.')

        try:
            self.sendImage(to, path)
        except Exception as e:
            raise e 
Example 13
Project: typhon   Author: atmtools   File: topography.py    License: MIT License 6 votes vote down vote up
def download_tile(name):
        """
        This function will download and extract the tile with the given name.
        The data is stored in the path pointed to by the :code:`_data_path`
        attribute of the module.

        Args:
            name(str): The name of the tile to download.
        """
        base_url = "https://dds.cr.usgs.gov/srtm/version2_1/SRTM30"
        url = base_url + "/" + name + "/" + name + ".dem.zip"
        r = urllib.request.urlopen(url)

        filename = os.path.join(_get_data_path(), name + ".dem.zip")
        path = os.path.join(filename)
        with open(path, 'wb') as f:
            shutil.copyfileobj(r, f)

        # Extract zip file.
        with zipfile.ZipFile(filename, "r") as zip_ref:
            zip_ref.extractall(os.path.dirname(filename)) 
Example 14
Project: stdpopsim   Author: popsim-consortium   File: cli.py    License: GNU General Public License v3.0 6 votes vote down vote up
def write_output(ts, args):
    """
    Adds provenance information to the specified tree sequence (ensuring that the
    output is reproducible) and write the resulting tree sequence to output.
    """
    tables = ts.dump_tables()
    logger.debug("Updating provenance")
    provenance = get_provenance_dict()
    tables.provenances.add_row(json.dumps(provenance))
    ts = tables.tree_sequence()
    if args.output is None:
        # There's no way to get tskit to write directly to stdout, so we write
        # to a tempfile first.
        with tempfile.TemporaryDirectory() as tmpdir:
            tmpfile = pathlib.Path(tmpdir) / "tmp.trees"
            ts.dump(tmpfile)
            with open(tmpfile, "rb") as f:
                shutil.copyfileobj(f, sys.stdout.buffer)
    else:
        logger.debug(f"Writing to {args.output}")
        ts.dump(args.output) 
Example 15
Project: pyaff4   Author: aff4   File: aff4.py    License: Apache License 2.0 6 votes vote down vote up
def extractFromVolume(container_urn, volume, imageURNs, destFolder):
    printVolumeInfo(container_urn.original_filename, volume)
    resolver = volume.resolver
    for imageUrn in imageURNs:
        imageUrn = utils.SmartUnicode(imageUrn)

        pathName = next(resolver.QuerySubjectPredicate(volume.urn, imageUrn, volume.lexicon.pathName))

        with resolver.AFF4FactoryOpen(imageUrn) as srcStream:
            if destFolder != "-":
                pathName = escaping.arnPathFragment_from_path(pathName.value)
                while pathName.startswith("/"):
                    pathName = pathName[1:]
                destFile = os.path.join(destFolder, pathName)
                if not os.path.exists(os.path.dirname(destFile)):
                    try:
                        os.makedirs(os.path.dirname(destFile))
                    except OSError as exc:  # Guard against race condition
                        if exc.errno != errno.EEXIST:
                            raise
                with open(destFile, "wb") as destStream:
                    shutil.copyfileobj(srcStream, destStream, length=32 * 2014)
                    print("\tExtracted %s to %s" % (pathName, destFile))
            else:
                shutil.copyfileobj(srcStream, sys.stdout) 
Example 16
Project: aetros-cli   Author: aetros   File: auto_dataset.py    License: MIT License 6 votes vote down vote up
def download_image(url, path):
    if os.path.exists(path):
        return True

    headers = {
        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.101 Safari/537.36'
    }
    try:

        r = requests.get(url, stream=True, timeout=9, headers=headers)
        if r.status_code == 200:
            with open(path, 'wb') as f:
                r.raw.decode_content = True
                shutil.copyfileobj(r.raw, f)
                return True
        else:
            print(("Could not download image %s, response %d" % (url, r.status_code)))
    except Exception as e:
        if hasattr(e, 'message'):
            print(("Could not download image %s due to %s" % (url, e.message)))
        else:
            print(("Could not download image %s due to %s" % (url, repr(e))))

    return False 
Example 17
Project: cloudml-samples   Author: GoogleCloudPlatform   File: dataset.py    License: Apache License 2.0 6 votes vote down vote up
def download(directory, filename):
  """Download (and unzip) a file from the MNIST dataset if not already done."""
  filepath = os.path.join(directory, filename)
  if tf.gfile.Exists(filepath):
    return filepath
  if not tf.gfile.Exists(directory):
    tf.gfile.MakeDirs(directory)
  # CVDF mirror of http://yann.lecun.com/exdb/mnist/
  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'
  _, zipped_filepath = tempfile.mkstemp(suffix='.gz')
  print('Downloading %s to %s' % (url, zipped_filepath))
  urllib.request.urlretrieve(url, zipped_filepath)
  with gzip.open(zipped_filepath, 'rb') as f_in, \
      tf.gfile.Open(filepath, 'wb') as f_out:
    shutil.copyfileobj(f_in, f_out)
  os.remove(zipped_filepath)
  return filepath 
Example 18
Project: BASS   Author: Cisco-Talos   File: bindiff.py    License: GNU General Public License v2.0 5 votes vote down vote up
def bindiff_pickle_export(self, sample, is_64_bit = True, timeout = None):
        """
        Load a sample into IDA Pro, perform autoanalysis and export a pickle file. 
        :param sample: The sample's path
        :param is_64_bit: If the sample needs to be analyzed by the 64 bit version of IDA
        :param timeout: Timeout for the analysis in seconds
        :return: The file name of the exported pickle database. The file needs
        to be deleted by the caller. Returns None on error.
        """

        data_to_send = {
            "timeout": timeout,
            "is_64_bit": is_64_bit}
        url = "%s/binexport_pickle" % next(self._urls)
        log.debug("curl -XPOST --data '%s' '%s'", json.dumps(data_to_send), url)
        response = requests.post(url, data = data_to_send, files = {os.path.basename(sample): open(sample, "rb")})
        if response.status_code == 200:
            handle_tar, path_tar = tempfile.mkstemp(suffix = ".tar.gz")
            with os.fdopen(handle_tar, "wb") as f:
                map(f.write, response.iter_content(1024))
            directory = tempfile.mkdtemp()
            subprocess.check_call(["tar", "xf", path_tar], cwd = directory)

            handle_bindiff, output_bindiff = tempfile.mkstemp(suffix = ".BinExport")
            with os.fdopen(handle_bindiff, "wb") as f:
                with open(os.path.join(directory, "output.BinExport"), "rb") as f2:
                    shutil.copyfileobj(f2, f)
            handle_pickle, output_pickle = tempfile.mkstemp(suffix = ".pickle")
            with os.fdopen(handle_pickle, "wb") as f:
                with open(os.path.join(directory, "output.pickle"), "rb") as f2:
                    shutil.copyfileobj(f2, f)
            os.unlink(path_tar)
            shutil.rmtree(directory)
            return output_bindiff, output_pickle
        else:
            log.error("Bindiff server responded with status code %d: %s", response.status_code, response.content)
            return None 
Example 19
Project: dustmaps   Author: gregreen   File: fetch_utils.py    License: GNU General Public License v2.0 5 votes vote down vote up
def download(url, fname=None):
    """
    Downloads a file.

    Args:
        url (str): The URL to download.
        fname (Optional[str]): The filename to store the downloaded file in. If
            `None`, take the filename from the URL. Defaults to `None`.

    Returns:
          The filename the URL was downloaded to.

    Raises:
        requests.exceptions.HTTPError: There was a problem connecting to the
            URL.
    """
    # Determine the filename
    if fname is None:
        fname = url.split('/')[-1]

    # Stream the URL as a file, copying to local disk
    with contextlib.closing(requests.get(url, stream=True)) as r:
        try:
            r.raise_for_status()
        except requests.exceptions.HTTPError as error:
            print('Error connecting to URL: "{}"'.format(url))
            print(r.text)
            raise error

        with open(fname, 'wb') as f:
            shutil.copyfileobj(r.raw, f)

    return fname 
Example 20
Project: deep-learning-note   Author: wdxtub   File: utils.py    License: MIT License 5 votes vote down vote up
def download_one_file(download_url,
                      local_dest,
                      expected_byte=None,
                      unzip_and_remove=False):
    """
    Download the file from download_url into local_dest
    if the file doesn't already exists.
    If expected_byte is provided, check if
    the downloaded file has the same number of bytes.
    If unzip_and_remove is True, unzip the file and remove the zip file
    """
    if os.path.exists(local_dest) or os.path.exists(local_dest[:-3]):
        print('%s already exists' % local_dest)
    else:
        print('Downloading %s' % download_url)
        local_file, _ = urllib.request.urlretrieve(download_url, local_dest)
        file_stat = os.stat(local_dest)
        if expected_byte:
            if file_stat.st_size == expected_byte:
                print('Successfully downloaded %s' % local_dest)
                if unzip_and_remove:
                    with gzip.open(local_dest, 'rb') as f_in, open(local_dest[:-3], 'wb') as f_out:
                        shutil.copyfileobj(f_in, f_out)
                    os.remove(local_dest)
            else:
                print('The downloaded file has unexpected number of bytes') 
Example 21
Project: video2commons   Author: toolforge   File: upload.py    License: GNU General Public License v3.0 5 votes vote down vote up
def handle_chunked(f, permpath, content_range):
    try:
        content_range = RE_CONTENT_RANGE.match(content_range)
        assert content_range, 'Invalid content range!'

        cr1, cr2, cr3 = [int(content_range.group(i)) for i in range(1, 4)]

        if os.path.isfile(permpath):
            size = stat(permpath)
        else:
            size = 0

        if size != cr1:
            raise WrongOffset(size)

        with open(permpath, 'ab') as dest:
            shutil.copyfileobj(f, dest)

    except WrongOffset as e:
        size = e.offset
    else:
        size = stat(permpath)
    if size < cr3:
        return 'Continue', {'offset': size}
    elif size > cr3:
        raise RuntimeError('What?! Uploaded file is larger than '
                           'what it is supposed to be?')
    return 'Success', {} 
Example 22
Project: google_streetview   Author: rrwen   File: helpers.py    License: MIT License 5 votes vote down vote up
def download(url, file_path):
  r = requests.get(url, stream=True)
  if r.status_code == 200: # if request is successful
    with open(file_path, 'wb') as f:
      r.raw.decode_content = True
      shutil.copyfileobj(r.raw, f) 
Example 23
Project: neuropythy   Author: noahbenson   File: filemap.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def url_download(url, topath=None, create_dirs=True):
    '''
    url_download(url) yields the contents of the given url as a byte-string.
    url_download(url, topath) downloads the given url to the given path, topath and yields that path
      on success.

    The option create_dirs (default: True) may be set to False to prevent the topath directory from
    being created.
    '''
    # ensure directory exists
    if topath: topath = os.path.expanduser(os.path.expandvars(topath))
    if create_dirs and topath:
        dnm = os.path.dirname(topath)
        if not os.path.isdir(dnm): os.makedirs(os.path.abspath(dnm), 0o755)
    if six.PY2:
        response = urllib.request.urlopen(url)
        if topath is None: topath = response.read()
        else:
            with open(topath, 'wb') as fl:
                shutil.copyfileobj(response, fl)
    else:
        with urllib.request.urlopen(url) as response:
            if topath is None: topath = response.read()
            else:
                with open(topath, 'wb') as fl:
                    shutil.copyfileobj(response, fl)
    return topath 
Example 24
Project: CyberTK-Self   Author: CyberTKR   File: LineApi.py    License: GNU General Public License v2.0 5 votes vote down vote up
def sendVideoWithURL(self, to_, url):
        path = '%s/pythonLine-%i.data' % (tempfile.gettempdir(), randint(0, 9))

        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'w') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download video failure.')

        try:
            self.sendVideo(to_, path)
        except Exception as e:
            raise (e) 
Example 25
Project: CyberTK-Self   Author: CyberTKR   File: LineApi.py    License: GNU General Public License v2.0 5 votes vote down vote up
def sendAudioWithUrl(self, to_, url):
        path = '%s/pythonLine-%1.data' % (tempfile.gettempdir(), randint(0, 9))

        r = requests.get(url, stream=True)
        if r.status_code == 200:
            with open(path, 'w') as f:
                shutil.copyfileobj(r.raw, f)
        else:
            raise Exception('Download audio failure.')

        try:
            self.sendAudio(to_, path)
        except Exception as e:
            raise (e) 
Example 26
Project: arxiv-collector   Author: djsutherland   File: arxiv_collector.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_latexmk(version="ctan", dest="latexmk", verbose=True):
    try:
        from urllib.request import urlopen
    except ImportError:
        from urllib2 import urlopen as _urlopen
        from contextlib import closing

        def urlopen(*args, **kwargs):
            return closing(_urlopen(*args, **kwargs))

    import shutil
    import zipfile

    if version.lower() == "ctan":
        url = "http://mirrors.ctan.org/support/latexmk.zip"
    else:
        v = version.replace(".", "")
        url = "http://personal.psu.edu/jcc8/software/latexmk-jcc/latexmk-{}.zip".format(v)

    with io.BytesIO() as bio:
        if verbose:
            print("Downloading latexmk {}...".format(version), file=sys.stderr, end="")
        with urlopen(url) as web:
            shutil.copyfileobj(web, bio, length=131072)

        with zipfile.ZipFile(bio) as z:
            for zinfo in z.infolist():
                if os.path.basename(zinfo.filename) == "latexmk.pl":
                    with z.open(zinfo) as script, io.open(dest, "wb") as out:
                        shutil.copyfileobj(script, out)

                    # executable: https://stackoverflow.com/a/30463972/344821
                    mode = os.stat(dest).st_mode
                    mode |= (mode & 0o444) >> 2  # copy R bits to X
                    os.chmod(dest, mode)
                    break
            else:
                raise ValueError("Couldn't find latexmk.pl in {}".format(url))

        if verbose:
            print("saved to `{}`.".format(dest), file=sys.stderr) 
Example 27
Project: twitter-image-downloader   Author: morinokami   File: twt_img.py    License: MIT License 5 votes vote down vote up
def save_image(self, image, path, timestamp, size="large"):
        """Download and save image to path.

        Args:
            image: The url of the image.
            path: The directory where the image will be saved.
            timestamp: The time that the image was uploaded.
                It is used for naming the image.
            size: Which size of images to download.
        """

        def print_status(s):
            import sys

            sys.stdout.write("\u001b[1K")
            spinner = ["-", "\\", "|", "/"][self.count % 4]
            print(f"\r{spinner} {s}", end="")

        if image:
            # image's path with a new name
            ext = os.path.splitext(image)[1]
            name = timestamp + ext
            save_dest = os.path.join(path, name)

            # save the image in the specified directory if
            if not (os.path.exists(save_dest)):

                r = requests.get(image + ":" + size, stream=True)
                if r.status_code == 200:
                    with open(save_dest, "wb") as f:
                        r.raw.decode_content = True
                        shutil.copyfileobj(r.raw, f)
                    self.count += 1
                    print_status(f"{name} saved")

            else:
                print_status(f"Skipping {name}: already downloaded") 
Example 28
Project: macops   Author: google   File: can_haz_image.py    License: Apache License 2.0 5 votes vote down vote up
def DownloadFile(self, fileurl, dlfile):
    """Download a file."""
    try:
      file_to_dl = urllib2.urlopen(fileurl)
      tmpfile = open(dlfile, 'wb')
      shutil.copyfileobj(file_to_dl, tmpfile)
    except urllib2.URLError, e:
      print 'Download of %s failed with error %s' % (fileurl, e)
      sys.exit() 
Example 29
Project: botbuilder-python   Author: microsoft   File: bidaf_model_runtime.py    License: MIT License 5 votes vote down vote up
def init_bidaf(bidaf_model_dir: str, download_ntlk_punkt: bool = False) -> bool:
        if os.path.isdir(bidaf_model_dir):
            print("bidaf model directory already present..", file=sys.stderr)
        else:
            print("Creating bidaf model directory..", file=sys.stderr)
            os.makedirs(bidaf_model_dir, exist_ok=True)

        # Download Punkt Sentence Tokenizer
        if download_ntlk_punkt:
            nltk.download("punkt", download_dir=bidaf_model_dir)
            nltk.download("punkt")

        # Download bidaf onnx model
        onnx_model_file = os.path.abspath(os.path.join(bidaf_model_dir, "bidaf.onnx"))

        print(f"Checking file {onnx_model_file}..", file=sys.stderr)
        if os.path.isfile(onnx_model_file):
            print("bidaf.onnx downloaded already!", file=sys.stderr)
        else:
            print("Downloading bidaf.onnx...", file=sys.stderr)
            response = requests.get(
                "https://onnxzoo.blob.core.windows.net/models/opset_9/bidaf/bidaf.onnx",
                stream=True,
            )
            with open(onnx_model_file, "wb") as f:
                response.raw.decode_content = True
                shutil.copyfileobj(response.raw, f)
        return True 
Example 30
Project: JJMumbleBot   Author: DuckBoss   File: image_helper.py    License: GNU General Public License v3.0 5 votes vote down vote up
def download_image_requests(img_url):
    dir_utils.clear_directory(f'{dir_utils.get_temp_med_dir()}/internal/images')
    img_ext = img_url.rsplit('.', 1)[1]
    s = requests.Session()
    r = s.get(img_url, headers={'User-Agent': 'Mozilla/5.0'})
    if r.status_code == 200:
        with open(f"{dir_utils.get_temp_med_dir()}/internal/images/_image.{img_ext}", 'wb') as f:
            r.raw.decode_content = True
            shutil.copyfileobj(r.raw, f)
        dprint(f"Downloaded image from: {img_url}")
    else:
        dprint(f"{r.status_code} Error! - {img_url}")