Python requests.utils.urlparse() Examples

def _fetch_package(self):
        parse = urlparse(self._deploy_name)
        if parse.scheme in ["http", "https"]:
            # @TODO
            pass
        elif parse.scheme == "file":
            parts = parse.path.split("/")
            _, ext = os.path.splitext(parts[-1])
            if ext == ".gz":
                filepath = parse.path
            else:
                filepath = tempfile.NamedTemporaryFile().name
                packager.pack_kub(filepath)
            with open(filepath, "rb") as tarf:
                return tarf.read()
        else:
            return self._registry.pull_json(self._deploy_name, self._deploy_version,
                                            self.media_type)['blob'] 

def __init__(self, namespace=None, endpoint=None, body=None, proxy=None):

        self.proxy = None
        if endpoint is not None and endpoint[0] == "/":
            endpoint = endpoint[1:-1]
        self.endpoint = endpoint
        self.body = body
        self.obj = None
        self.protected = False
        self._resource_load()
        self.kind = self.obj['kind'].lower()
        self.name = self.obj['metadata']['name']
        self.force_rotate = ANNOTATIONS['rand'] in self.obj['metadata'].get('annotations', {})
        self.namespace = self._namespace(namespace)
        self.result = None
        if proxy:
            self.proxy = urlparse(proxy) 

def _configure_endpoint(self, endpoint):
        if endpoint is None:
            endpoint = DEFAULT_REGISTRY
        alias = self.config.get_registry_alias(endpoint)
        if alias:
            endpoint = alias
        if not re.match("https?://", endpoint):
            if endpoint.startswith("localhost"):
                scheme = "http://"
            else:
                scheme = "https://"
            endpoint = scheme + endpoint
        return urlparse(endpoint + DEFAULT_PREFIX) 

def is_url(url):
    """Check if provided string is a url.

    Args:
        url (str): url to check

    Returns:
        bool: True if arg url is a valid url

    """
    scheme = requtil.urlparse(str(url)).scheme
    return scheme in ('http', 'https',) 

def get_url_filename(url):
    """Parse filename from url.

    Args:
        url (str): url to parse

    Returns:
        str: filename of url

    """
    path = requtil.urlparse(url).path
    file_name = Path(path).name
    return file_name 

def download_model(url, model_dir, hash_prefix, progress=True):
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    parts = urlparse(url)
    filename = os.path.basename(parts.path)
    cached_file = os.path.join(model_dir, filename)
    if not os.path.exists(cached_file):
        sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
        _download_url_to_file(url, cached_file, hash_prefix, progress=progress)
        unzip_file(cached_file, os.path.join(model_dir, filename.split('.')[0])) 

def load_url(url, model_dir=None, map_location=None):
    r"""Loads the Torch serialized object at the given URL.

    If the object is already present in `model_dir`, it's deserialized and
    returned. The filename part of the URL should follow the naming convention
    ``filename-<sha256>.ext`` where ``<sha256>`` is the first eight or more
    digits of the SHA256 hash of the contents of the file. The hash is used to
    ensure unique names and to verify the contents of the file.

    The default value of `model_dir` is ``$TORCH_HOME/models`` where
    ``$TORCH_HOME`` defaults to ``~/.torch``. The default directory can be
    overriden with the ``$TORCH_MODEL_ZOO`` environment variable.

    Args:
        url (string): URL of the object to download
        model_dir (string, optional): directory in which to save the object
        map_location (optional): a function or a dict specifying how to remap storage locations (see torch.load)

    Example:
        >>> state_dict = torch.utils.model_zoo.load_url('https://s3.amazonaws.com/pytorch/models/resnet18-5c106cde.pth')

    """
    if model_dir is None:
        seeta_home = os.path.expanduser(os.getenv('SEETA_HOME', '~/.pyseeta'))
        model_dir = os.getenv('SEETA_MODEL_ZOO', os.path.join(seeta_home, 'models'))
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    parts = urlparse(url)
    filename = os.path.basename(parts.path)
    cached_file = os.path.join(model_dir, filename)
    if not os.path.exists(cached_file):
        sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
        hash_prefix = HASH_REGEX.search(filename).group(1)
        _download_url_to_file(url, cached_file, hash_prefix)
    # return torch.load(cached_file, map_location=map_location)
    return cached_file 

def fixurl(url):
  # Inspired from https://stackoverflow.com/a/804380 but using requests
  from requests.utils import urlparse, urlunparse, quote, unquote

  # turn string into unicode
  if not isinstance(url, unicode):
    url = url.decode('utf8')

  # parse it
  parsed = urlparse(url)

  # divide the netloc further
  userpass, at, hostport = parsed.netloc.rpartition('@')
  user, colon1, pass_ = userpass.partition(':')
  host, colon2, port = hostport.partition(':')

  # encode each component
  scheme = parsed.scheme.encode('utf8')
  user = quote(user.encode('utf8'))
  colon1 = colon1.encode('utf8')
  pass_ = quote(pass_.encode('utf8'))
  at = at.encode('utf8')
  host = host.encode('idna')
  colon2 = colon2.encode('utf8')
  port = port.encode('utf8')
  path = '/'.join(  # could be encoded slashes!
    quote(unquote(pce).encode('utf8'), '')
    for pce in parsed.path.split('/')
  )
  query = quote(unquote(parsed.query).encode('utf8'), '=&?/')
  fragment = quote(unquote(parsed.fragment).encode('utf8'))

  # put it back together
  netloc = ''.join((user, colon1, pass_, at, host, colon2, port))
  #urlunparse((scheme, netloc, path, params, query, fragment))
  params = ''
  return urlunparse((scheme, netloc, path, params, query, fragment)) 

def load_url(url, model_dir=None, map_location=None, progress=True):
    if model_dir is None:
        torch_home = os.path.expanduser(os.getenv('TORCH_HOME', '~/.torch'))
        model_dir = os.getenv('TORCH_MODEL_ZOO', os.path.join(torch_home, 'models'))
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)
    parts = urlparse(url)
    filename = os.path.basename(parts.path)
    cached_file = os.path.join(model_dir, filename)
    if not os.path.exists(cached_file):
        sys.stderr.write('Downloading: "{}" to {}\n'.format(url, cached_file))
        hash_prefix = HASH_REGEX.search(filename).group(1)
        _download_url_to_file(url, cached_file, hash_prefix, progress=progress)
    return torch.load(cached_file, map_location=map_location)