Python requests.compat.urlparse() Examples

def authenticate_server(self, response):
        """
        Uses GSSAPI to authenticate the server.

        Returns True on success, False on failure.
        """

        log.debug("authenticate_server(): Authenticate header: {0}".format(
            _negotiate_value(response)))

        host = urlparse(response.url).hostname

        try:
            # If the handshake isn't complete here, nothing we can do
            self.context[host].step(_negotiate_value(response))
        except gssapi.exceptions.GSSError as error:
            log.exception("authenticate_server(): context stepping failed:")
            log.exception(error.gen_message())
            return False

        log.debug("authenticate_server(): returning {0}".format(response))
        return True 

def authenticate_user(self, response, **kwargs):
        """Handles user authentication with GSSAPI"""

        host = urlparse(response.url).hostname

        try:
            auth_header = self.generate_request_header(response, host)
        except SPNEGOExchangeError:
            # GSS Failure, return existing response
            return response

        log.debug("authenticate_user(): Authorization header: {0}".format(
            auth_header))
        response.request.headers['Authorization'] = auth_header

        # Consume the content so we can reuse the connection for the next
        # request.
        response.content
        response.raw.release_conn()

        _r = response.connection.send(response.request, **kwargs)
        _r.history.append(response)

        log.debug("authenticate_user(): returning {0}".format(_r))
        return _r 

def test_explicit_mech(self):
        with patch.multiple("gssapi.Credentials", __new__=fake_creds), \
             patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            fake_mech = b'fake mech'
            auth = requests_gssapi.HTTPSPNEGOAuth(mech=fake_mech)
            auth.generate_request_header(response, host)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                usage="initiate", flags=gssflags,
                creds=None, mech=b'fake mech')
            fake_resp.assert_called_with(b"token") 

def test_explicit_creds(self):
        with patch.multiple("gssapi.Credentials", __new__=fake_creds), \
             patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            creds = gssapi.Credentials()
            auth = requests_gssapi.HTTPSPNEGOAuth(creds=creds)
            auth.generate_request_header(response, host)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                usage="initiate", flags=gssflags,
                creds=b"fake creds", mech=None)
            fake_resp.assert_called_with(b"token") 

def test_principal_override(self):
        with patch.multiple("gssapi.Credentials", __new__=fake_creds), \
             patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPKerberosAuth(principal="user@REALM")
            auth.generate_request_header(response, host)
            fake_creds.assert_called_with(gssapi.creds.Credentials,
                                          usage="initiate",
                                          name=gssapi_name("user@REALM"))
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                usage="initiate", flags=gssflags,
                creds=b"fake creds", mech=None) 

def get_connection(self, url, proxies=None):
        proxies = proxies or {}
        proxy = proxies.get(urlparse(url.lower()).scheme)

        if proxy:
            raise ValueError('%s does not support specifying proxies'
                             % self.__class__.__name__)

        with self.pools.lock:
            pool = self.pools.get(url)
            if pool:
                return pool

            pool = UnixHTTPConnectionPool(url, self.timeout)
            self.pools[url] = pool

        return pool 

def get_string_to_sign(self, request, headers, created_timestamp, expires_timestamp):
        sts = []
        for header in headers:
            if header == "(request-target)":
                path_url = requests.models.RequestEncodingMixin.path_url.fget(request)
                sts.append("{}: {} {}".format(header, request.method.lower(), path_url))
            elif header == "(created)":
                sts.append("{}: {}".format(header, created_timestamp))
            elif header == "(expires)":
                assert (expires_timestamp is not None), \
                    'You should provide the "expires_in" argument when using the (expires) header'
                sts.append("{}: {}".format(header, int(expires_timestamp)))
            else:
                if header.lower() == "host":
                    url = urlparse(request.url)
                    value = request.headers.get("host", url.hostname)
                    if url.scheme == "http" and url.port not in [None, 80] or url.scheme == "https" \
                            and url.port not in [443, None]:
                        value = "{}:{}".format(value, url.port)
                else:
                    value = request.headers[header]
                sts.append("{k}: {v}".format(k=header.lower(), v=value))
        return "\n".join(sts).encode() 

def get_connection(self, url, proxies=None):
        proxies = proxies or {}
        proxy = proxies.get(urlparse(url.lower()).scheme)

        if proxy:
            raise ValueError('%s does not support specifying proxies'
                             % self.__class__.__name__)

        with self.pools.lock:
            pool = self.pools.get(url)
            if pool:
                return pool

            pool = UnixHTTPConnectionPool(url, self.timeout)
            self.pools[url] = pool

        return pool 

def _build_request_path(url, proxy_info):
    uri = compat.urlparse(url)
    proxy_url = proxy_info.get('request_path')
    if proxy_url is not None:
        request_path = _coerce_to_bytes(proxy_url)
        return request_path, uri

    request_path = _coerce_to_bytes(uri.path)
    if uri.query:
        request_path += b'?' + _coerce_to_bytes(uri.query)

    return request_path, uri 

def _key_from_url(url):
        parsed = urlparse(url)
        return urlunparse((parsed.scheme.lower(),
                           parsed.netloc.lower(),
                           '', '', '', '')) 

def _build_request_path(url, proxy_info):
    uri = compat.urlparse(url)
    proxy_url = proxy_info.get('request_path')
    if proxy_url is not None:
        request_path = _coerce_to_bytes(proxy_url)
        return request_path, uri

    request_path = _coerce_to_bytes(uri.path)
    if uri.query:
        request_path += b'?' + _coerce_to_bytes(uri.query)

    return request_path, uri 

def _key_from_url(url):
        parsed = urlparse(url)
        return urlunparse((parsed.scheme.lower(),
                           parsed.netloc.lower(),
                           '', '', '', '')) 

def connect(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.settimeout(self.timeout)
        socket_path = unquote(urlparse(self.unix_socket_url).netloc)
        sock.connect(socket_path)
        self.sock = sock 

def _is_subdomain_of_tld(url1, url2):
    orig_host = urlparse(url1).hostname
    re_host = urlparse(url2).hostname
    return orig_host.split('.')[-2:] == re_host.split('.')[-2:] 

def get_filename_from_url(url):
    """Get a filename from a URL.

    >>> from planet.api import utils
    >>> urls = [
    ...     'https://planet.com/',
    ...     'https://planet.com/path/to/',
    ...     'https://planet.com/path/to/example.tif',
    ...     'https://planet.com/path/to/example.tif?foo=f6f1&bar=baz',
    ...     'https://planet.com/path/to/example.tif?foo=f6f1&bar=baz#quux'
    ... ]
    >>> for url in urls:
    ...     print('{} -> {}'.format(url, utils.get_filename_from_url(url)))
    ...
    https://planet.com/ -> None
    https://planet.com/path/to/ -> None
    https://planet.com/path/to/example.tif -> example.tif
    https://planet.com/path/to/example.tif?foo=f6f1&bar=baz -> example.tif
    https://planet.com/path/to/example.tif?foo=f6f1&bar=baz#quux -> example.tif
    >>>

    :returns: a filename (i.e. ``basename``)
    :rtype: str or None
    """
    path = urlparse(url).path
    name = path[path.rfind('/')+1:]
    return name or None 

def test_generate_request_header(self):
        with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPKerberosAuth()
            self.assertEqual(
                auth.generate_request_header(response, host),
                b64_negotiate_response)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                creds=None, mech=None, flags=gssflags, usage="initiate")
            fake_resp.assert_called_with(b"token") 

def test_generate_request_header_init_error(self):
        with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fail_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPKerberosAuth()
            self.assertRaises(requests_gssapi.exceptions.SPNEGOExchangeError,
                              auth.generate_request_header, response, host)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                usage="initiate", flags=gssflags, creds=None, mech=None) 

def test_generate_request_header_step_error(self):
        with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fail_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPKerberosAuth()
            self.assertRaises(requests_gssapi.exceptions.SPNEGOExchangeError,
                              auth.generate_request_header, response, host)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@www.example.org"),
                usage="initiate", flags=gssflags, creds=None, mech=None)
            fail_resp.assert_called_with(b"token") 

def test_generate_request_header_custom_service(self):
        with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPKerberosAuth(service="barfoo")
            auth.generate_request_header(response, host),
            fake_init.assert_called_with(
                name=gssapi_name("barfoo@www.example.org"),
                usage="initiate", flags=gssflags, creds=None, mech=None)
            fake_resp.assert_called_with(b"token") 

def test_target_name(self):
        with patch.multiple("gssapi.SecurityContext", __init__=fake_init,
                            step=fake_resp):
            response = requests.Response()
            response.url = "http://www.example.org/"
            response.headers = {'www-authenticate': b64_negotiate_token}
            host = urlparse(response.url).hostname
            auth = requests_gssapi.HTTPSPNEGOAuth(
                target_name="HTTP@otherhost.otherdomain.org")
            auth.generate_request_header(response, host)
            fake_init.assert_called_with(
                name=gssapi_name("HTTP@otherhost.otherdomain.org"),
                usage="initiate", flags=gssflags, creds=None, mech=None)
            fake_resp.assert_called_with(b"token") 

def get_connection(self, url, proxies=None):
        """Returns a urllib3 connection for the given URL. This should not be
        called from user code, and is only exposed for use when subclassing the
        :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`.

        :param url: The URL to connect to.
        :param proxies: (optional) A Requests-style dictionary of proxies used on this request.
        """
        proxies = proxies or {}
        proxy = proxies.get(urlparse(url.lower()).scheme)

        if proxy:
            proxy_headers = self.proxy_headers(proxy)

            if proxy not in self.proxy_manager:
                self.proxy_manager[proxy] = proxy_from_url(
                    proxy,
                    proxy_headers=proxy_headers,
                    num_pools=self._pool_connections,
                    maxsize=self._pool_maxsize,
                    block=self._pool_block
                )

            conn = self.proxy_manager[proxy].connection_from_url(url)
        else:
            # Only scheme should be lower case
            parsed = urlparse(url)
            url = parsed.geturl()
            conn = self.poolmanager.connection_from_url(url)

        self.connections.append(conn)
        return conn 

def __init__(self, count, url, cls, session, params=None, etag=None,
                 headers=None):
        GitHubCore.__init__(self, {}, session)
        #: Original number of items requested
        self.original = count
        #: Number of items left in the iterator
        self.count = count
        #: URL the class used to make it's first GET
        self.url = url
        #: Last URL that was requested
        self.last_url = None
        self._api = self.url
        #: Class for constructing an item to return
        self.cls = cls
        #: Parameters of the query string
        self.params = params or {}
        self._remove_none(self.params)
        # We do not set this from the parameter sent. We want this to
        # represent the ETag header returned by GitHub no matter what.
        # If this is not None, then it won't be set from the response and
        # that's not what we want.
        #: The ETag Header value returned by GitHub
        self.etag = None
        #: Headers generated for the GET request
        self.headers = headers or {}
        #: The last response seen
        self.last_response = None
        #: Last status code received
        self.last_status = 0

        if etag:
            self.headers.update({'If-None-Match': etag})

        self.path = urlparse(self.url).path 

def get_connection(self, url, proxies=None):
        # proxies, silently ignored
        path = unquote(urlparse(url).netloc)
        return HTTPUnixConnectionPool(path) 

def connect(self):
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        sock.settimeout(self.timeout)
        socket_path = unquote(urlparse(self.unix_socket_url).netloc)
        sock.connect(socket_path)
        self.sock = sock 

def get_connection(self, socket_path, proxies=None):
        proxies = proxies or {}
        proxy = proxies.get(urlparse(socket_path.lower()).scheme)

        if proxy:
            raise ValueError('%s does not support specifying proxies'
                             % self.__class__.__name__)
        return UnixHTTPConnectionPool(socket_path, self.timeout) 

def cloudflare_is_bypassed(self, url, resp=None):
        cookie_domain = ".{}".format(urlparse(url).netloc)
        return (
                self.cookies.get("cf_clearance", None, domain=cookie_domain) or
                (resp and resp.cookies.get("cf_clearance", None, domain=cookie_domain))
        ) 

def cloudflare_is_bypassed(self, url, resp=None):
        cookie_domain = ".{}".format(urlparse(url).netloc)
        return (
                self.cookies.get("cf_clearance", None, domain=cookie_domain) or
                (resp and resp.cookies.get("cf_clearance", None, domain=cookie_domain))
        ) 

def _get_domain(url): 
    parsed_url = urlparse(url)
    scheme = parsed_url.scheme if parsed_url.scheme != '' else 'https'
    return "%s://%s" % (scheme, parsed_url.netloc) 

def get(self, url, headers={}, allow_redirects=True):
        parsed_url = urlparse(url)

        response = self.head(_get_domain(url))
        if response is None:
            return None

        (url, status_code) = response
        if status_code != 200:
            return None

        resolved_url = urlparse(url)
        url = urlunparse(
            (
                resolved_url.scheme,
                resolved_url.netloc,
                parsed_url.path,
                parsed_url.params,
                parsed_url.query,
                parsed_url.fragment,
            )
        )

        tools.log('GET: %s' % url, 'info')
        request = lambda x: _get(self._cfscrape, url, headers, self._timeout, allow_redirects, x)
        request.url = url

        return self._request_core(request) 

def get_uri_name(url: str) -> str:
    """Get the file name from the end of the URL."""
    url_parsed = urlparse(url)
    if url.startswith(_FRAUNHOFER_RESOURCES):
        return url_parsed.query.split('=')[-1]
    else:
        url_parts = url_parsed.path.split('/')
        return url_parts[-1]