Python twisted.web.http.urlparse() Examples

def fromBytes(cls, uri, defaultPort=None):
        """
        Parse the given URI into a L{URI}.

        @type uri: C{bytes}
        @param uri: URI to parse.

        @type defaultPort: C{int} or L{None}
        @param defaultPort: An alternate value to use as the port if the URI
            does not include one.

        @rtype: L{URI}
        @return: Parsed URI instance.
        """
        uri = uri.strip()
        scheme, netloc, path, params, query, fragment = http.urlparse(uri)

        if defaultPort is None:
            if scheme == b'https':
                defaultPort = 443
            else:
                defaultPort = 80

        if b':' in netloc:
            host, port = netloc.rsplit(b':', 1)
            try:
                port = int(port)
            except ValueError:
                host, port = netloc, defaultPort
        else:
            host, port = netloc, defaultPort
        return cls(scheme, netloc, host, port, path, params, query, fragment) 

def _urljoin(base, url):
    """
    Construct a full ("absolute") URL by combining a "base URL" with another
    URL. Informally, this uses components of the base URL, in particular the
    addressing scheme, the network location and (part of) the path, to provide
    missing components in the relative URL.

    Additionally, the fragment identifier is preserved according to the HTTP
    1.1 bis draft.

    @type base: C{bytes}
    @param base: Base URL.

    @type url: C{bytes}
    @param url: URL to combine with C{base}.

    @return: An absolute URL resulting from the combination of C{base} and
        C{url}.

    @see: L{urlparse.urljoin}

    @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-22#section-7.1.2}
    """
    base, baseFrag = urldefrag(base)
    url, urlFrag = urldefrag(urljoin(base, url))
    return urljoin(url, b'#' + (urlFrag or baseFrag)) 

def test_urlparseRejectsUnicode(self):
        """
        L{http.urlparse} should reject unicode input early.
        """
        self.assertRaises(TypeError, http.urlparse, u'http://example.org/path') 

def dummyRequest(method, path, headers, body=b""):
    """
    Construct a new dummy L{IRequest} provider.

    @param method: The HTTP method of the request.  For example, C{b"GET"}.
    @type method: L{bytes}

    @param path: The encoded path part of the URI of the request.  For example,
        C{b"/foo"}.
    @type path: L{bytes}

    @param headers: The headers of the request.
    @type headers: L{Headers}

    @param body: The bytes that make up the request body.
    @type body: L{bytes}

    @return: A L{IRequest} which can be used to render an L{IResource} using
        only in-memory data structures.
    """
    parsed = urlparse(path)
    if parsed.query:
        # Oops, dropped params.  Good thing no one cares.
        new_path = parsed.path + "?" + parsed.query
    else:
        new_path = parsed.path
    return _DummyRequest(
        next(_dummyRequestCounter),
        method, new_path, headers, body) 

def fromBytes(cls, uri, defaultPort=None):
        """
        Parse the given URI into a L{URI}.

        @type uri: C{bytes}
        @param uri: URI to parse.

        @type defaultPort: C{int} or L{None}
        @param defaultPort: An alternate value to use as the port if the URI
            does not include one.

        @rtype: L{URI}
        @return: Parsed URI instance.
        """
        uri = uri.strip()
        scheme, netloc, path, params, query, fragment = http.urlparse(uri)

        if defaultPort is None:
            if scheme == b'https':
                defaultPort = 443
            else:
                defaultPort = 80

        if b':' in netloc:
            host, port = netloc.rsplit(b':', 1)
            try:
                port = int(port)
            except ValueError:
                host, port = netloc, defaultPort
        else:
            host, port = netloc, defaultPort
        return cls(scheme, netloc, host, port, path, params, query, fragment) 

def _urljoin(base, url):
    """
    Construct a full ("absolute") URL by combining a "base URL" with another
    URL. Informally, this uses components of the base URL, in particular the
    addressing scheme, the network location and (part of) the path, to provide
    missing components in the relative URL.

    Additionally, the fragment identifier is preserved according to the HTTP
    1.1 bis draft.

    @type base: C{bytes}
    @param base: Base URL.

    @type url: C{bytes}
    @param url: URL to combine with C{base}.

    @return: An absolute URL resulting from the combination of C{base} and
        C{url}.

    @see: L{urlparse.urljoin}

    @see: U{https://tools.ietf.org/html/draft-ietf-httpbis-p2-semantics-22#section-7.1.2}
    """
    base, baseFrag = urldefrag(base)
    url, urlFrag = urldefrag(urljoin(base, url))
    return urljoin(url, b'#' + (urlFrag or baseFrag)) 

def test_urlparseRejectsUnicode(self):
        """
        L{http.urlparse} should reject unicode input early.
        """
        self.assertRaises(TypeError, http.urlparse, u'http://example.org/path') 

def _parse(url, defaultPort=None):
    """
    Split the given URL into the scheme, host, port, and path.

    @type url: C{str}
    @param url: An URL to parse.

    @type defaultPort: C{int} or C{None}
    @param defaultPort: An alternate value to use as the port if the URL does
    not include one.

    @return: A four-tuple of the scheme, host, port, and path of the URL.  All
    of these are C{str} instances except for port, which is an C{int}.
    """
    url = url.strip()
    parsed = http.urlparse(url)
    scheme = parsed[0]
    path = urlunparse(('', '') + parsed[2:])

    if defaultPort is None:
        if scheme == 'https':
            defaultPort = 443
        else:
            defaultPort = 80

    host, port = parsed[1], defaultPort
    if ':' in host:
        host, port = host.split(':')
        try:
            port = int(port)
        except ValueError:
            port = defaultPort

    if path == '':
        path = '/'

    return scheme, host, port, path 

def test_urlparseRejectsUnicode(self):
        """
        L{http.urlparse} should reject unicode input early.
        """
        self.assertRaises(TypeError, http.urlparse, u'http://example.org/path') 

def test_urlparse(self):
        """
        For a given URL, L{http.urlparse} should behave the same as
        L{urlparse}, except it should always return C{str}, never C{unicode}.
        """
        def urls():
            for scheme in ('http', 'https'):
                for host in ('example.com',):
                    for port in (None, 100):
                        for path in ('', 'path'):
                            if port is not None:
                                host = host + ':' + str(port)
                                yield urlunsplit((scheme, host, path, '', ''))


        def assertSameParsing(url, decode):
            """
            Verify that C{url} is parsed into the same objects by both
            L{http.urlparse} and L{urlparse}.
            """
            urlToStandardImplementation = url
            if decode:
                urlToStandardImplementation = url.decode('ascii')
            standardResult = urlparse(urlToStandardImplementation)
            scheme, netloc, path, params, query, fragment = http.urlparse(url)
            self.assertEqual(
                (scheme, netloc, path, params, query, fragment),
                standardResult)
            self.assertTrue(isinstance(scheme, str))
            self.assertTrue(isinstance(netloc, str))
            self.assertTrue(isinstance(path, str))
            self.assertTrue(isinstance(params, str))
            self.assertTrue(isinstance(query, str))
            self.assertTrue(isinstance(fragment, str))

        # With caching, unicode then str
        clear_cache()
        for url in urls():
            assertSameParsing(url, True)
            assertSameParsing(url, False)

        # With caching, str then unicode
        clear_cache()
        for url in urls():
            assertSameParsing(url, False)
            assertSameParsing(url, True)

        # Without caching
        for url in urls():
            clear_cache()
            assertSameParsing(url, True)
            clear_cache()
            assertSameParsing(url, False) 

def get_patched_URI():
    """Create the patched `twisted.web.client.URI` to handle IPv6."""
    import re
    from twisted.web import http
    from twisted.web.client import URI

    class PatchedURI(URI):
        @classmethod
        def fromBytes(cls, uri, defaultPort=None):
            """Patched replacement for `twisted.web.client._URI.fromBytes`.

            The Twisted version of this function breaks when you give it a URL
            whose netloc is based on an IPv6 address.
            """
            uri = uri.strip()
            scheme, netloc, path, params, query, fragment = http.urlparse(uri)

            if defaultPort is None:
                scheme_ports = {b"https": 443, b"http": 80}
                defaultPort = scheme_ports.get(scheme, 80)

            if b"[" in netloc:
                # IPv6 address.  This is complicated.
                parsed_netloc = re.match(
                    b"\\[(?P<host>[0-9A-Fa-f:.]+)\\]([:](?P<port>[0-9]+))?$",
                    netloc,
                )
                host, port = parsed_netloc.group("host", "port")
            elif b":" in netloc:
                # IPv4 address or hostname, with port spec.  This is easy.
                host, port = netloc.split(b":")
            else:
                # IPv4 address or hostname, without port spec.
                # This is trivial.
                host = netloc
                port = None

            if port is None:
                port = defaultPort
            try:
                port = int(port)
            except ValueError:
                port = defaultPort

            return cls(
                scheme, netloc, host, port, path, params, query, fragment
            )

    return PatchedURI