Python httplib.HTTP Examples

def test_chunked_head(self):
        chunked_start = (
            'HTTP/1.1 200 OK\r\n'
            'Transfer-Encoding: chunked\r\n\r\n'
            'a\r\n'
            'hello world\r\n'
            '1\r\n'
            'd\r\n'
        )
        sock = FakeSocket(chunked_start + '0\r\n')
        resp = httplib.HTTPResponse(sock, method="HEAD")
        resp.begin()
        self.assertEqual(resp.read(), '')
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, 'OK')
        self.assertTrue(resp.isclosed()) 

def test_host_port(self):
        # Check invalid host_port

        # Note that httplib does not accept user:password@ in the host-port.
        for hp in ("www.python.org:abc", "user:password@www.python.org"):
            self.assertRaises(httplib.InvalidURL, httplib.HTTP, hp)

        for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", "fe80::207:e9ff:fe9b",
                          8000),
                         ("www.python.org:80", "www.python.org", 80),
                         ("www.python.org", "www.python.org", 80),
                         ("www.python.org:", "www.python.org", 80),
                         ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80)):
            http = httplib.HTTP(hp)
            c = http._conn
            if h != c.host:
                self.fail("Host incorrectly parsed: %s != %s" % (h, c.host))
            if p != c.port:
                self.fail("Port incorrectly parsed: %s != %s" % (p, c.host)) 

def test_local_bad_hostname(self):
        # The (valid) cert doesn't validate the HTTP hostname
        import ssl
        server = self.make_server(CERT_fakehostname)
        context = ssl.SSLContext(ssl.PROTOCOL_TLS)
        context.verify_mode = ssl.CERT_REQUIRED
        context.check_hostname = True
        context.load_verify_locations(CERT_fakehostname)
        h = httplib.HTTPSConnection('localhost', server.port, context=context)
        with self.assertRaises(ssl.CertificateError):
            h.request('GET', '/')
        h.close()
        # With context.check_hostname=False, the mismatching is ignored
        context.check_hostname = False
        h = httplib.HTTPSConnection('localhost', server.port, context=context)
        h.request('GET', '/nonexistent')
        resp = h.getresponse()
        self.assertEqual(resp.status, 404) 

def test_host_port(self):
        # Check invalid host_port

        # Note that httplib does not accept user:password@ in the host-port.
        for hp in ("www.python.org:abc", "user:password@www.python.org"):
            self.assertRaises(httplib.InvalidURL, httplib.HTTP, hp)

        for hp, h, p in (("[fe80::207:e9ff:fe9b]:8000", "fe80::207:e9ff:fe9b",
                          8000),
                         ("www.python.org:80", "www.python.org", 80),
                         ("www.python.org", "www.python.org", 80),
                         ("www.python.org:", "www.python.org", 80),
                         ("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80)):
            http = httplib.HTTP(hp)
            c = http._conn
            if h != c.host:
                self.fail("Host incorrectly parsed: %s != %s" % (h, c.host))
            if p != c.port:
                self.fail("Port incorrectly parsed: %s != %s" % (p, c.host)) 

def test_status_lines(self):
        # Test HTTP status lines

        body = "HTTP/1.1 200 Ok\r\n\r\nText"
        sock = FakeSocket(body)
        resp = httplib.HTTPResponse(sock)
        resp.begin()
        self.assertEqual(resp.read(0), '')  # Issue #20007
        self.assertFalse(resp.isclosed())
        self.assertEqual(resp.read(), 'Text')
        self.assertTrue(resp.isclosed())

        body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText"
        sock = FakeSocket(body)
        resp = httplib.HTTPResponse(sock)
        self.assertRaises(httplib.BadStatusLine, resp.begin) 

def test_parse_all_octets(self):
        # Ensure no valid header field octet breaks the parser
        body = (
            b'HTTP/1.1 200 OK\r\n'
            b"!#$%&'*+-.^_`|~: value\r\n"  # Special token characters
            b'VCHAR: ' + bytearray(range(0x21, 0x7E + 1)) + b'\r\n'
            b'obs-text: ' + bytearray(range(0x80, 0xFF + 1)) + b'\r\n'
            b'obs-fold: text\r\n'
            b' folded with space\r\n'
            b'\tfolded with tab\r\n'
            b'Content-Length: 0\r\n'
            b'\r\n'
        )
        sock = FakeSocket(body)
        resp = httplib.HTTPResponse(sock)
        resp.begin()
        self.assertEqual(resp.getheader('Content-Length'), '0')
        self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value')
        vchar = ''.join(map(chr, range(0x21, 0x7E + 1)))
        self.assertEqual(resp.getheader('VCHAR'), vchar)
        self.assertIsNotNone(resp.getheader('obs-text'))
        folded = resp.getheader('obs-fold')
        self.assertTrue(folded.startswith('text'))
        self.assertIn(' folded with space', folded)
        self.assertTrue(folded.endswith('folded with tab')) 

def test_response_headers(self):
        # test response with multiple message headers with the same field name.
        text = ('HTTP/1.1 200 OK\r\n'
                'Set-Cookie: Customer="WILE_E_COYOTE";'
                ' Version="1"; Path="/acme"\r\n'
                'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";'
                ' Path="/acme"\r\n'
                '\r\n'
                'No body\r\n')
        hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"'
               ', '
               'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"')
        s = FakeSocket(text)
        r = httplib.HTTPResponse(s)
        r.begin()
        cookies = r.getheader("Set-Cookie")
        if cookies != hdr:
            self.fail("multiple headers not combined properly") 

def test_response_headers(self):
        # test response with multiple message headers with the same field name.
        text = ('HTTP/1.1 200 OK\r\n'
                'Set-Cookie: Customer="WILE_E_COYOTE";'
                ' Version="1"; Path="/acme"\r\n'
                'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";'
                ' Path="/acme"\r\n'
                '\r\n'
                'No body\r\n')
        hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"'
               ', '
               'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"')
        s = FakeSocket(text)
        r = httplib.HTTPResponse(s)
        r.begin()
        cookies = r.getheader("Set-Cookie")
        if cookies != hdr:
            self.fail("multiple headers not combined properly") 

def test_ipv6host_header(self):
        # Default host header on IPv6 transaction should be wrapped by [] if
        # it is an IPv6 address
        expected = 'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \
                   'Accept-Encoding: identity\r\n\r\n'
        conn = httplib.HTTPConnection('[2001::]:81')
        sock = FakeSocket('')
        conn.sock = sock
        conn.request('GET', '/foo')
        self.assertTrue(sock.data.startswith(expected))

        expected = 'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \
                   'Accept-Encoding: identity\r\n\r\n'
        conn = httplib.HTTPConnection('[2001:102A::]')
        sock = FakeSocket('')
        conn.sock = sock
        conn.request('GET', '/foo')
        self.assertTrue(sock.data.startswith(expected)) 

def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
        if 'location' in headers:
            newurl = headers['location']
        elif 'uri' in headers:
            newurl = headers['uri']
        else:
            return
        fp.close()
        # In case the server sent a relative URL, join with original:
        newurl = basejoin(self.type + ":" + url, newurl)

        # For security reasons we do not allow redirects to protocols
        # other than HTTP, HTTPS or FTP.
        newurl_lower = newurl.lower()
        if not (newurl_lower.startswith('http://') or
                newurl_lower.startswith('https://') or
                newurl_lower.startswith('ftp://')):
            raise IOError('redirect error', errcode,
                          errmsg + " - Redirection to url '%s' is not allowed" %
                          newurl,
                          headers)

        return self.open(newurl) 

def test_ipv6host_header(self):
        # Default host header on IPv6 transaction should wrapped by [] if
        # its actual IPv6 address
        expected = 'GET /foo HTTP/1.1\r\nHost: [2001::]:81\r\n' \
                   'Accept-Encoding: identity\r\n\r\n'
        conn = httplib.HTTPConnection('[2001::]:81')
        sock = FakeSocket('')
        conn.sock = sock
        conn.request('GET', '/foo')
        self.assertTrue(sock.data.startswith(expected))

        expected = 'GET /foo HTTP/1.1\r\nHost: [2001:102A::]\r\n' \
                   'Accept-Encoding: identity\r\n\r\n'
        conn = httplib.HTTPConnection('[2001:102A::]')
        sock = FakeSocket('')
        conn.sock = sock
        conn.request('GET', '/foo')
        self.assertTrue(sock.data.startswith(expected)) 

def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
        if 'location' in headers:
            newurl = headers['location']
        elif 'uri' in headers:
            newurl = headers['uri']
        else:
            return
        void = fp.read()
        fp.close()
        # In case the server sent a relative URL, join with original:
        newurl = basejoin(self.type + ":" + url, newurl)

        # For security reasons we do not allow redirects to protocols
        # other than HTTP, HTTPS or FTP.
        newurl_lower = newurl.lower()
        if not (newurl_lower.startswith('http://') or
                newurl_lower.startswith('https://') or
                newurl_lower.startswith('ftp://')):
            raise IOError('redirect error', errcode,
                          errmsg + " - Redirection to url '%s' is not allowed" %
                          newurl,
                          headers)

        return self.open(newurl) 

def redirect_internal(self, url, fp, errcode, errmsg, headers, data):
        if 'location' in headers:
            newurl = headers['location']
        elif 'uri' in headers:
            newurl = headers['uri']
        else:
            return
        fp.close()
        # In case the server sent a relative URL, join with original:
        newurl = basejoin(self.type + ":" + url, newurl)

        # For security reasons we do not allow redirects to protocols
        # other than HTTP, HTTPS or FTP.
        newurl_lower = newurl.lower()
        if not (newurl_lower.startswith('http://') or
                newurl_lower.startswith('https://') or
                newurl_lower.startswith('ftp://')):
            raise IOError('redirect error', errcode,
                          errmsg + " - Redirection to url '%s' is not allowed" %
                          newurl,
                          headers)

        return self.open(newurl) 

def test_chunked_head(self):
        chunked_start = (
            'HTTP/1.1 200 OK\r\n'
            'Transfer-Encoding: chunked\r\n\r\n'
            'a\r\n'
            'hello world\r\n'
            '1\r\n'
            'd\r\n'
        )
        sock = FakeSocket(chunked_start + '0\r\n')
        resp = httplib.HTTPResponse(sock, method="HEAD")
        resp.begin()
        self.assertEqual(resp.read(), '')
        self.assertEqual(resp.status, 200)
        self.assertEqual(resp.reason, 'OK')
        self.assertTrue(resp.isclosed()) 

def do_search(self):
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/search/web/results/?q=%40" + self.word +
                     "&elements_per_page=50&start_index=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader(
            'Referer',
            "http://" +
            self.hostname +
            "/search/web/results/?q=%40" +
            self.word)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def encode_multipart_formdata(fields, files):  
    """ 
    fields is a sequence of (name, value) elements for regular form fields. 
    files is a sequence of (name, filename, value) elements for data to be uploaded as files 
    Return (content_type, body) ready for httplib.HTTP instance 
    """  
    BOUNDARY = '----------ThIs_Is_tHe_bouNdaRY_$'  
    CRLF = '\r\n'  
    L = []  
    for (key, value) in fields:  
        L.append('--' + BOUNDARY)  
        L.append('Content-Disposition: form-data; name="%s"' % key)  
        L.append('')  
        L.append(value)  
    for (key, filename, value) in files:  
        L.append('--' + BOUNDARY)  
        L.append('Content-Disposition: form-data; name="%s"; filename="%s"' % (key, filename))  
        L.append('Content-Type: %s' % get_content_type(filename))  
        L.append('')  
        L.append(value)  
    L.append('--' + BOUNDARY + '--')  
    L.append('')  
    body = CRLF.join(L)  
    content_type = 'multipart/form-data; boundary=%s' % BOUNDARY  
    return content_type, body 

def post_multipart(host, selector, fields, files):  
    """ 
    Post fields and files to an http host as multipart/form-data. 
    fields is a sequence of (name, value) elements for regular form fields. 
    files is a sequence of (name, filename, value) elements for data to be uploaded as files 
    Return the server's response page. 
    """
    content_type, body = encode_multipart_formdata(fields, files)  
    h = httplib.HTTP(host)  
    h.putrequest('POST', selector)  
    h.putheader('content-type', content_type)  
    h.putheader('content-length', str(len(body)))  
    h.endheaders()  
    h.send(body)  
    errcode, errmsg, headers = h.getreply()
    return h.file.read() 

def do_search_files(self, files):
        h = httplib.HTTP(self.server)
        h.putrequest(
            'GET',
            "search/web/results/?q=" +
            self.word +
            "filetype:" +
            self.files +
            "&elements_per_page=50&start_index=" +
            self.counter)
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search(self):
        h = httplib.HTTP(self.server)
        h.putrequest(
            'GET',
            "/FreeTextSearch.xhtml?opCode=search&autoSuggested=True&freeText=" +
            self.word)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search(self):
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/search?text=%40" + self.word +
                     "&numdoc=50&lr=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results
        print self.results 

def do_search(self):
        h = httplib.HTTP(self.server)

        h.putrequest('GET', "/search?p=\"%40" + self.word
                     + "\"&b=" + str(self.counter) + "&pz=10")
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()

        self.total_results += h.getfile().read() 

def do_search_files(self, files):  # TODO
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/search?text=%40" + self.word +
                     "&numdoc=50&lr=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search(self):
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/search?q=%40" + self.word +
                     "&count=50&first=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader('Cookie', 'SRCHHPGUSR=ADLT=DEMOTE&NRSLT=50')
        h.putheader('Accept-Language', 'en-us,en')
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search(self):
        h = httplib.HTTP(self.server)

        # Dogpile is hardcoded to return 10 results
        h.putrequest('GET', "/search/web?qsi=" + str(self.counter)
                     + "&q=\"%40" + self.word + "\"")
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()

        self.total_results += h.getfile().read() 

def do_search(self):
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/sets?hl=en&" + self.set)
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search_files(self):
		h = httplib.HTTP(self.server)
		h.putrequest('GET', "/search?num="+self.quantity+"&start=" + str(self.counter) + "&hl=en&meta=&q=filetype:"+self.filetype+"%20site:" + self.word)
		h.putheader('Host', self.hostname)
		h.putheader('User-agent', self.userAgent)	
		h.endheaders()
		returncode, returnmsg, headers = h.getreply()
		self.results = h.getfile().read()
		self.totalresults+= self.results 

def do_search(self):
        h = httplib.HTTP(self.server)

        h.putrequest('GET', "/s?wd=%40" + self.word
                     + "&pn=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()

        self.total_results += h.getfile().read() 

def do_search_api(self):
        h = httplib.HTTP(self.apiserver)
        h.putrequest('GET', "/xml.aspx?Appid=" + self.bingApi + "&query=%40" +
                     self.word + "&sources=web&web.count=40&web.offset=" + str(self.counter))
        h.putheader('Host', "api.search.live.net")
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def do_search_vhost(self):
        h = httplib.HTTP(self.server)
        h.putrequest('GET', "/search?q=ip:" + self.word +
                     "&go=&count=50&FORM=QBHL&qs=n&first=" + str(self.counter))
        h.putheader('Host', self.hostname)
        h.putheader(
            'Cookie', 'mkt=en-US;ui=en-US;SRCHHPGUSR=NEWWND=0&ADLT=DEMOTE&NRSLT=50')
        h.putheader('Accept-Language', 'en-us,en')
        h.putheader('User-agent', self.userAgent)
        h.endheaders()
        returncode, returnmsg, headers = h.getreply()
        self.results = h.getfile().read()
        self.totalresults += self.results 

def test_partial_reads_no_content_length(self):
        # when no length is present, the socket should be gracefully closed when
        # all data was read
        body = "HTTP/1.1 200 Ok\r\n\r\nText"
        sock = FakeSocket(body)
        resp = httplib.HTTPResponse(sock)
        resp.begin()
        self.assertEqual(resp.read(2), 'Te')
        self.assertFalse(resp.isclosed())
        self.assertEqual(resp.read(2), 'xt')
        self.assertEqual(resp.read(1), '')
        self.assertTrue(resp.isclosed())