Python urllib.addinfourl() Examples

def probe_html5(self, result):

        class NoRedirectHandler(urllib2.HTTPRedirectHandler):

            def http_error_302(self, req, fp, code, msg, headers):
                infourl = urllib.addinfourl(fp, headers, req.get_full_url())
                infourl.status = code
                infourl.code = code
                return infourl
            http_error_300 = http_error_302
            http_error_301 = http_error_302
            http_error_303 = http_error_302
            http_error_307 = http_error_302

        opener = urllib2.build_opener(NoRedirectHandler())
        urllib2.install_opener(opener)

        r = urllib2.urlopen(urllib2.Request(result['url'], headers=result['headers']))
        if r.code == 200:
            result['url'] = r.read()
        return result 

def _normalize_urllib_response(response, request=None):
    if six.PY2:
        if not isinstance(response, urllib.addinfourl):
            raise TypeError("Cannot normalize this response object")
    else:
        if not isinstance(response, http.client.HTTPResponse):
            raise TypeError("Cannot normalize this response object")

    url = response.url
    status_code = response.getcode()
    content_type = response.headers.get('Content-Type')

    return Response(
        request=request,
        content=response.read(),
        url=url,
        status_code=status_code,
        content_type=content_type,
        response=response,
    ) 

def geturl(self):
        """ Provide the geturl() method, used to be got from
            urllib.addinfourl, via. urllib.URLopener.* """
        return self.url 

def http_error_302(self, req, fp, code, msg, headers):
        if headers.dict.has_key('location'):
            infourl = urllib2.HTTPRedirectHandler.http_error_302(self, req, fp, code, msg, headers)
        else:
            infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        if not hasattr(infourl, 'status'):
            infourl.status = code
        return infourl 

def http_error_301(self, req, fp, code, msg, headers):
        if headers.dict.has_key('location'):
            infourl = urllib2.HTTPRedirectHandler.http_error_301(self, req, fp, code, msg, headers)
        else:
            infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        if not hasattr(infourl, 'status'):
            infourl.status = code
        return infourl 

def http_error_default(self, req, fp, code, msg, hdrs):
        infourl = urllib.addinfourl(fp, hdrs, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def data_open(self, req):
            # data URLs as specified in RFC 2397.
            #
            # ignores POSTed data
            #
            # syntax:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            url = req.get_full_url()

            scheme, data = url.split(':', 1)
            mediatype, data = data.split(',', 1)

            # even base64 encoded data URLs might be quoted so unquote in any case:
            data = compat_urllib_parse_unquote_to_bytes(data)
            if mediatype.endswith(';base64'):
                data = binascii.a2b_base64(data)
                mediatype = mediatype[:-7]

            if not mediatype:
                mediatype = 'text/plain;charset=US-ASCII'

            headers = email.message_from_string(
                'Content-type: %s\nContent-length: %d\n' % (mediatype, len(data)))

            return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url) 

def http_error_206(self, req, fp, code, msg, hdrs):
        # 206 Partial Content Response
        r = urllib.addinfourl(fp, hdrs, req.get_full_url())
        r.code = code
        r.msg = msg
        return r 

def http_error_206(self, req, fp, code, msg, hdrs):
        # 206 Partial Content Response
        r = urllib.addinfourl(fp, hdrs, req.get_full_url())
        r.code = code
        r.msg = msg
        return r 

def open_local_file(self, req):
        import mimetypes
        import mimetools
        host = req.get_host()
        file = req.get_selector()
        localfile = urllib.url2pathname(file)
        stats = os.stat(localfile)
        size = stats[stat.ST_SIZE]
        modified = rfc822.formatdate(stats[stat.ST_MTIME])
        mtype = mimetypes.guess_type(file)[0]
        if host:
            host, port = urllib.splitport(host)
            if port or socket.gethostbyname(host) not in self.get_names():
                raise urllib2.URLError('file not on local host')
        fo = open(localfile,'rb')
        brange = req.headers.get('Range',None)
        brange = range_header_to_tuple(brange)
        assert brange != ()
        if brange:
            (fb,lb) = brange
            if lb == '': lb = size
            if fb < 0 or fb > size or lb > size:
                raise RangeError(9, 'Requested Range Not Satisfiable')
            size = (lb - fb)
            fo = RangeableFileObject(fo, (fb,lb))
        headers = mimetools.Message(StringIO(
            'Content-Type: %s\nContent-Length: %d\nLast-modified: %s\n' %
            (mtype or 'text/plain', size, modified)))
        return urllib.addinfourl(fo, headers, 'file:'+file)


# FTP Range Support 
# Unfortunately, a large amount of base FTP code had to be copied
# from urllib and urllib2 in order to insert the FTP REST command.
# Code modifications for range support have been commented as 
# follows:
# -- range support modifications start/end here 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, headers["Location"])
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def get_response(self):
        """Returns a copy of the current response."""
        return urllib.addinfourl(StringIO(self.data), self._response.info(), self._response.geturl()) 

def _make_response(self, result, url):
        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])
        headers = httplib.HTTPMessage(StringIO(data))
        response = urllib.addinfourl(StringIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

def get_response(self):
        """Returns a copy of the current response."""
        return urllib.addinfourl(StringIO(self.data), self._response.info(), self._response.geturl()) 

def _make_response(self, result, url):
        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])
        headers = httplib.HTTPMessage(StringIO(data))
        response = urllib.addinfourl(StringIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

def http_error_206(self, req, fp, code, msg, hdrs):
        # 206 Partial Content Response
        r = urllib.addinfourl(fp, hdrs, req.get_full_url())
        r.code = code
        r.msg = msg
        return r 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def http_error_302(self, req, fp, code, msg, headers):
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        infourl.code = code
        return infourl 

def get_response(self):
        """Returns a copy of the current response."""
        return urllib.addinfourl(StringIO(self.data), self._response.info(), self._response.geturl()) 

def _make_response(self, result, url):
        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])
        headers = httplib.HTTPMessage(StringIO(data))
        response = urllib.addinfourl(StringIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

def wsopen(self, url, post, **params):
                noparam = params.pop('noparam',False)
                if noparam:
                        params = {}
                else:
                        if self.user is not None:
                                params['user'] = self.user
                        if self.password is not None:
                                params.pop('hmac', None)
                                HMAC=hmac.new(self.password)
                                for k,v in sorted(params.items()):
                                        HMAC.update("%s=%s" % (k,v))
                                params.update({'hmac':HMAC.hexdigest()})
                query = urllib.urlencode(params)
                if post:
                        body = query
                elif query:
                        url = "{}?{}".format(url, query)

                if self.debug:
                        if post:
                                print("POST:\n{}\n{!r}\n".format(url, body), file=sys.stderr)
                        else:
                                print("GET:\n{}\n".format(url), file=sys.stderr)

                class URLopener(urllib.FancyURLopener):
                        def http_error_default(self, url, fp, errcode, errmsg, headers):
                                return urllib.addinfourl(fp, headers, "http:" + url, errcode)
                try:
                        urllib._urlopener = URLopener()
                        if post:
                                resp = urllib.urlopen(url, body)
                        else:
                                resp = urllib.urlopen(url)
                except IOError as e:
                        raise WSError(url, msg=e)
                if self.debug:
                        print("RESPONSE:\n{}\n{}".format(resp.getcode(), resp.info()), file=sys.stderr)
                if resp.getcode() != 200:
                        raise WSError(url, resp.getcode(), resp.read())
                return resp 

def get_response(self):
        """Returns a copy of the current response."""
        return addinfourl(BytesIO(self.data), self._response.info(), self._response.geturl()) 

def _make_response(self, result, url):

        data = "\r\n".join(["%s: %s" % (k, v) for k, v in result.header_items])

        if PY2:
            headers = HTTPMessage(BytesIO(data))
        else:
            import email
            headers = email.message_from_string(data)

        response = addinfourl(BytesIO(result.data), headers, url)
        code, msg = result.status.split(None, 1)
        response.code, response.msg = int(code), msg
        return response 

def http_error_default(self, req, fp, code, msg, headers):
        if ((code / 100) == 3) and (code != 304):
            return self.http_error_302(req, fp, code, msg, headers)
        infourl = urllib.addinfourl(fp, headers, req.get_full_url())
        infourl.status = code
        return infourl