Python urllib2.urlopen() Examples

The following are code examples for showing how to use urllib2.urlopen(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: pyblish-win   Author: pyblish   File: urllib2.py    GNU Lesser General Public License v3.0 7 votes vote down vote up
def urlopen(url, data=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
            cafile=None, capath=None, cadefault=False, context=None):
    global _opener
    if cafile or capath or cadefault:
        if context is not None:
            raise ValueError(
                "You can't pass both context and any of cafile, capath, and "
                "cadefault"
            )
        if not _have_ssl:
            raise ValueError('SSL support not available')
        context = ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH,
                                             cafile=cafile,
                                             capath=capath)
        https_handler = HTTPSHandler(context=context)
        opener = build_opener(https_handler)
    elif context:
        https_handler = HTTPSHandler(context=context)
        opener = build_opener(https_handler)
    elif _opener is None:
        _opener = opener = build_opener()
    else:
        opener = _opener
    return opener.open(url, data, timeout) 
Example 2
Project: pyblish-win   Author: pyblish   File: checkpip.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def main():
    outofdate = False

    for project, version in ensurepip._PROJECTS:
        data = json.loads(urllib2.urlopen(
            "https://pypi.python.org/pypi/{}/json".format(project),
        ).read().decode("utf8"))
        upstream_version = data["info"]["version"]

        if version != upstream_version:
            outofdate = True
            print("The latest version of {} on PyPI is {}, but ensurepip "
                  "has {}".format(project, upstream_version, version))

    if outofdate:
        sys.exit(1) 
Example 3
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_redirection(self):
        expected_response = 'We got here...'
        responses = [
            (302, [('Location', 'http://localhost:%s/somewhere_else')], ''),
            (200, [], expected_response)
        ]

        handler = self.start_server(responses)

        try:
            f = urllib2.urlopen('http://localhost:%s/' % handler.port)
            data = f.read()
            f.close()

            self.assertEqual(data, expected_response)
            self.assertEqual(handler.requests, ['/', '/somewhere_else'])
        finally:
            self.server.stop() 
Example 4
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_https_with_cafile(self):
        handler = self.start_https_server(certfile=CERT_localhost)
        import ssl
        # Good cert
        data = self.urlopen("https://localhost:%s/bizarre" % handler.port,
                            cafile=CERT_localhost)
        self.assertEqual(data, b"we care a bit")
        # Bad cert
        with self.assertRaises(urllib2.URLError):
            self.urlopen("https://localhost:%s/bizarre" % handler.port,
                         cafile=CERT_fakehostname)
        # Good cert, but mismatching hostname
        handler = self.start_https_server(certfile=CERT_fakehostname)
        with self.assertRaises(ssl.CertificateError):
            self.urlopen("https://localhost:%s/bizarre" % handler.port,
                         cafile=CERT_fakehostname) 
Example 5
Project: pyblish-win   Author: pyblish   File: test_urllib2.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def test_trivial(self):
        # A couple trivial tests

        self.assertRaises(ValueError, urllib2.urlopen, 'bogus url')

        # XXX Name hacking to get this to work on Windows.
        fname = os.path.abspath(urllib2.__file__).replace('\\', '/')

        # And more hacking to get it to work on MacOS. This assumes
        # urllib.pathname2url works, unfortunately...
        if os.name == 'riscos':
            import string
            fname = os.expand(fname)
            fname = fname.translate(string.maketrans("/.", "./"))

        if os.name == 'nt':
            file_url = "file:///%s" % fname
        else:
            file_url = "file://%s" % fname

        f = urllib2.urlopen(file_url)

        buf = f.read()
        f.close() 
Example 6
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: tbtools.py    Apache License 2.0 6 votes vote down vote up
def paste(self):
        """Create a paste and return the paste id."""
        data = json.dumps({
            'description': 'Werkzeug Internal Server Error',
            'public': False,
            'files': {
                'traceback.txt': {
                    'content': self.plaintext
                }
            }
        }).encode('utf-8')
        try:
            from urllib2 import urlopen
        except ImportError:
            from urllib.request import urlopen
        rv = urlopen('https://api.github.com/gists', data=data)
        resp = json.loads(rv.read().decode('utf-8'))
        rv.close()
        return {
            'url': resp['html_url'],
            'id': resp['id']
        } 
Example 7
Project: Flask-Python-GAE-Login-Registration   Author: orymeyer   File: tbtools.py    Apache License 2.0 6 votes vote down vote up
def paste(self):
        """Create a paste and return the paste id."""
        data = json.dumps({
            'description': 'Werkzeug Internal Server Error',
            'public': False,
            'files': {
                'traceback.txt': {
                    'content': self.plaintext
                }
            }
        }).encode('utf-8')
        try:
            from urllib2 import urlopen
        except ImportError:
            from urllib.request import urlopen
        rv = urlopen('https://api.github.com/gists', data=data)
        resp = json.loads(rv.read().decode('utf-8'))
        rv.close()
        return {
            'url': resp['html_url'],
            'id': resp['id']
        } 
Example 8
Project: crawler   Author: fst034356   File: BDTBwithbs4.py    MIT License 6 votes vote down vote up
def getpage(self, pagenum):

        try:

            url = self.baseurl + self.seeLZ + '&pn=' + str(pagenum)

            request = urllib2.Request(url)

            response = urllib2.urlopen(request)

            page = BeautifulSoup(response, "html5lib")

            return page

        except urllib2.URLError, e:

            if hasattr(e, 'reason'):

                print u"连接百度贴吧失败,错误原因", e.reason

                return None 
Example 9
Project: L.E.S.M.A   Author: NatanaelAntonioli   File: L.E.S.M.A. - Fabrica de Noobs Speedtest.py    Apache License 2.0 6 votes vote down vote up
def run(self):
		request = self.request
		try:
			if ((timeit.default_timer() - self.starttime) <= self.timeout and
					not SHUTDOWN_EVENT.isSet()):
				try:
					f = urlopen(request)
				except TypeError:
					# PY24 expects a string or buffer
					# This also causes issues with Ctrl-C, but we will concede
					# for the moment that Ctrl-C on PY24 isn't immediate
					request = build_request(self.request.get_full_url(),
											data=request.data.read(self.size))
					f = urlopen(request)
				f.read(11)
				f.close()
				self.result = sum(self.request.data.total)
			else:
				self.result = 0
		except (IOError, SpeedtestUploadTimeout):
			self.result = sum(self.request.data.total) 
Example 10
Project: neural-fingerprinting   Author: StephanZheng   File: download_images.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def download_image(image_id, url, x1, y1, x2, y2, output_dir):
    """Downloads one image, crops it, resizes it and saves it locally."""
    output_filename = os.path.join(output_dir, image_id + '.png')
    if os.path.exists(output_filename):
        # Don't download image if it's already there
        return True
    try:
        # Download image
        url_file = urlopen(url)
        if url_file.getcode() != 200:
            return False
        image_buffer = url_file.read()
        # Crop, resize and save image
        image = Image.open(BytesIO(image_buffer)).convert('RGB')
        w = image.size[0]
        h = image.size[1]
        image = image.crop((int(x1 * w), int(y1 * h), int(x2 * w),
                            int(y2 * h)))
        image = image.resize((299, 299), resample=Image.ANTIALIAS)
        image.save(output_filename)
    except IOError:
        return False
    return True 
Example 11
Project: flasky   Author: RoseOu   File: tbtools.py    MIT License 6 votes vote down vote up
def paste(self):
        """Create a paste and return the paste id."""
        data = json.dumps({
            'description': 'Werkzeug Internal Server Error',
            'public': False,
            'files': {
                'traceback.txt': {
                    'content': self.plaintext
                }
            }
        }).encode('utf-8')
        try:
            from urllib2 import urlopen
        except ImportError:
            from urllib.request import urlopen
        rv = urlopen('https://api.github.com/gists', data=data)
        resp = json.loads(rv.read().decode('utf-8'))
        rv.close()
        return {
            'url': resp['html_url'],
            'id': resp['id']
        } 
Example 12
Project: flasky   Author: RoseOu   File: utils.py    MIT License 6 votes vote down vote up
def is_url_connectable(port):
    """
    Tries to connect to the HTTP server at /status path
    and specified port to see if it responds successfully.

    :Args:
     - port: The port to connect.
    """
    try:
        from urllib import request as url_request
    except ImportError:
        import urllib2 as url_request

    try:
        res = url_request.urlopen("http://127.0.0.1:%s/status" % port)
        if res.getcode() == 200:
            return True
        else:
            return False
    except:
        return False 
Example 13
Project: core   Author: lifemapper   File: gbifquery.py    GNU General Public License v3.0 6 votes vote down vote up
def _getGBIFDownloadRequestResults(self, taxonKey):
      if not self._signedIn:
         raise LMError('Must be signed in to access secure GBIF services')
      queryParams = GBIF.QUERY_PARAMS[GBIF.OCCURRENCE_SERVICE].copy()
      queryParams[GBIF.REQUEST_TAXON_KEY] = taxonKey
      
      jsonPredicate = self._assembleDownloadPredicate(queryParams)
      url = '%s/%s/%s/%s'% (GBIF.REST_URL, GBIF.OCCURRENCE_SERVICE, 
                         GBIF.DOWNLOAD_COMMAND, GBIF.DOWNLOAD_REQUEST_COMMAND)
      headers = {'Content-Type': 'application/json'}
   
      # POST
      self._gbifQueryTime = dt.gmt().mjd
      try:
         req = urllib2.Request(url, jsonPredicate, headers)
         response = urllib2.urlopen(req)
      except urllib2.HTTPError, e:
         msg = ('Failed GBIF request with url %s; predicate %s (args: %s, reason: %s)' 
                % (url, str(jsonPredicate), str(e.args), str(e.reason)))
         self.log.error(msg)
         self._nubUUID = None
         raise LmHTTPError(e.code, msg=msg) 
Example 14
Project: dynamic-training-with-apache-mxnet-on-aws   Author: awslabs   File: diagnose.py    Apache License 2.0 6 votes vote down vote up
def test_connection(name, url, timeout=10):
    """Simple connection test"""
    urlinfo = urlparse(url)
    start = time.time()
    try:
        ip = socket.gethostbyname(urlinfo.netloc)
    except Exception as e:
        print('Error resolving DNS for {}: {}, {}'.format(name, url, e))
        return
    dns_elapsed = time.time() - start
    start = time.time()
    try:
        _ = urlopen(url, timeout=timeout)
    except Exception as e:
        print("Error open {}: {}, {}, DNS finished in {} sec.".format(name, url, e, dns_elapsed))
        return
    load_elapsed = time.time() - start
    print("Timing for {}: {}, DNS: {:.4f} sec, LOAD: {:.4f} sec.".format(name, url, dns_elapsed, load_elapsed)) 
Example 15
Project: sqliv   Author: the-robot   File: web.py    GNU General Public License v3.0 6 votes vote down vote up
def gethtml(url, lastURL=False):
    """return HTML of the given url"""

    if not (url.startswith("http://") or url.startswith("https://")):
        url = "http://" + url

    header = useragents.get()
    request = urllib2.Request(url, None, header)
    html = None

    try:
        reply = urllib2.urlopen(request, timeout=10)

    except urllib2.HTTPError, e:
        # read html content anyway for reply with HTTP500
        if e.getcode() == 500:
            html = e.read()
        #print >> sys.stderr, "[{}] HTTP error".format(e.code)
        pass 
Example 16
Project: sqliv   Author: the-robot   File: reverseip.py    GNU General Public License v3.0 6 votes vote down vote up
def reverseip(url):
    """return domains from given the same server"""

    # get only domain name
    url = urlparse(url).netloc if urlparse(url).netloc != '' else urlparse(url).path.split("/")[0]

    source = "http://domains.yougetsignal.com/domains.php"
    useragent = useragents.get()
    contenttype = "application/x-www-form-urlencoded; charset=UTF-8"

    # POST method
    opener = urllib2.build_opener(
        urllib2.HTTPHandler(), urllib2.HTTPSHandler())
    data = urllib.urlencode([('remoteAddress', url), ('key', '')])

    request = urllib2.Request(source, data)
    request.add_header("Content-type", contenttype)
    request.add_header("User-Agent", useragent)

    try:
        result = urllib2.urlopen(request).read()

    except urllib2.HTTPError, e:
        print >> sys.stderr, "[{}] HTTP error".format(e.code) 
Example 17
Project: sqliv   Author: the-robot   File: yahoo.py    GNU General Public License v3.0 6 votes vote down vote up
def search(self, query, per_page=10, pages=1):
        """search urls from yahoo search"""

        # store searched urls
        urls = []

        for page in range(pages):
            yahoosearch = self.yahoosearch % (query, per_page, (pages+1)*10)

            request = urllib2.Request(yahoosearch)
            request.add_header("Content-type", self.contenttype)
            request.add_header("User-Agent", self.useragent)

            result = urllib2.urlopen(request).read()
            urls += self.parse_links(result)

        return urls 
Example 18
Project: malcode   Author: moonsea   File: zeustracker.py    GNU General Public License v3.0 5 votes vote down vote up
def GetPage(url):
    htmlpage = urllib2.urlopen(url).read()
    soup = BeautifulSoup(htmlpage)

    return soup 
Example 19
Project: malcode   Author: moonsea   File: vxheaven.py    GNU General Public License v3.0 5 votes vote down vote up
def GetPage(url):
    htmlpage = urllib2.urlopen(url).read()
    soup = BeautifulSoup(htmlpage)

    return soup 
Example 20
Project: pyblish-win   Author: pyblish   File: urllib2.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def __str__(self):
        return '<urlopen error %s>' % self.reason 
Example 21
Project: pyblish-win   Author: pyblish   File: urllib2.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def open_local_file(self, req):
        import email.utils
        import mimetypes
        host = req.get_host()
        filename = req.get_selector()
        localfile = url2pathname(filename)
        try:
            stats = os.stat(localfile)
            size = stats.st_size
            modified = email.utils.formatdate(stats.st_mtime, usegmt=True)
            mtype = mimetypes.guess_type(filename)[0]
            headers = mimetools.Message(StringIO(
                'Content-type: %s\nContent-length: %d\nLast-modified: %s\n' %
                (mtype or 'text/plain', size, modified)))
            if host:
                host, port = splitport(host)
            if not host or \
                (not port and _safe_gethostbyname(host) in self.get_names()):
                if host:
                    origurl = 'file://' + host + filename
                else:
                    origurl = 'file://' + filename
                return addinfourl(open(localfile, 'rb'), headers, origurl)
        except OSError, msg:
            # urllib2 users shouldn't expect OSErrors coming from urlopen()
            raise URLError(msg) 
Example 22
Project: pyblish-win   Author: pyblish   File: register.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def classifiers(self):
        ''' Fetch the list of classifiers from the server.
        '''
        response = urllib2.urlopen(self.repository+'?:action=list_classifiers')
        log.info(response.read()) 
Example 23
Project: pyblish-win   Author: pyblish   File: test_support.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def open_urlresource(url, check=None):
    import urlparse, urllib2

    filename = urlparse.urlparse(url)[2].split('/')[-1] # '/': it's URL!

    fn = os.path.join(os.path.dirname(__file__), "data", filename)

    def check_valid_file(fn):
        f = open(fn)
        if check is None:
            return f
        elif check(f):
            f.seek(0)
            return f
        f.close()

    if os.path.exists(fn):
        f = check_valid_file(fn)
        if f is not None:
            return f
        unlink(fn)

    # Verify the requirement before downloading the file
    requires('urlfetch')

    print >> get_original_stdout(), '\tfetching %s ...' % url
    f = urllib2.urlopen(url, timeout=15)
    try:
        with open(fn, "wb") as out:
            s = f.read()
            while s:
                out.write(s)
                s = f.read()
    finally:
        f.close()

    f = check_valid_file(fn)
    if f is not None:
        return f
    raise TestFailed('invalid resource "%s"' % fn) 
Example 24
Project: pyblish-win   Author: pyblish   File: test_urllib2net.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_urlwithfrag(self):
        urlwith_frag = "http://www.pythontest.net/index.html#frag"
        with test_support.transient_internet(urlwith_frag):
            req = urllib2.Request(urlwith_frag)
            res = urllib2.urlopen(req)
            self.assertEqual(res.geturl(),
                    "http://www.pythontest.net/index.html#frag") 
Example 25
Project: pyblish-win   Author: pyblish   File: test_urllib2net.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_sites_no_connection_close(self):
        # Some sites do not send Connection: close header.
        # Verify that those work properly. (#issue12576)

        URL = 'http://www.imdb.com' # No Connection:close
        with test_support.transient_internet(URL):
            req = urllib2.urlopen(URL)
            res = req.read()
            self.assertTrue(res) 
Example 26
Project: pyblish-win   Author: pyblish   File: test_urllib2net.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def _test_urls(self, urls, handlers, retry=True):
        import time
        import logging
        debug = logging.getLogger("test_urllib2").debug

        urlopen = urllib2.build_opener(*handlers).open
        if retry:
            urlopen = _wrap_with_retry_thrice(urlopen, urllib2.URLError)

        for url in urls:
            if isinstance(url, tuple):
                url, req, expected_err = url
            else:
                req = expected_err = None
            with test_support.transient_internet(url):
                debug(url)
                try:
                    f = urlopen(url, req, TIMEOUT)
                except EnvironmentError as err:
                    debug(err)
                    if expected_err:
                        msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" %
                               (expected_err, url, req, type(err), err))
                        self.assertIsInstance(err, expected_err, msg)
                except urllib2.URLError as err:
                    if isinstance(err[0], socket.timeout):
                        print >>sys.stderr, "<timeout: %s>" % url
                        continue
                    else:
                        raise
                else:
                    try:
                        with test_support.transient_internet(url):
                            buf = f.read()
                            debug("read %d bytes" % len(buf))
                    except socket.timeout:
                        print >>sys.stderr, "<timeout: %s>" % url
                    f.close()
            debug("******** next url coming up...")
            time.sleep(0.1) 
Example 27
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_basic_auth_httperror(self):
        ah = urllib2.HTTPBasicAuthHandler()
        ah.add_password(self.REALM, self.server_url, self.USER,
                        self.INCORRECT_PASSWD)
        urllib2.install_opener(urllib2.build_opener(ah))
        self.assertRaises(urllib2.HTTPError, urllib2.urlopen, self.server_url) 
Example 28
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def urlopen(self, url, data=None, **kwargs):
        l = []
        f = urllib2.urlopen(url, data, **kwargs)
        try:
            # Exercise various methods
            l.extend(f.readlines(200))
            l.append(f.readline())
            l.append(f.read(1024))
            l.append(f.read())
        finally:
            f.close()
        return b"".join(l) 
Example 29
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_404(self):
        expected_response = 'Bad bad bad...'
        handler = self.start_server([(404, [], expected_response)])

        try:
            try:
                urllib2.urlopen('http://localhost:%s/weeble' % handler.port)
            except urllib2.URLError, f:
                pass
            else: 
Example 30
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_200(self):
        expected_response = 'pycon 2008...'
        handler = self.start_server([(200, [], expected_response)])

        try:
            f = urllib2.urlopen('http://localhost:%s/bizarre' % handler.port)
            data = f.read()
            f.close()

            self.assertEqual(data, expected_response)
            self.assertEqual(handler.requests, ['/bizarre'])
        finally:
            self.server.stop() 
Example 31
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_https(self):
        handler = self.start_https_server()
        context = ssl.create_default_context(cafile=CERT_localhost)
        data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context)
        self.assertEqual(data, b"we care a bit") 
Example 32
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_https_with_cadefault(self):
        handler = self.start_https_server(certfile=CERT_localhost)
        # Self-signed cert should fail verification with system certificate store
        with self.assertRaises(urllib2.URLError):
            self.urlopen("https://localhost:%s/bizarre" % handler.port,
                         cadefault=True) 
Example 33
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_https_sni(self):
        if ssl is None:
            self.skipTest("ssl module required")
        if not ssl.HAS_SNI:
            self.skipTest("SNI support required in OpenSSL")
        sni_name = [None]
        def cb_sni(ssl_sock, server_name, initial_context):
            sni_name[0] = server_name
        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
        context.set_servername_callback(cb_sni)
        handler = self.start_https_server(context=context, certfile=CERT_localhost)
        context = ssl.create_default_context(cafile=CERT_localhost)
        self.urlopen("https://localhost:%s" % handler.port, context=context)
        self.assertEqual(sni_name[0], "localhost") 
Example 34
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_sending_headers(self):
        handler = self.start_server([(200, [], "we don't care")])

        try:
            req = urllib2.Request("http://localhost:%s/" % handler.port,
                                  headers={'Range': 'bytes=20-39'})
            urllib2.urlopen(req)
            self.assertEqual(handler.headers_received['Range'], 'bytes=20-39')
        finally:
            self.server.stop() 
Example 35
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_info(self):
        handler = self.start_server([(200, [], "we don't care")])

        try:
            open_url = urllib2.urlopen("http://localhost:%s" % handler.port)
            info_obj = open_url.info()
            self.assertIsInstance(info_obj, mimetools.Message,
                                  "object returned by 'info' is not an "
                                  "instance of mimetools.Message")
            self.assertEqual(info_obj.getsubtype(), "plain")
        finally:
            self.server.stop() 
Example 36
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_geturl(self):
        # Make sure same URL as opened is returned by geturl.
        handler = self.start_server([(200, [], "we don't care")])

        try:
            open_url = urllib2.urlopen("http://localhost:%s" % handler.port)
            url = open_url.geturl()
            self.assertEqual(url, "http://localhost:%s" % handler.port)
        finally:
            self.server.stop() 
Example 37
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_bad_address(self):
        # Make sure proper exception is raised when connecting to a bogus
        # address.

        # as indicated by the comment below, this might fail with some ISP,
        # so we run the test only when -unetwork/-uall is specified to
        # mitigate the problem a bit (see #17564)
        test_support.requires('network')
        self.assertRaises(IOError,
                          # Given that both VeriSign and various ISPs have in
                          # the past or are presently hijacking various invalid
                          # domain name requests in an attempt to boost traffic
                          # to their own sites, finding a domain name to use
                          # for this test is difficult.  RFC2606 leads one to
                          # believe that '.invalid' should work, but experience
                          # seemed to indicate otherwise.  Single character
                          # TLDs are likely to remain invalid, so this seems to
                          # be the best choice. The trailing '.' prevents a
                          # related problem: The normal DNS resolver appends
                          # the domain names from the search path if there is
                          # no '.' the end and, and if one of those domains
                          # implements a '*' rule a result is returned.
                          # However, none of this will prevent the test from
                          # failing if the ISP hijacks all invalid domain
                          # requests.  The real solution would be to be able to
                          # parameterize the framework with a mock resolver.
                          urllib2.urlopen, "http://sadflkjsasf.i.nvali.d./") 
Example 38
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_iteration(self):
        expected_response = "pycon 2008..."
        handler = self.start_server([(200, [], expected_response)])
        try:
            data = urllib2.urlopen("http://localhost:%s" % handler.port)
            for line in data:
                self.assertEqual(line, expected_response)
        finally:
            self.server.stop() 
Example 39
Project: pyblish-win   Author: pyblish   File: test_urllib2_localnet.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def ztest_line_iteration(self):
        lines = ["We\n", "got\n", "here\n", "verylong " * 8192 + "\n"]
        expected_response = "".join(lines)
        handler = self.start_server([(200, [], expected_response)])
        try:
            data = urllib2.urlopen("http://localhost:%s" % handler.port)
            for index, line in enumerate(data):
                self.assertEqual(line, lines[index],
                                 "Fetched line number %s doesn't match expected:\n"
                                 "    Expected length was %s, got %s" %
                                 (index, len(lines[index]), len(line)))
        finally:
            self.server.stop()
        self.assertEqual(index + 1, len(lines)) 
Example 40
Project: pyblish-win   Author: pyblish   File: test_urllib2.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def test_cafile_and_context(self):
        context = ssl.create_default_context()
        with self.assertRaises(ValueError):
            urllib2.urlopen(
                "https://localhost", cafile="/nonexistent/path", context=context
            ) 
Example 41
Project: RF-Monitor   Author: EarToEarOak   File: push.py    GNU General Public License v2.0 5 votes vote down vote up
def __send(self, uri, data):
        req = urllib2.Request(uri)
        req.add_header('Content-Type', 'application/json')

        event = None
        try:
            urllib2.urlopen(req, data)
        except ValueError as error:
            event = Event(Events.PUSH_ERROR, msg=error.message)
        except URLError as error:
            event = Event(Events.PUSH_ERROR, msg=error.reason.strerror)

        if event is not None:
            self._failed.append(data)
            post_event(self._handler, event) 
Example 42
Project: World-Bank-Python   Author: dpfens   File: api.py    MIT License 5 votes vote down vote up
def _request(url, **kwargs):
    headers = kwargs.get('headers', {})
    parameters = kwargs.get('parameters', {})
    data = kwargs.get('data', {})

    parameters.setdefault('format', 'json')

    encoded_parameters = urlencode(parameters)
    if encoded_parameters:
        url = '%s?%s' % (url, encoded_parameters)

    logger.debug(url)
    request = Request(url)
    for key, value in headers.items():
        request.add_header(key, value)

    for key, value in data.items():
        request.add_data(key, value)

    try:
        response = urlopen(request)
    except Exception as e:
        logger.error(e)
        raise
    raw_response_data = response.read().decode("utf-8")
    logger.debug(raw_response_data)
    response_data = json.loads(raw_response_data)
    return response_data 
Example 43
Project: mietmap-scraper   Author: CodeforKarlsruhe   File: scrape.py    MIT License 5 votes vote down vote up
def download_as_unicode(url):
    """
    Download document at URL and return it as a Unicode string.
    """
    request = urllib2.urlopen(url)
    return unicode(request.read(), request.headers.getparam('charset')) 
Example 44
Project: slidoc   Author: mitotic   File: sdprint.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def http_get(url, params_dict):
    try:
        return urllib2.urlopen(url + ('?'+urllib.urlencode(params_dict) if params_dict else '')).read()
    except Exception, excp:
        sys.exit('ERROR in accessing GET URL %s: %s' % (url, excp)) 
Example 45
Project: slidoc   Author: mitotic   File: sdprint.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def http_post(url, params_dict):
    data = urllib.urlencode(params_dict)
    req = urllib2.Request(url, data)
    try:
        response = urllib2.urlopen(req)
    except Exception, excp:
        sys.exit('ERROR in accessing POST URL %s: %s' % (url, excp)) 
Example 46
Project: slidoc   Author: mitotic   File: sdviewer.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def do_GET(self):
        parsed_url = urlparse.urlparse(self.path)
        params = urlparse.parse_qs(parsed_url.query)
        if parsed_url.path.startswith('/http:') or parsed_url.path.startswith('/https:'):
            url = parsed_url.path[1:]
            if parsed_url.query:
                url += '?' + parsed_url.query
            parsed_suburl = urlparse.urlparse(url)
            filename = os.path.basename(parsed_suburl.path) or 'file.md'
            req = urllib2.Request(url)
            try:
                response = urllib2.urlopen(req)
                file = StringIO(response.read())
            except Exception, excp:
                self.respond(cgi.escape('ERROR in accessing URL %s: %s' % (url, excp)))
                return
            cmd_args = []
            for arg in ("pace", "gsheet_url"):
                if arg in params and params[arg][0]:
                    cmd_args.append("--"+arg+"="+params[arg][0])
                    if arg == "gsheet_url":
                        cmd_args.append("--gsheet_login=")
            errmsg, outname, html, messages = process_files([file], [filename], cmd_args)
            if errmsg:
                self.respond('ERROR: '+errmsg)
            else:
                self.respond(html) 
Example 47
Project: slidoc   Author: mitotic   File: sliauth.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def http_post(url, params_dict=None, add_size_info=False):
    req = urllib2.Request(url, urllib.urlencode(params_dict)) if params_dict else urllib2.Request(url)
    try:
        response = urllib2.urlopen(req)
    except Exception, excp:
        raise Exception('ERROR in accessing URL %s: %s' % (url, excp)) 
Example 48
Project: slidoc   Author: mitotic   File: sdproxy.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def lockUpstreamProxy(sheetName, unlock=False):
    # Lock (or unlock) sheet in upstream proxy
    lockURL = Settings['lock_proxy_url']
    if Settings['site_name']:
        lockURL += '/' + Settings['site_name']
    lockURL += '/_%s/%s' % ('unlock' if unlock else 'lock', sheetName)
    req = urllib2.Request(lockURL+'?token='+Settings['auth_key']+'&type=proxy')
    response = urllib2.urlopen(req)
    if unlock:
        Locked_proxy_sheets.discard(sheetName)
    else:
        Locked_proxy_sheets.add(sheetName)
    if Settings['debug']:
        print("DEBUG:lockUpstreamProxy: %s %s %s (%s)" % (unlock, sheetName, lockURL, response.read()), file=sys.stderr) 
Example 49
Project: Learning-Concurrency-in-Python   Author: PacktPublishing   File: ioBottleneck2.py    MIT License 5 votes vote down vote up
def getLinks():
  req = urllib2.urlopen('http://www.example.com')
  soup = BeautifulSoup(req.read())
  for link in soup.findAll('a'):
    linkArray.append(link.get('href'))
    print(len(linkArray)) 
Example 50
Project: L.E.S.M.A   Author: NatanaelAntonioli   File: L.E.S.M.A. - Fabrica de Noobs Speedtest.py    Apache License 2.0 5 votes vote down vote up
def catch_request(request):
	"""Helper function to catch common exceptions encountered when
	establishing a connection with a HTTP/HTTPS request

	"""

	try:
		uh = urlopen(request)
		return uh, False
	except HTTP_ERRORS:
		e = get_exception()
		return None, e