Python httplib.HTTPException() Examples

def ping():
    """Check if container is listening on the specified port."""
    try:
        host = sys.argv[1]
        port = int(sys.argv[2])
    except (IndexError, ValueError):
        host = '0.0.0.0'
        port = 8080

    con = None
    success = True
    try:
        con = httplib.HTTPConnection(host, port)
        con.connect()
    except (socket.error, httplib.HTTPException):
        success = False
    finally:
        if con:
            con.close()
    if success:
        logging.info('success')
        sys.exit(0)
    logging.info('failure')
    sys.exit(1) 

def get_flows(address, dpid):
    assert type(dpid) == int

    flows = []
    try:
        path = '%s%d' % (_FLOW_PATH_BASE, dpid)
        flows = json.loads(_do_request(address, path).read())[str(dpid)]
    except IOError as e:
        LOG.error('REST API(%s) is not available.', address)
        raise
    except httplib.HTTPException as e:
        if e[0].status == httplib.NOT_FOUND:
            pass  # switch already deleted
        else:
            LOG.error('REST API(%s, path=%s) request error.', address, path)
            raise
    return flows 

def _start_transaction(self, h, req):
        try:
            if req.has_data():
                data = req.data
                if hasattr(req, 'selector'):
                    h.putrequest(req.get_method() or 'POST', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                else:
                    h.putrequest(req.get_method() or 'POST', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                if not req.headers.has_key('Content-type'):
                    h.putheader('Content-type',
                                'application/x-www-form-urlencoded')
                if not req.headers.has_key('Content-length'):
                    h.putheader('Content-length', '%d' % len(data))
            else:
                if hasattr(req, 'selector'):
                    h.putrequest(req.get_method() or 'GET', req.selector, skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
                else:
                    h.putrequest(req.get_method() or 'GET', req.get_selector(), skip_host=req.has_header("Host"), skip_accept_encoding=req.has_header("Accept-encoding"))
        except (socket.error, httplib.HTTPException), err:
            raise urllib2.URLError(err) 

def _polling_loop(self):
        LOG.debug('TopologyWatcher: Enter polling loop')
        while self.is_active:
            try:
                switches_json = self.tc.list_switches().read()
                links_json = self.tc.list_links().read()
            except (SocketError, HTTPException) as e:
                LOG.debug('TopologyWatcher: REST API(%s) is not avaliable.' %
                          self.address)
                LOG.debug('        wait %d secs...' %
                          self._REST_RETRY_WAIT)
                self._call_rest_error_handler(e)
                gevent.sleep(self._REST_RETRY_WAIT)
                continue

            if self._is_updated(switches_json, links_json):
                LOG.debug('TopologyWatcher: topology updated')
                new_topo = Topology(switches_json, links_json)
                delta = new_topo - self.topo
                self.topo = new_topo

                self._call_update_handler(delta)

            gevent.sleep(self._LOOP_WAIT) 

def GetUserAttributes(self, user_email):
    """Helper to retrieve user attributes from the Admin SDK API.

    Args:
      user_email: String email address of the form user@domain.com.

    Returns:
      Dictionary of user_attributes discovered.

    Raises:
      MessageRecallError: If unable to execute the API call.
    """
    request = self._users_collection.get(userKey=user_email)
    try:
      return request.execute(
          http=credentials_utils.GetAuthorizedHttp(user_email))
    except (HttpError, httplib.HTTPException) as e:
      if e.resp.status == 403:  # If user is not an admin...
        return {}
      raise 

def _FetchUserListPage(self, next_page_token=None):
    """Helper that handles exceptions retrieving pages of users.

    Args:
      next_page_token: Used for ongoing paging of users.

    Returns:
      List of users retrieved (one page with default page size: 100 users).
    """
    # 'deleted' users are not examined.
    # https://developers.google.com/admin-sdk/directory/v1/reference/users/list
    request = self._users_collection.list(domain=self._user_domain,
                                          maxResults=_MAX_RESULT_PAGE_SIZE,
                                          query=self._search_query,
                                          pageToken=next_page_token)
    # Not infrequently seeing:
    # 'HTTPException: Deadline exceeded while waiting for HTTP response '
    # 'from URL: https://www.googleapis.com/admin/directory/v1/users'
    # '?query=email%3A6%2A&domain=capgsfishing.com&alt=json&maxResults=500'
    # Default socket timeout seems to be 5s so increasing it to 10s
    # in GetAuthorizedHttp() seems to have helped.
    return request.execute(http=self._http) 

def do_http_get(self, base_url, get_url):
        """
            Generic http get function
        """
        try:
            connection = httplib.HTTPConnection(base_url)
            connection.request('GET', get_url)
            resp = connection.getresponse()
        except httplib.HTTPException:
            print ("LOGGER: Unable to perform fetch to the"
                   " given url due to HTTPLIB exception",
                   base_url + get_url)
            return (False, None)
        except Exception, exc:
            print ("LOGGER: Unable to perform fetch to the given"
                   " url {0} due to exception {1}"
                   .format(base_url + get_url, exc))
            raise 

def send(verb, endpoint, body):
    __API_LISTENER = __IP_ADDR + ":" + __PORT
    iprint("sending to: " + __API_LISTENER)
    try:
        conn = httplib.HTTPConnection(__API_LISTENER)
        if len(body) != 0:
            conn.request(
                verb,
                endpoint,
                body,
                {"Content-Type": "application/json"}
            )
        else :
            conn.request(verb, endpoint)

        response = conn.getresponse()
        body = response.read()
        return response.status, response.reason, body
    except (httplib.HTTPException, socket.error) as ex:
        print ("Error: %s" % ex)
        quit() 

def _conn_request(self, conn, request_uri, method, body, headers):
        i = 0
        seen_bad_status_line = False
        while i < RETRIES:
            i += 1
            try:
                if hasattr(conn, 'sock') and conn.sock is None:
                    conn.connect()
                conn.request(method, request_uri, body, headers)
            except socket.timeout:
                raise
            except socket.gaierror:
                conn.close()
                raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
            except ssl_SSLError:
                conn.close()
                raise
            except socket.error, e:
                err = 0
                if hasattr(e, 'args'):
                    err = getattr(e, 'args')[0]
                else:
                    err = e.errno
                if err in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES:
                    continue  # retry on potentially transient socket errors
                raise
            except httplib.HTTPException:
                # Just because the server closed the connection doesn't apparently mean
                # that the server didn't send a response.
                if hasattr(conn, 'sock') and conn.sock is None:
                    if i < RETRIES-1:
                        conn.close()
                        conn.connect()
                        continue
                    else:
                        conn.close()
                        raise
                if i < RETRIES-1:
                    conn.close()
                    conn.connect()
                    continue 

def get_live(self, first_comment_created_at=0):
        comments_collected = self.comments

        before_count = len(comments_collected)
        try:
            comments_res = self.api.broadcast_comments(
                self.broadcast.get('id'), last_comment_ts=first_comment_created_at)
            comments = comments_res.get('comments', [])
            first_comment_created_at = (
                comments[0]['created_at_utc'] if comments else int(time.time() - 5))
            comments_collected.extend(comments)
            after_count = len(comments_collected)
            if after_count > before_count:
                broadcast = self.broadcast.copy()
                broadcast.pop('segments', None)  # save space
                broadcast['comments'] = comments_collected
                with open(self.destination_file, 'w') as outfile:
                    json.dump(broadcast, outfile, indent=2)
            self.comments = comments_collected

        except (SSLError, timeout, URLError, HTTPException, SocketError) as e:
            logger.warn('Comment downloading error: %s' % e)
        except ClientError as e:
            if e.code == 500:
                logger.warn('Comment downloading ClientError: %d %s' %
                            (e.code, e.error_response))
            elif e.code == 400 and not e.msg:
                logger.warn('Comment downloading ClientError: %d %s' %
                            (e.code, e.error_response))
            else:
                raise e
        finally:
            try:
                time.sleep(4)
            except KeyboardInterrupt:
                return first_comment_created_at
        return first_comment_created_at 

def do_open(self, req):
        host = req.host
        if not host:
            raise urllib2.URLError('no host given')

        try:
            h = self._cm.get_ready_conn(host)
            while h:
                r = self._reuse_connection(h, req, host)

                # if this response is non-None, then it worked and we're
                # done.  Break out, skipping the else block.
                if r: break

                # connection is bad - possibly closed by server
                # discard it and ask for the next free connection
                h.close()
                self._cm.remove(h)
                h = self._cm.get_ready_conn(host)
            else:
                # no (working) free connections were found.  Create a new one.
                h = self._get_connection(host)
                if DEBUG: DEBUG.info("creating new connection to %s (%d)",
                                     host, id(h))
                self._cm.add(host, h, 0)
                self._start_transaction(h, req)
                r = h.getresponse()
        except (socket.error, httplib.HTTPException), err:
            raise urllib2.URLError(err) 

def __init__(self, *args, **kw):
        unittest.TestCase.__init__(self, *args, **kw)
        try:
            self.open_mapping_file().close() # test it to report the error early
        except (IOError, HTTPException):
            self.skipTest("Could not retrieve "+self.mapfileurl) 

def test_too_many_headers(self):
        headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n'
        text = ('HTTP/1.1 200 OK\r\n' + headers)
        s = FakeSocket(text)
        r = httplib.HTTPResponse(s)
        self.assertRaises(httplib.HTTPException, r.begin) 

def _post_request(self, body, headers):
        retry_count = self.max_retries
        while True:
            logger.debug("POST %s %r %r", self.url.path, body, headers)
            try:
                self.connection.request('POST', self.url.path, body=body, headers=headers)
                response = self.connection.getresponse()
            except httplib.HTTPException as e:
                if retry_count > 0:
                    delay = math.exp(-retry_count)
                    logger.debug("HTTP protocol error, will retry in %s seconds...", delay, exc_info=True)
                    self.close()
                    self.connect()
                    time.sleep(delay)
                    retry_count -= 1
                    continue
                raise errors.InterfaceError('RPC request failed', cause=e)
            else:
                if response.status == httplib.SERVICE_UNAVAILABLE:
                    if retry_count > 0:
                        delay = math.exp(-retry_count)
                        logger.debug("Service unavailable, will retry in %s seconds...", delay, exc_info=True)
                        time.sleep(delay)
                        retry_count -= 1
                        continue
                return response 

def close(self):
        """Closes the HTTP connection to the RPC server."""
        if self.connection is not None:
            logger.debug("Closing connection to %s:%s", self.url.hostname, self.url.port)
            try:
                self.connection.close()
            except httplib.HTTPException:
                logger.warning("Error while closing connection", exc_info=True)
            self.connection = None 

def connect(self):
        """Opens a HTTP connection to the RPC server."""
        logger.debug("Opening connection to %s:%s", self.url.hostname, self.url.port)
        try:
            self.connection = httplib.HTTPConnection(self.url.hostname, self.url.port)
            self.connection.connect()
        except (httplib.HTTPException, socket.error) as e:
            raise errors.InterfaceError('Unable to connect to the specified service', e) 

def _conn_request(self, conn, request_uri, method, body, headers):
        i = 0
        seen_bad_status_line = False
        while i < RETRIES:
            i += 1
            try:
                if hasattr(conn, 'sock') and conn.sock is None:
                    conn.connect()
                conn.request(method, request_uri, body, headers)
            except socket.timeout:
                raise
            except socket.gaierror:
                conn.close()
                raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
            except ssl_SSLError:
                conn.close()
                raise
            except socket.error, e:
                err = 0
                if hasattr(e, 'args'):
                    err = getattr(e, 'args')[0]
                else:
                    err = e.errno
                if err in (errno.ENETUNREACH, errno.EADDRNOTAVAIL) and i < RETRIES:
                    continue  # retry on potentially transient socket errors
                raise
            except httplib.HTTPException:
                # Just because the server closed the connection doesn't apparently mean
                # that the server didn't send a response.
                if hasattr(conn, 'sock') and conn.sock is None:
                    if i < RETRIES-1:
                        conn.close()
                        conn.connect()
                        continue
                    else:
                        conn.close()
                        raise
                if i < RETRIES-1:
                    conn.close()
                    conn.connect()
                    continue 

def __init__(self, *args, **kw):
        unittest.TestCase.__init__(self, *args, **kw)
        try:
            self.open_mapping_file().close() # test it to report the error early
        except (IOError, HTTPException):
            self.skipTest("Could not retrieve "+self.mapfileurl) 

def get_status(host, path, file):
    try:
        conn = HTTPConnection(host)
        conn.request('HEAD', '/{0}/{1}'.format(path, file))
        res = conn.getresponse()
    except (HTTPException, socket.timeout, socket.error):
        return 0
    else:
        return res.status 

def test_http_call_failed(self, mock_conn):
        client = PrestoClient('any_host', 'any_user', 8080)
        mock_conn.side_effect = HTTPException('Error')
        self.assertFalse(client.execute_query('any_sql'))

        mock_conn.side_effect = socket.error('Error')
        self.assertFalse(client.execute_query('any_sql')) 

def get_http_exceptions():
    global _HTTP_EXCEPTIONS
    if _HTTP_EXCEPTIONS is not None:
        return _HTTP_EXCEPTIONS
    HTTP_EXCEPTIONS = (HTTPError, URLError, SocketError, HTTPException)
    _HTTP_EXCEPTIONS = HTTP_EXCEPTIONS
    return _HTTP_EXCEPTIONS 

def _conn_request(self, conn, request_uri, method, body, headers):
        i = 0
        seen_bad_status_line = False
        while i < RETRIES:
            i += 1
            try:
                if hasattr(conn, 'sock') and conn.sock is None:
                    conn.connect()
                conn.request(method, request_uri, body, headers)
            except socket.timeout:
                raise
            except socket.gaierror:
                conn.close()
                raise ServerNotFoundError("Unable to find the server at %s" % conn.host)
            except ssl_SSLError:
                conn.close()
                raise
            except socket.error, e:
                err = 0
                if hasattr(e, 'args'):
                    err = getattr(e, 'args')[0]
                else:
                    err = e.errno
                if err == errno.ECONNREFUSED: # Connection refused
                    raise
            except httplib.HTTPException:
                # Just because the server closed the connection doesn't apparently mean
                # that the server didn't send a response.
                if hasattr(conn, 'sock') and conn.sock is None:
                    if i < RETRIES-1:
                        conn.close()
                        conn.connect()
                        continue
                    else:
                        conn.close()
                        raise
                if i < RETRIES-1:
                    conn.close()
                    conn.connect()
                    continue 

def get_live(self, first_comment_created_at=0):
        comments_collected = self.comments
        commenter_ids = self.user_config.commenters or []

        before_count = len(comments_collected)
        try:
            comments_res = self.api.broadcast_comments(
                self.broadcast['id'], last_comment_ts=first_comment_created_at)
            comments = comments_res.get('comments', [])
            first_comment_created_at = (
                comments[0]['created_at_utc'] if comments else int(time.time() - 5))
            # save comment if it's in list of commenter IDs or if user is verified
            comments_collected.extend(
                list(filter(
                    lambda x: (str(x['user_id']) in commenter_ids or
                               x['user']['username'] in commenter_ids or
                               x['user']['is_verified']),
                    comments)))
            after_count = len(comments_collected)
            if after_count > before_count:
                # save intermediately to avoid losing comments due to unexpected errors
                broadcast = self.broadcast.copy()
                broadcast.pop('segments', None)     # save space
                broadcast['comments'] = comments_collected
                with open(self.destination_file, 'w') as outfile:
                    json.dump(broadcast, outfile, indent=2)
            self.comments = comments_collected

        except (SSLError, timeout, URLError, HTTPException, SocketError) as e:
            # Probably transient network error, ignore and continue
            self.logger.warning('Comment collection error: %s' % e)
        except ClientError as e:
            if e.code == 500:
                self.logger.warning('Comment collection ClientError: %d %s' % (e.code, e.error_response))
            elif e.code == 400 and not e.msg:   # 400 error fail but no error message
                self.logger.warning('Comment collection ClientError: %d %s' % (e.code, e.error_response))
            else:
                raise e
        finally:
            time.sleep(4)
        return first_comment_created_at 

def open_url(connection, url, return_redirect_url=False):
    '''Tries to open url and return page's html'''
    if 'goodreads.com' in url:
        url = url[url.find('goodreads.com') + len('goodreads.com'):]
    try:
        connection.request('GET', url, headers=HEADERS)
        response = connection.getresponse()
        if response.status == 301 or response.status == 302:
            if return_redirect_url:
                return response.msg['location']
            response = open_url(connection, response.msg['location'])
        else:
            response = response.read()
    except (HTTPException, socket.error):
        time.sleep(1)
        connection.close()
        connection.connect()
        connection.request('GET', url, headers=HEADERS)
        response = connection.getresponse()
        if response.status == 301 or response.status == 302:
            if return_redirect_url:
                return response.msg['location']
            response = open_url(connection, response.msg['location'])
        else:
            response = response.read()

    if 'Page Not Found' in response:
        raise PageDoesNotExist('Page not found.')

    return response 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content 

def request(self, uri, method="GET", body=None, *args, **kwargs):
    request_bytes = 0
    if body is not None:
      request_bytes = len(body)
    http_metrics.request_bytes.add(request_bytes, fields=self.fields)

    start_time = self.time_fn()
    try:
      response, content = super(InstrumentedHttp, self).request(
          uri, method, body, *args, **kwargs)
    except socket.timeout:
      self._update_metrics(http_metrics.STATUS_TIMEOUT, start_time)
      raise
    except (socket.error, socket.herror, socket.gaierror):
      self._update_metrics(http_metrics.STATUS_ERROR, start_time)
      raise
    except (httplib.HTTPException, httplib2.HttpLib2Error) as ex:
      status = http_metrics.STATUS_EXCEPTION
      if 'Deadline exceeded while waiting for HTTP response' in str(ex):
        # Raised on Appengine (gae_override/httplib.py).
        status = http_metrics.STATUS_TIMEOUT
      self._update_metrics(status, start_time)
      raise
    http_metrics.response_bytes.add(len(content), fields=self.fields)

    self._update_metrics(response.status, start_time)

    return response, content