Python aiohttp.ClientResponseError() Examples

def _get(self, url, data=None, headers=None, method='GET'):
        page = ''
        try:
            timeout = aiohttp.ClientTimeout(total=self._timeout)
            async with self._sem_provider, self._session.request(
                method, url, data=data, headers=headers, timeout=timeout
            ) as resp:
                page = await resp.text()
                if resp.status != 200:
                    log.debug(
                        'url: %s\nheaders: %s\ncookies: %s\npage:\n%s'
                        % (url, resp.headers, resp.cookies, page)
                    )
                    raise BadStatusError('Status: %s' % resp.status)
        except (
            UnicodeDecodeError,
            BadStatusError,
            asyncio.TimeoutError,
            aiohttp.ClientOSError,
            aiohttp.ClientResponseError,
            aiohttp.ServerDisconnectedError,
        ) as e:
            page = ''
            log.debug('%s is failed. Error: %r;' % (url, e))
        return page 

def add_safe(self, name, url, author_id):
		"""Try to add an emote. Returns a string that should be sent to the user."""
		if not re.fullmatch(r'\w{2,32}', name, re.ASCII):
			return _(
				'{name} is not a valid emote name; use 2–32 English letters, numbers and underscores.'
			).format(name=discord.utils.escape_mentions(name))
		try:
			emote = await self.add_from_url(name, url, author_id)
		except discord.HTTPException as ex:
			return (
				_('An error occurred while creating the emote:\n')
				+ utils.format_http_exception(ex))
		except ValueError:
			return _('Error: Invalid URL.')
		except aiohttp.ServerDisconnectedError:
			return _('Error: The connection was closed early by the remote host.')
		except aiohttp.ClientResponseError as exc:
			raise errors.HTTPException(exc.status)
		else:
			return _('Emote {emote} successfully created.').format(emote=emote) 

def fetch(self, session, url):
        """Fetch URL.
        
        :param session: aiohttp.ClientSession
        :param url: URL
        :return: Response in JSON
        """
        print(url)
        try:
            with async_timeout.timeout(Settings.timeout):
                async with session.get(url) as response:
                    return await response.json()
        except asyncio.TimeoutError:
            return None
        except aiohttp.ClientResponseError:
            return None 

def _parse(response):
    try:
        data = await response.json()
        if data is None:
            raise ValueError()
    except (ValueError, json.JSONDecodeError, aiohttp.ClientResponseError):
        text = await response.text()
        raise exception.BadHTTPResponse(response.status, text, response)

    if data['ok']:
        return data['result']
    else:
        description, error_code = data['description'], data['error_code']

        # Look for specific error ...
        for e in exception.TelegramError.__subclasses__():
            n = len(e.DESCRIPTION_PATTERNS)
            if any(map(re.search, e.DESCRIPTION_PATTERNS, n*[description], n*[re.IGNORECASE])):
                raise e(description, error_code, data)

        # ... or raise generic error
        raise exception.TelegramError(description, error_code, data) 

def get_picture_urls(dates, verbose=False):
    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
    tasks = [get_picture_url(date, semaphore) for date in dates]
    urls = []
    count = 0
    # get results as jobs are done
    for job in asyncio.as_completed(tasks, timeout=GLOBAL_TIMEOUT):
        try:
            url = yield from job
        except NoPictureForDate as exc:
            if verbose:
                print('*** {!r} ***'.format(exc))
            continue
        except aiohttp.ClientResponseError as exc:
            print('****** {!r} ******'.format(exc))
            continue
        count += 1
        if verbose:
            print(format(count, '3d'), end=' ')
            print(url.split('/')[-1])
        else:
            print(url)
        urls.append(url)
    return urls 

def read_crd(
        *,
        resource: resources.Resource,
        default: Union[_T, _UNSET] = _UNSET.token,
        context: Optional[auth.APIContext] = None,  # injected by the decorator
) -> Union[bodies.RawBody, _T]:
    if context is None:
        raise RuntimeError("API instance is not injected by the decorator.")

    try:
        response = await context.session.get(
            url=CRD_CRD.get_url(server=context.server, name=resource.name),
        )
        response.raise_for_status()
        respdata = await response.json()
        return cast(bodies.RawBody, respdata)

    except aiohttp.ClientResponseError as e:
        if e.status in [403, 404] and not isinstance(default, _UNSET):
            return default
        raise 

def download(self, session, update):
        if self.path is None:
            return
        if update and not await self.needsUpdate():
            return

        self.path.parent.mkdir(parents=True, exist_ok=True)

        async with session.get(self.url) as response:
            try:
                response.raise_for_status()
            except aiohttp.ClientResponseError as e:
                # I don't like returning Exceptions, but I can't find a better way to pass a single error in an async loop
                return (self, e)
            with open(self.path, "wb") as out_file:
                while True:
                    chunk = await response.content.read(8192)
                    if not chunk:
                        break
                    out_file.write(chunk)

        url_timestamp = getTimestamp(self.lastModified)
        os.utime(self.path, (url_timestamp, url_timestamp))
        return (self, None) 

def request(self, method, url, data, headers, **kwargs):
        try:
            # print(url)
            with async_timeout.timeout(TIMEOUT):
                async with self._session.request(
                    method, url, headers=headers, data=data
                ) as response:
                    # print(response)
                    if response.status == 200 or response.status == 202:
                        return await response.json(loads=json_loads)
                    else:
                        raise ClientResponseError(
                            response.request_info,
                            response.history,
                            status=response.status,
                            message=response.reason,
                        )
        except TimeoutError:
            raise TimeoutError("Timeout error")
        except Exception:
            raise 

def hastebin(self, ctx, *, message):
        """Upload text to hastebin"""
        haste_url = os.environ.get("HASTE_URL", "https://hasteb.in")

        try:
            async with self.bot.session.post(
                haste_url + "/documents", data=message
            ) as resp:
                key = (await resp.json())["key"]
                embed = Embed(
                    title="Your uploaded file",
                    color=self.bot.main_color,
                    description=f"{haste_url}/" + key,
                )
        except (JSONDecodeError, ClientResponseError, IndexError):
            embed = Embed(
                color=self.bot.main_color,
                description="Something went wrong. "
                "We're unable to upload your text to hastebin.",
            )
            embed.set_footer(text="Hastebin Plugin")
        await ctx.send(embed=embed) 

def __init__(self, exception):
        if isinstance(exception, HTTPError):
            super().__init__(
                "Error by connection with Instagram to '%s' with response code '%s'" % (
                    exception.request.url,
                    exception.response.status_code,
                ),
            )
            self.request = exception.request
            self.response = exception.response
        elif isinstance(exception, ClientResponseError):
            super().__init__(
                "Error by connection with Instagram to '%s' with response code '%s'" % (
                    exception.request_info.real_url,
                    exception.status,
                ),
            ) 

def get_picture_urls(dates, verbose=False):
    semaphore = asyncio.Semaphore(MAX_CONCURRENT_REQUESTS)
    tasks = [get_picture_url(date, semaphore) for date in dates]
    urls = []
    count = 0
    # get results as jobs are done
    for job in asyncio.as_completed(tasks, timeout=GLOBAL_TIMEOUT):
        try:
            url = yield from job
        except NoPictureForDate as exc:
            if verbose:
                print('*** {!r} ***'.format(exc))
            continue
        except aiohttp.ClientResponseError as exc:
            print('****** {!r} ******'.format(exc))
            continue
        count += 1
        if verbose:
            print(format(count, '3d'), end=' ')
            print(url.split('/')[-1])
        else:
            print(url)
        urls.append(url)
    return urls 

def _raise_response_exceptions(response):
    try:
        response.raise_for_status()
    except ClientResponseError as err:
        if err.status == 422:
            raise AugustApiAIOHTTPError(
                "The operation failed because the bridge (connect) is offline.",
            ) from err
        if err.status == 423:
            raise AugustApiAIOHTTPError(
                "The operation failed because the bridge (connect) is in use.",
            ) from err
        if err.status == 408:
            raise AugustApiAIOHTTPError(
                "The operation timed out because the bridge (connect) failed to respond.",
            ) from err
        raise err 

def test_single_proxy(self, proxy):
        """
        text one proxy, if valid, put them to usable_proxies.
        """
        try:
            async with aiohttp.ClientSession() as session:
                try:
                    if isinstance(proxy, bytes):
                        proxy = proxy.decode('utf-8')
                    real_proxy = 'http://' + proxy
                    print('Testing', proxy)
                    async with session.get(self.test_api, proxy=real_proxy, timeout=get_proxy_timeout) as response:
                        if response.status == 200:
                            self._conn.put(proxy)
                            print('Valid proxy', proxy)
                except (ProxyConnectionError, TimeoutError, ValueError):
                    print('Invalid proxy', proxy)
        except (ServerDisconnectedError, ClientResponseError,ClientConnectorError) as s:
            print(s)
            pass 

def refresh_token(self, refresh_token):
        """
        :param refresh_token: an openid refresh-token from a previous token request
        """
        async with self._client_session() as client:
            well_known = await self._get_well_known(client)

            try:
                return await self._post(
                    client,
                    well_known['token_endpoint'],
                    data={
                        'grant_type': GRANT_TYPE_REFRESH_TOKEN,
                        'refresh_token': refresh_token,
                    }
                )
            except aiohttp.ClientResponseError:
                raise ConfigException('oidc: failed to refresh access token') 

def get_text_from_url(self, url: str) -> str:
        """Fetch content at **url** and return as text

        - On non-permanent errors (429, 502, 503, 504), the GET is retried 10 times with
          increasing wait times according to fibonacci series.
        - Permanent errors raise a ClientResponseError
        """
        if self.cache and url in self.cache["url_text"]:
            return self.cache["url_text"][url]

        async with self.session.get(url) as resp:
            resp.raise_for_status()
            res = await resp.text()

        if self.cache:
            self.cache["url_text"][url] = res

        return res 

def leave_room(self, user: User, room: Room) -> bool:
        """Remove **user** from **room**"""
        try:
            await self._make_request('DELETE', self._ROOM_USERS,
                                     {'roomId': room.id, 'userId': user.id})
        except aiohttp.ClientResponseError as exc:
            if exc.code in (404,):
                return False
        return True 

def get_request(self, *args, **kwargs):
        try:
            return await self.session.get(*args, **kwargs)
        except aiohttp.ClientResponseError as exception:
            raise InternetException(exception) 

def post_request(self, *args, **kwargs):
        try:
            return await self.session.post(*args, **kwargs)
        except aiohttp.ClientResponseError as exception:
            raise InternetException(exception) 

def _query(server, method, parameters, timeout=DEFAULT_TIMEOUT, verify_ssl=True):
    """Formats and performs the asynchronous query against the API

    :param server: The server to query.
    :param method: The method name.
    :param parameters: A dict of parameters to send
    :param timeout: The timeout to make the call, in seconds. By default, this is 300 seconds (or 5 minutes).
    :param verify_ssl: Whether or not to verify SSL connections
    :return: The JSON-decoded result from the server
    :raise MyGeotabException: Raises when an exception occurs on the MyGeotab server
    :raise TimeoutException: Raises when the request does not respond after some time.
    :raise aiohttp.ClientResponseError: Raises when there is an HTTP status code that indicates failure.
    """
    api_endpoint = api.get_api_url(server)
    params = dict(id=-1, method=method, params=parameters)
    headers = get_headers()
    conn = aiohttp.TCPConnector(ssl=ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) if verify_ssl else False)
    try:
        async with aiohttp.ClientSession(connector=conn) as session:
            response = await session.post(
                api_endpoint, data=json_serialize(params), headers=headers, timeout=timeout, allow_redirects=True
            )
            response.raise_for_status()
            content_type = response.headers.get("Content-Type")
            body = await response.text()
    except TimeoutError:
        raise TimeoutException(server)
    if content_type and "application/json" not in content_type.lower():
        return body
    return api._process(json_deserialize(body)) 

def update_vehicle_statusreport(self):
        if not self.support_status_report:
            return

        try:
            status = await self._audi_service.get_stored_vehicle_data(self._vehicle.vin)
            self._vehicle.fields = {
                status.data_fields[i].name: status.data_fields[i].value
                for i in range(0, len(status.data_fields))
            }
            self._vehicle.state["last_update_time"] = datetime.strptime(
                status.data_fields[0].send_time, "%Y-%m-%dT%H:%M:%S"
            )

        except TimeoutError:
            raise
        except ClientResponseError as resp_exception:
            if resp_exception.status == 403 or resp_exception.status == 502:
                self.support_status_report = False
            else:
                self.log_exception_once(
                    resp_exception,
                    "Unable to obtain the vehicle status report of {}".format(
                        self._vehicle.vin
                    ),
                )
        except Exception as exception:
            self.log_exception_once(
                exception,
                "Unable to obtain the vehicle status report of {}".format(
                    self._vehicle.vin
                ),
            ) 

def update_vehicle_position(self):
        if not self.support_position:
            return

        try:
            resp = await self._audi_service.get_stored_position(self._vehicle.vin)
            if resp.get("findCarResponse") is not None:
                position = resp["findCarResponse"]

            if (
                position.get("Position") is not None
                and position["Position"].get("carCoordinate") is not None
            ):
                self._vehicle.state["position"] = {
                    "latitude": get_attr(position, "Position.carCoordinate.latitude")
                    / 1000000,
                    "longitude": get_attr(position, "Position.carCoordinate.longitude")
                    / 1000000,
                    "timestamp": get_attr(position, "Position.timestampCarSentUTC"),
                    "parktime": position.get("parkingTimeUTC"),
                }

        except TimeoutError:
            raise
        except ClientResponseError as resp_exception:
            if resp_exception.status == 403 or resp_exception.status == 502:
                self.support_position = False
            else:
                self.log_exception_once(
                    resp_exception,
                    "Unable to update the vehicle position of {}".format(
                        self._vehicle.vin
                    ),
                )
        except Exception as exception:
            self.log_exception_once(
                exception,
                "Unable to update the vehicle position of {}".format(self._vehicle.vin),
            ) 

def update_vehicle_climater(self):
        if not self.support_climater:
            return

        try:
            result = await self._audi_service.get_climater(self._vehicle.vin)
            if result:
                self._vehicle.state["climatisationState"] = get_attr(
                    result,
                    "climater.status.climatisationStatusData.climatisationState.content",
                )

        except TimeoutError:
            raise
        except ClientResponseError as resp_exception:
            if resp_exception.status == 403 or resp_exception.status == 502:
                self.support_climater = False
            else:
                self.log_exception_once(
                    resp_exception,
                    "Unable to obtain the vehicle climatisation state for {}".format(
                        self._vehicle.vin
                    ),
                )
        except Exception as exception:
            self.log_exception_once(
                exception,
                "Unable to obtain the vehicle climatisation state for {}".format(
                    self._vehicle.vin
                ),
            ) 

def fetchMetadata(cls, session, url, path):
        async with session.head(url, allow_redirects=True) as response:
            try:
                response.raise_for_status()
            except aiohttp.ClientResponseError as e:
                # I don't like returning Exceptions, but I can't find a better way to pass a single error in an async loop
                return (None, e)
            size = int(response.headers.get("content-length", 0))
            lastModified = parsedateOrNone(response.headers.get("last-modified", None))
        return (cls(url, size, lastModified, path), None) 

def fetch(self, uri):
        sleep_times = self.sleeps(uri)
        self._exns = set()
        self._state = state.green
        for s in sleep_times:
            headers = {"Accept": "application/json"}
            params = {"embed": "body"}
            try:
                async with self.session.get(uri, params=params, headers=headers) as response:
                    # 200 OK? Return the Page
                    if(response.status == 200):
                        self._log.debug("Fetched %s", uri)
                        js = await response.json()
                        return Page(js, self._log)
            # Wonky URI? Raise to caller
            except ValueError as e:
                raise UrlError(e)
            # Error from the HTTP response?
            except aiohttp.ClientResponseError as e:
                self.log(e, uri)
                # For a 404, raise HttpNotFound
                if e.code == 404:
                    raise HttpNotFoundError(uri, 404)
                # For a client error other than a timeout, raise HttpClientError
                # Timeouts should log and sleep
                if e.code < 500 and e.code != 408:
                    raise HttpClientError(uri, e.code)
            # Other connection errors and malformed payloads just log and sleep
            except (aiohttp.ClientError) as e:
                self.log(e, uri)
            # Http timeout? Log and sleep
            except asyncio.TimeoutError as e:
                self.log(e, uri)

            await self.sleep(s) 

def read_obj(
        *,
        resource: resources.Resource,
        namespace: Optional[str] = None,
        name: Optional[str] = None,
        default: Union[_T, _UNSET] = _UNSET.token,
        context: Optional[auth.APIContext] = None,  # injected by the decorator
) -> Union[bodies.RawBody, _T]:
    if context is None:
        raise RuntimeError("API instance is not injected by the decorator.")

    is_namespaced = await discovery.is_namespaced(resource=resource, context=context)
    namespace = namespace if is_namespaced else None

    try:
        response = await context.session.get(
            url=resource.get_url(server=context.server, namespace=namespace, name=name),
        )
        response.raise_for_status()
        respdata = await response.json()
        return cast(bodies.RawBody, respdata)

    except aiohttp.ClientResponseError as e:
        if e.status in [403, 404] and not isinstance(default, _UNSET):
            return default
        raise 

def reauthenticated_request(fn: _F) -> _F:
    """
    A client-specific decorator to re-authenticate a one-time request.

    If a wrapped function fails on the authentication, this will be reported
    back to the credentials container, which will trigger the re-authentication
    activity. Meanwhile, the request-performing function will be awaiting
    for the new credentials, and re-executed once they are available.
    """
    @functools.wraps(fn)
    async def wrapper(*args: Any, **kwargs: Any) -> Any:

        # If a context is explicitly passed, make it a simple call without re-auth.
        # Exceptions are escalated to a caller, which is probably wrapped itself.
        if 'context' in kwargs:
            return await fn(*args, **kwargs)

        # Otherwise, attempt the execution with the vault credentials and re-authenticate on 401s.
        vault: credentials.Vault = vault_var.get()
        async for key, info, context in vault.extended(APIContext, 'contexts'):
            try:
                return await fn(*args, **kwargs, context=context)
            except aiohttp.ClientResponseError as e:
                if e.status == 401:
                    await vault.invalidate(key, exc=e)
                else:
                    raise
        raise credentials.LoginError("Ran out of connection credentials.")
    return cast(_F, wrapper) 

def reauthenticated_stream(fn: _F) -> _F:
    """
    A client-specific decorator to re-authenticate an iterator-generator.

    If a wrapped function fails on the authentication, this will be reported
    back to the credentials source, which will trigger the re-authentication
    activity. Meanwhile, the function will be awaiting for the new credentials,
    and re-executed once they are available.
    """
    @functools.wraps(fn)
    async def wrapper(*args: Any, **kwargs: Any) -> Any:

        # If a context is explicitly passed, make it a simple call without re-auth.
        # Exceptions are escalated to a caller, which is probably wrapped itself.
        if 'context' in kwargs:
            async for item in fn(*args, **kwargs):
                yield item
            return

        # Otherwise, attempt the execution with the vault credentials and re-authenticate on 401s.
        vault: credentials.Vault = vault_var.get()
        async for key, info, context in vault.extended(APIContext, 'contexts'):
            try:
                async for item in fn(*args, **kwargs, context=context):
                    yield item
                return
            except aiohttp.ClientResponseError as e:
                if e.status == 401:
                    await vault.invalidate(key, exc=e)
                else:
                    raise
        raise credentials.LoginError("Ran out of connection credentials.")
    return cast(_F, wrapper) 

def test_infinite_watch_never_exits_normally(
        settings, resource, stream, namespace, aresponses):

    error = aresponses.Response(status=555, reason='stop-infinite-cycle')
    stream.feed(
        STREAM_WITH_ERROR_410GONE,          # watching restarted
        STREAM_WITH_UNKNOWN_EVENT,          # event ignored
        error,                              # to finally exit it somehow
        namespace=namespace,
    )
    stream.close(namespace=namespace)

    events = []
    with pytest.raises(aiohttp.ClientResponseError) as e:
        async for event in infinite_watch(settings=settings,
                                          resource=resource,
                                          namespace=namespace):
            events.append(event)

    assert e.value.status == 555
    assert e.value.message == 'stop-infinite-cycle'

    assert len(events) == 3
    assert events[0]['object']['spec'] == 'a'
    assert events[1]['object']['spec'] == 'a'
    assert events[2]['object']['spec'] == 'b'


# See: See: https://github.com/zalando-incubator/kopf/issues/275 

def handle_exception():
    """
    Context manager translating network related exceptions
    to custom :mod:`~galaxy.api.errors`.
    """
    try:
        yield
    except asyncio.TimeoutError:
        raise BackendTimeout()
    except aiohttp.ServerDisconnectedError:
        raise BackendNotAvailable()
    except aiohttp.ClientConnectionError:
        raise NetworkError()
    except aiohttp.ContentTypeError as error:
        raise UnknownBackendResponse(error.message)
    except aiohttp.ClientResponseError as error:
        if error.status == HTTPStatus.UNAUTHORIZED:
            raise AuthenticationRequired(error.message)
        if error.status == HTTPStatus.FORBIDDEN:
            raise AccessDenied(error.message)
        if error.status == HTTPStatus.SERVICE_UNAVAILABLE:
            raise BackendNotAvailable(error.message)
        if error.status == HTTPStatus.TOO_MANY_REQUESTS:
            raise TooManyRequests(error.message)
        if error.status >= 500:
            raise BackendError(error.message)
        if error.status >= 400:
            logger.warning(
                "Got status %d while performing %s request for %s",
                error.status, error.request_info.method, str(error.request_info.url)
            )
            raise UnknownError(error.message)
    except aiohttp.ClientError as e:
        logger.exception("Caught exception while performing request")
        raise UnknownError(repr(e)) 

def http_post(session, url, data, headers):
    def _format_error(e, text):
        return "{}. {}".format(e, text)

    backoff = INITIAL_BACKOFF
    retries = 0

    while retries < MAX_RETRIES:
        if retries > 0:
            print("Retrying in {} seconds".format(backoff))
            await asyncio.sleep(backoff)
            backoff *= BACKOFF_MULTIPLIER

        retries += 1

        try:
            resp = await session.post(url, data=data, headers=headers)
            resp.raise_for_status()
            return resp.status, resp.url
        except aiohttp.ClientResponseError as e:
            if e.status == 400:
                raise BadRequestException(_format_error(e, "Unexpected payload"))
            elif e.status == 403:
                raise BadRequestException(_format_error(e, "Review your license key"))
            elif e.status == 404:
                raise BadRequestException(
                    _format_error(e, "Review the region endpoint")
                )
            elif e.status == 429:
                print("There was a {} error. Reason: {}".format(e.status, e.message))
                # Now retry the request
                continue
            elif 400 <= e.status < 500:
                raise BadRequestException(e)

    raise MaxRetriesException()