Python aiohttp.ClientPayloadError() Examples

def init(self, url: str, key: str):
        """

        :param url:
        :type url:
        :param key:
        :type key:
        :return:
        :rtype:
        """
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(url + key) as data:
                    world_state_item = await data.read()
                    self.raw = json.loads(world_state_item)
        except aiohttp.ClientPayloadError:
            pass
        return self 

def get_plains_data(db):
    """

    :param db:
    :type db:
    :return:
    :rtype:
    """
    world_state = 'http://content.warframe.com/dynamic/worldState.php'
    try:
        async with aiohttp.ClientSession() as session:
            async with session.get(world_state) as data:
                data = await data.read()
                data = json.loads(data)
        synd_missions = data['SyndicateMissions']
        poe_data = None
        for synd_mission in synd_missions:
            if synd_mission['Tag'] == 'CetusSyndicate':
                poe_data = synd_mission
    except aiohttp.ClientPayloadError:
        poe_data = None
    plains_out = None
    triggers = []
    if poe_data:
        now = arrow.utcnow().float_timestamp
        sta = int(poe_data['Activation']['$date']['$numberLong']) / 1000
        nox = (int(poe_data['Activation']['$date']['$numberLong']) + (1000 * 60 * 100)) / 1000
        end = int(poe_data['Expiry']['$date']['$numberLong']) / 1000
        is_day = now < nox
        oid = poe_data.get('_id', {}).get('$oid') if is_day else f"night_{poe_data.get('_id', {}).get('$oid')}"
        triggers = ['day'] if is_day else ['night']
        db_check = await db[db.db_nam].WarframeCache.find_one({'event_id': oid})
        if not db_check:
            plains_out = {'sta': sta, 'end': end, 'nox': nox, 'now': now, 'day': is_day}
            await db[db.db_nam].WarframeCache.insert_one({'event_id': oid, 'created': now})
    return plains_out, triggers 

def read(self, url, *args, use_proxy=False, **kwargs) -> FeedResponse:
        headers = content = None
        try:
            if use_proxy:
                headers, content, url, status = await self._read_by_proxy(url, *args, **kwargs)
            else:
                headers, content, url, status = await self._read(url, *args, **kwargs)
        except (socket.gaierror, aiodns.error.DNSError):
            status = FeedResponseStatus.DNS_ERROR.value
        except (socket.timeout, TimeoutError, aiohttp.ServerTimeoutError,
                asyncio.TimeoutError, concurrent.futures.TimeoutError):
            status = FeedResponseStatus.CONNECTION_TIMEOUT.value
        except (ssl.SSLError, ssl.CertificateError,
                aiohttp.ServerFingerprintMismatch,
                aiohttp.ClientSSLError,
                aiohttp.ClientConnectorSSLError,
                aiohttp.ClientConnectorCertificateError):
            status = FeedResponseStatus.SSL_ERROR.value
        except (aiohttp.ClientProxyConnectionError,
                aiohttp.ClientHttpProxyError):
            status = FeedResponseStatus.PROXY_ERROR.value
        except (ConnectionError,
                aiohttp.ServerDisconnectedError,
                aiohttp.ServerConnectionError,
                aiohttp.ClientConnectionError,
                aiohttp.ClientConnectorError):
            status = FeedResponseStatus.CONNECTION_RESET.value
        except (aiohttp.WSServerHandshakeError, aiohttp.ClientOSError):
            status = FeedResponseStatus.CONNECTION_ERROR.value
        except aiohttp.ClientPayloadError:
            status = FeedResponseStatus.CHUNKED_ENCODING_ERROR.value
        except UnicodeDecodeError:
            status = FeedResponseStatus.CONTENT_DECODING_ERROR.value
        except FeedReaderError as ex:
            status = ex.status
            LOG.warning(type(ex).__name__ + " url=%s %s", url, ex)
        except (aiohttp.ClientResponseError, aiohttp.ContentTypeError) as ex:
            status = ex.status
        except (aiohttp.ClientError, aiohttp.InvalidURL):
            status = FeedResponseStatus.UNKNOWN_ERROR.value
        builder = FeedResponseBuilder(use_proxy=use_proxy)
        builder.url(url)
        builder.status(status)
        builder.content(content)
        builder.headers(headers)
        return builder.build() 

def spooled_data_from_url(
    url: str,
    headers: Dict[str, str] = {},
    timeout: aiohttp.ClientTimeout = None,
    *,
    ssl: Optional[ssl.SSLContext] = None,
):
    """
    Download `url` to a tempfile and yield `(bytesio, headers, charset)`.

    `bytesio` is backed by a temporary file: the file at path `bytesio.name`
    will exist within this context.

    Raise aiohttp.ClientError on generic error. Subclasses of note:
    * aiohttp.InvalidURL on invalid URL
    * aiohttp.ClientResponseError when HTTP status is not 200
    * aiohttp.ClientPayloadError when server closes connection prematurely
    * aiohttp.ClientConnectionError (OSError) when connection fails

    Raise asyncio.TimeoutError when `timeout` seconds have expired.
    """

    # aiohttp internally performs URL canonization before sending
    # request. DISABLE THIS: it breaks oauth and user's expectations.
    #
    # https://github.com/aio-libs/aiohttp/issues/3424
    url = yarl.URL(url, encoded=True)  # prevent magic
    if url.scheme not in ("http", "https"):
        raise aiohttp.InvalidURL("URL must start with http:// or https://")

    with tempfile_context(prefix="loadurl") as spool_path:
        async with aiohttp.ClientSession() as session:
            # raise aiohttp.ClientError, asyncio.TimeoutError
            async with session.get(
                url, headers=headers, timeout=timeout, ssl=ssl
            ) as response:
                # raise aiohttp.ClientResponseError
                response.raise_for_status()
                headers = response.headers
                charset = response.charset

                with spool_path.open("wb") as spool:
                    # raise aiohttp.ClientPayloadError
                    async for blob in response.content.iter_chunked(_ChunkSize):
                        spool.write(blob)

        yield spool_path.open("rb"), headers, charset


# dependency-injection, so unit tests can mock our functions 

def watch_objs(
        *,
        settings: configuration.OperatorSettings,
        resource: resources.Resource,
        namespace: Optional[str] = None,
        timeout: Optional[float] = None,
        since: Optional[str] = None,
        context: Optional[auth.APIContext] = None,  # injected by the decorator
        freeze_waiter: asyncio_Future,
) -> AsyncIterator[bodies.RawInput]:
    """
    Watch objects of a specific resource type.

    The cluster-scoped call is used in two cases:

    * The resource itself is cluster-scoped, and namespacing makes not sense.
    * The operator serves all namespaces for the namespaced custom resource.

    Otherwise, the namespace-scoped call is used:

    * The resource is namespace-scoped AND operator is namespaced-restricted.
    """
    if context is None:
        raise RuntimeError("API instance is not injected by the decorator.")

    is_namespaced = await discovery.is_namespaced(resource=resource, context=context)
    namespace = namespace if is_namespaced else None

    params: Dict[str, str] = {}
    params['watch'] = 'true'
    if since is not None:
        params['resourceVersion'] = since
    if timeout is not None:
        params['timeoutSeconds'] = str(timeout)

    # Talk to the API and initiate a streaming response.
    response = await context.session.get(
        url=resource.get_url(server=context.server, namespace=namespace, params=params),
        timeout=aiohttp.ClientTimeout(
            total=settings.watching.client_timeout,
            sock_connect=settings.watching.connect_timeout,
        ),
    )
    response.raise_for_status()

    # Stream the parsed events from the response until it is closed server-side,
    # or until it is closed client-side by the freeze-waiting future's callbacks.
    response_close_callback = lambda _: response.close()
    freeze_waiter.add_done_callback(response_close_callback)
    try:
        async with response:
            async for line in _iter_jsonlines(response.content):
                raw_input = cast(bodies.RawInput, json.loads(line.decode("utf-8")))
                yield raw_input
    except (aiohttp.ClientConnectionError, aiohttp.ClientPayloadError):
        pass
    finally:
        freeze_waiter.remove_done_callback(response_close_callback) 

def _longpoll_request(self):
        """Open a long-polling request and receive arrays.

        This method uses keep-alive to make re-opening the request faster, but
        the remote server will set the "Connection: close" header once an hour.

        Raises hangups.NetworkError or ChannelSessionError.
        """
        params = {
            'VER': 8,  # channel protocol version
            'gsessionid': self._gsessionid_param,
            'RID': 'rpc',  # request identifier
            't': 1,  # trial
            'SID': self._sid_param,  # session ID
            'CI': 0,  # 0 if streaming/chunked requests should be used
            'ctype': 'hangouts',  # client type
            'TYPE': 'xmlhttp',  # type of request
        }
        logger.info('Opening new long-polling request')
        try:
            async with self._session.fetch_raw('GET', CHANNEL_URL,
                                               params=params) as res:

                if res.status != 200:
                    if res.status == 400 and res.reason == 'Unknown SID':
                        raise ChannelSessionError('SID became invalid')
                    raise exceptions.NetworkError(
                        'Request return unexpected status: {}: {}'.format(
                            res.status, res.reason))

                while True:
                    async with async_timeout.timeout(PUSH_TIMEOUT):
                        chunk = await res.content.read(MAX_READ_BYTES)
                    if not chunk:
                        break

                    await self._on_push_data(chunk)

        except asyncio.TimeoutError:
            raise exceptions.NetworkError('Request timed out')
        except aiohttp.ServerDisconnectedError as err:
            raise exceptions.NetworkError(
                'Server disconnected error: %s' % err)
        except aiohttp.ClientPayloadError:
            raise ChannelSessionError('SID is about to expire')
        except aiohttp.ClientError as err:
            raise exceptions.NetworkError('Request connection error: %s' % err)