Python asyncio.IncompleteReadError() Examples

def __read_message_frame(self):
        length_bytes = await self.__reader.read(n=4)
        if len(length_bytes) == 0:
            raise asyncio.IncompleteReadError(length_bytes, 4)

        length = Util.int_from_bytes(length_bytes)
        operation = Util.int_from_bytes(await self.__reader.read(n=1))

        # Read from the socket until we have read the full packet
        payload = bytearray()
        read_bytes = 1
        while read_bytes < length:
            next_payload = await self.__reader.read(n=length - read_bytes)
            if len(next_payload) == 0:
                raise asyncio.IncompleteReadError(next_payload, length - read_bytes)
            payload.extend(next_payload)
            read_bytes += len(next_payload)

        try:
            op = Operation.from_dict(operation)
        except ValueError:
            self.logger.error("Found unknown operation %d", operation)
            op = Operation.Unknown

        return MessageFrame(operation=op, payload=bytes(payload)) 

def connect_reader_to_writer(reader, writer):
    BUF_SIZE = 8192
    try:
        while True:
            data = await reader.read(BUF_SIZE)

            if not data:
                if not writer.transport.is_closing():
                    writer.write_eof()
                    await writer.drain()
                return

            writer.write(data)
            await writer.drain()
    except (OSError, asyncio.IncompleteReadError) as e:
        pass 

def run(self):
        await self._client_connected()
        while not self.__writer.is_closing():
            try:
                data = await self.__reader.readuntil(
                    separator=Spheniscidae.Delimiter)
                if data:
                    await self.__data_received(data)
                else:
                    self.__writer.close()
                await self.__writer.drain()
            except IncompleteReadError:
                self.__writer.close()
            except CancelledError:
                self.__writer.close()
            except ConnectionResetError:
                self.__writer.close()
            except LimitOverrunError:
                self.__writer.close()
            except BaseException as e:
                self.logger.exception(e.__traceback__)

        await self._client_disconnected() 

def test_readexactly_eof(self):
        # Read exact number of bytes (eof).
        stream = asyncio.StreamReader(loop=self.loop)
        n = 2 * len(self.DATA)
        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)

        def cb():
            stream.feed_data(self.DATA)
            stream.feed_eof()
        self.loop.call_soon(cb)

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(read_task)
        self.assertEqual(cm.exception.partial, self.DATA)
        self.assertEqual(cm.exception.expected, n)
        self.assertEqual(str(cm.exception),
                         '18 bytes read on a total of 36 expected bytes')
        self.assertEqual(b'', stream._buffer) 

def _handle_packets(self):
        data = True
        while data:
            try:
                data = await self._reader.readuntil(b"}}")
                if not data:
                    break
                message = data.decode("ascii")
                await self.parse_api_messages(message)

            except (asyncio.IncompleteReadError, TimeoutError, ConnectionResetError, OSError) as e:
                _LOGGER.error(
                    "pyIntesisHome lost connection to the %s server. Exception: %s", self._device_type, e
                )
                break

        self._connected = False
        self._connecting = False
        self._authToken = None
        self._reader = None
        self._writer = None
        self._sendQueueTask.cancel()
        await self._send_update_callback()
        return 

def test_readexactly_eof(self):
        # Read exact number of bytes (eof).
        stream = asyncio.StreamReader(loop=self.loop)
        n = 2 * len(self.DATA)
        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)

        def cb():
            stream.feed_data(self.DATA)
            stream.feed_eof()
        self.loop.call_soon(cb)

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(read_task)
        self.assertEqual(cm.exception.partial, self.DATA)
        self.assertEqual(cm.exception.expected, n)
        self.assertEqual(str(cm.exception),
                         '18 bytes read on a total of 36 expected bytes')
        self.assertEqual(b'', stream._buffer) 

def connect(self):
        loop = asyncio.get_event_loop()
        while True:
            try:
                self.reader, self.writer = await asyncio.open_connection(
                    self.server.config['masterserver_ip'],
                    self.server.config['masterserver_port'],
                    loop=loop)
                await self.handle_connection()
            except (ConnectionRefusedError, TimeoutError,
                    ConnectionResetError, asyncio.IncompleteReadError):
                logger.debug('Connection error occurred.')
                self.writer = None
                self.reader = None
            finally:
                logger.debug('Retrying MS connection in 30 seconds.')
                await asyncio.sleep(30) 

def tg_connect_reader_to_writer(rd, wr, user, rd_buf_size, is_upstream):
    try:
        while True:
            data = await rd.read(rd_buf_size)
            if isinstance(data, tuple):
                data, extra = data
            else:
                extra = {}

            if not data:
                wr.write_eof()
                await wr.drain()
                return
            else:
                if is_upstream:
                    update_user_stats(user, octets_from_client=len(data), msgs_from_client=1)
                else:
                    update_user_stats(user, octets_to_client=len(data), msgs_to_client=1)

                wr.write(data, extra)
                await wr.drain()
    except (OSError, asyncio.IncompleteReadError) as e:
        # print_err(e)
        pass 

def fetch(self):
        while True:

            try:
                hdrlen = constants.STREAM_HEADER_SIZE_BYTES
                header = yield from self._response.content.readexactly(hdrlen)

                _, length = struct.unpack(">BxxxL", header)
                if not length:
                    continue

                data = yield from self._response.content.readexactly(length)

            except (
                aiohttp.ClientConnectionError,
                aiohttp.ServerDisconnectedError,
                asyncio.IncompleteReadError,
            ):
                break
            return data 

def test_readexactly_eof(self):
        # Read exact number of bytes (eof).
        stream = asyncio.StreamReader(loop=self.loop)
        n = 2 * len(self.DATA)
        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)

        def cb():
            stream.feed_data(self.DATA)
            stream.feed_eof()
        self.loop.call_soon(cb)

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(read_task)
        self.assertEqual(cm.exception.partial, self.DATA)
        self.assertEqual(cm.exception.expected, n)
        self.assertEqual(str(cm.exception),
                         '18 bytes read on a total of 36 expected bytes')
        self.assertEqual(b'', stream._buffer) 

def test_readexactly_eof(self):
        # Read exact number of bytes (eof).
        stream = asyncio.StreamReader(loop=self.loop)
        n = 2 * len(self.DATA)
        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)

        def cb():
            stream.feed_data(self.DATA)
            stream.feed_eof()
        self.loop.call_soon(cb)

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(read_task)
        self.assertEqual(cm.exception.partial, self.DATA)
        self.assertEqual(cm.exception.expected, n)
        self.assertEqual(str(cm.exception),
                         '18 bytes read on a total of 36 expected bytes')
        self.assertEqual(b'', stream._buffer) 

def test_readexactly_eof(self):
        # Read exact number of bytes (eof).
        stream = asyncio.StreamReader(loop=self.loop)
        n = 2 * len(self.DATA)
        read_task = asyncio.Task(stream.readexactly(n), loop=self.loop)

        def cb():
            stream.feed_data(self.DATA)
            stream.feed_eof()
        self.loop.call_soon(cb)

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(read_task)
        self.assertEqual(cm.exception.partial, self.DATA)
        self.assertEqual(cm.exception.expected, n)
        self.assertEqual(str(cm.exception),
                         '18 bytes read on a total of 36 expected bytes')
        self.assertEqual(b'', stream._buffer) 

def __call__(self, reader, writer):
        if self.framed:
            reader = TFramedTransport(reader)
            writer = TFramedTransport(writer)

        iproto = self.protocol_cls(reader)
        oproto = self.protocol_cls(writer)
        while not reader.at_eof():
            try:
                with async_timeout.timeout(self.timeout):
                    await self.processor.process(iproto, oproto)
            except ConnectionError:
                logger.debug("client has closed the connection")
                writer.close()
            except asyncio.TimeoutError:
                logger.debug("timeout when processing the client request")
                writer.close()
            except asyncio.IncompleteReadError:
                logger.debug("client has closed the connection")
                writer.close()
            except Exception:
                # app exception
                logger.exception("unhandled app exception")
                writer.close()
        writer.close() 

def test_streaming_with_timeout__expects_timeout(self):

        app = Vibora()

        async def stream():
            for _ in range(0, 100):
                await asyncio.sleep(2)
                yield b'1'

        @app.route('/')
        async def home():
            return StreamingResponse(stream, complete_timeout=1)

        async with app.test_client() as client:
            try:
                await client.get('/', timeout=3)
                self.fail('Vibora should have closed the connection because a streaming timeout is not recoverable.')
            except asyncio.IncompleteReadError:
                pass
            except futures.TimeoutError:
                pass 

def _handle(self):
		try:
			await self._send_welcome_message()
			await asyncio.gather(self._handle_commands(), self._handle_events())
		except ConnectionResetError:
			pass
		except asyncio.IncompleteReadError:
			pass 

def read_bytes(
            self,
            n: Optional[int] = None) -> Optional[bytes]:
        # 不支持的原因是，tcp 会切数据，自己拼装过于复杂
        if n is None or n <= 0:
            return None
        try:
            bytes_data = await asyncio.wait_for(
                self._reader.readexactly(n), timeout=self._receive_timeout)
        except (OSError, asyncio.TimeoutError, asyncio.IncompleteReadError):
            return None
        except asyncio.CancelledError:
            # print('asyncio.CancelledError', 'read_bytes')
            return None
        return bytes_data 

def readexactly(self, size):
        data = self.read(size)
        if len(data) < size:
            raise asyncio.IncompleteReadError(data, size)
        else:
            return data


# TODO: Document properly 

def read_or_raise(reader, n=-1):
    """
    Read a given byte number from Stream. NoDataException is raised if read gives no data
    :param reader: reader adapter
    :param n: number of bytes to read
    :return: bytes read
    """
    try:
        data = yield from reader.read(n)
    except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError):
        data = None
    if not data:
        raise NoDataException("No more data")
    return data 

def test_readuntil_eof(self):
        stream = asyncio.StreamReader(loop=self.loop)
        stream.feed_data(b'some dataAA')
        stream.feed_eof()

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))
        self.assertEqual(cm.exception.partial, b'some dataAA')
        self.assertIsNone(cm.exception.expected)
        self.assertEqual(b'', stream._buffer) 

def connection_handler(execute_rpc: Callable[[Any], Any],
                             reader: asyncio.StreamReader,
                             writer: asyncio.StreamWriter) -> None:
    """
    Catch fatal errors, log them, and close the connection
    """

    try:
        await connection_loop(execute_rpc, reader, writer),
    except (ConnectionResetError, asyncio.IncompleteReadError):
        logger.debug("Client closed connection")
    except Exception:
        logger.exception("Unrecognized exception while handling requests")
    finally:
        writer.close() 

def read(self, n: int) -> bytes:
        self.logger.debug2("Waiting for %s bytes from %s", n, self.remote)
        try:
            return await asyncio.wait_for(self._reader.readexactly(n), timeout=CONN_IDLE_TIMEOUT)
        except (asyncio.IncompleteReadError, ConnectionResetError, BrokenPipeError) as err:
            raise PeerConnectionLost(f"Lost connection to {self.remote}") from err 

def test_simple_streaming_with_chunk_timeout(self):

        app = Vibora(server_limits=ServerLimits(write_buffer=1))

        async def stream():
            for _ in range(0, 5):
                await asyncio.sleep(0)
                yield b'1' * 1024 * 1024 * 100

        @app.route('/')
        async def home():
            return StreamingResponse(stream, chunk_timeout=3, complete_timeout=999)

        async with app.test_client() as client:
            response = await client.get('/', stream=True)
            try:
                first = True
                chunk_size = 1 * 1024 * 1024
                async for chunk in response.stream(chunk_size=chunk_size):
                    if first:
                        await asyncio.sleep(5)
                        first = False
                    self.assertTrue(len(chunk) <= chunk_size)
                self.fail('Vibora should have closed the connection because of a chunk timeout.')
            except asyncio.IncompleteReadError:
                pass 

def readuntil(self, delim):
        if not self.current_event:
            if not self.events:
                raise asyncio.IncompleteReadError(b'', delim)
            yield from asyncio.sleep(self.delay)
            self.current_event = self.events.pop(0)
        data, rest = self.current_event.split(delim, 1)
        self.current_event = rest
        return data + delim 

def stream_connected(self, reader, writer, listener_name):
        try:
            yield from super().stream_connected(reader, writer, listener_name)
        except IncompleteReadError as e:
            log.warning("Broker says: %s" % e)
            # It's annoying. https://github.com/beerfactory/hbmqtt/issues/119 

def test_readuntil_eof(self):
        stream = asyncio.StreamReader(loop=self.loop)
        stream.feed_data(b'some dataAA')
        stream.feed_eof()

        with self.assertRaises(asyncio.IncompleteReadError) as cm:
            self.loop.run_until_complete(stream.readuntil(b'AAA'))
        self.assertEqual(cm.exception.partial, b'some dataAA')
        self.assertIsNone(cm.exception.expected)
        self.assertEqual(b'', stream._buffer) 

def test_IncompleteReadError_pickleable(self):
        e = asyncio.IncompleteReadError(b'abc', 10)
        for proto in range(pickle.HIGHEST_PROTOCOL + 1):
            with self.subTest(pickle_protocol=proto):
                e2 = pickle.loads(pickle.dumps(e, protocol=proto))
                self.assertEqual(str(e), str(e2))
                self.assertEqual(e.partial, e2.partial)
                self.assertEqual(e.expected, e2.expected) 

def _read_bytes(self, num_bytes):
        try:
            data = await self._reader.readexactly(num_bytes)
        except asyncio.IncompleteReadError as e:
            msg = "Lost connection to MySQL server during query"
            raise OperationalError(2013, msg) from e
        except (IOError, OSError) as e:
            msg = "Lost connection to MySQL server during query (%s)" % (e,)
            raise OperationalError(2013, msg) from e
        return data 

def _recv_loop(self):
        """
        This loop is constantly putting items on the queue as they're read.
        """
        while self._connected:
            try:
                data = await self._recv()
            except asyncio.CancelledError:
                break
            except Exception as e:
                if isinstance(e, (IOError, asyncio.IncompleteReadError)):
                    msg = 'The server closed the connection'
                    self._log.info(msg)
                elif isinstance(e, InvalidChecksumError):
                    msg = 'The server response had an invalid checksum'
                    self._log.info(msg)
                else:
                    msg = 'Unexpected exception in the receive loop'
                    self._log.exception(msg)

                await self.disconnect()

                # Add a sentinel value to unstuck recv
                if self._recv_queue.empty():
                    self._recv_queue.put_nowait(None)

                break

            try:
                await self._recv_queue.put(data)
            except asyncio.CancelledError:
                break 

def _recv(self, length=0, head_only=False):
        resp = b''
        if length:
            try:
                resp = await self.reader.readexactly(length)
            except asyncio.IncompleteReadError as e:
                resp = e.partial
        else:
            body_size, body_recv, chunked = 0, 0, None
            while not self.reader.at_eof():
                line = await self.reader.readline()
                resp += line
                if body_size:
                    body_recv += len(line)
                    if body_recv >= body_size:
                        break
                elif chunked and line == b'0\r\n':
                    break
                elif not body_size and line == b'\r\n':
                    if head_only:
                        break
                    headers = parse_headers(resp)
                    body_size = int(headers.get('Content-Length', 0))
                    if not body_size:
                        chunked = headers.get('Transfer-Encoding') == 'chunked'
        return resp 

def read_packet(self, reader):
        while True:
            line = await reader.readline()
            if not line or line[-1] != b'\n':
                raise asyncio.IncompleteReadError(line, None)

            if line.lower().startswith(b'content-length: '):
                await reader.readexactly(2)
                length = int(line[16:-2])
                return await reader.readexactly(length)