Python zmq.asyncio() Examples

def can_connect(self, server, client):
        """Check if client can connect to server using tcp transport"""
        @asyncio.coroutine
        def go():
            result = False
            iface = 'tcp://127.0.0.1'
            port = server.bind_to_random_port(iface)
            client.connect("%s:%i" % (iface, port))
            msg = [b"Hello World"]
            yield from server.send_multipart(msg)
            if (yield from client.poll(1000)):
                rcvd_msg = yield from client.recv_multipart()
                self.assertEqual(rcvd_msg, msg)
                result = True
            return result
        return self.loop.run_until_complete(go()) 

def test_poll_base_socket(self):
        @asyncio.coroutine
        def test():
            ctx = zmq.Context()
            url = 'inproc://test'
            a = ctx.socket(zmq.PUSH)
            b = ctx.socket(zmq.PULL)
            self.sockets.extend([a, b])
            a.bind(url)
            b.connect(url)

            poller = zaio.Poller()
            poller.register(b, zmq.POLLIN)

            f = poller.poll(timeout=1000)
            assert not f.done()
            a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, [(b, zmq.POLLIN)])
            recvd = b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def _do_heartbeat(self):
        while True:
            try:
                if self._socket.getsockopt(zmq.TYPE) == zmq.ROUTER:
                    yield from self._do_router_heartbeat()
                elif self._socket.getsockopt(zmq.TYPE) == zmq.DEALER:
                    yield from self._do_dealer_heartbeat()
                yield from asyncio.sleep(self._heartbeat_interval,
                                         loop=self._event_loop)
            except CancelledError:  # pylint: disable=try-except-raise
                # The concurrent.futures.CancelledError is caught by asyncio
                # when the Task associated with the coroutine is cancelled.
                # The raise is required to stop this component.
                raise
            except Exception as e:  # pylint: disable=broad-except
                LOGGER.exception(
                    "An error occurred while sending heartbeat: %s", e) 

def __frame_generator(self):
        """
        Returns a default frame-generator for NetGear's Server Handler. 
        """
        # start stream
        self.__stream.start()
        # loop over stream until its terminated
        while not self.__terminate:
            # read frames
            frame = self.__stream.read()
            # break if NoneType
            if frame is None:
                break
            # yield frame
            yield frame
            # sleep for sometime
            await asyncio.sleep(0.00001) 

def launch(self):
        """
        Launches an asynchronous loop executors for respective task.
        """
        # check if receive mode enabled
        if self.__receive_mode:
            if self.__logging:
                logger.debug("Launching NetGear asynchronous generator!")
            # run loop executor for Receiver asynchronous generator
            self.loop.run_in_executor(None, self.recv_generator)
            # return instance
            return self
        else:
            # Otherwise launch Server handler
            if self.__logging:
                logger.debug("Creating NetGear asynchronous server handler!")
            # create task for Server Handler
            self.task = asyncio.ensure_future(self.__server_handler(), loop=self.loop)
            # return instance
            return self 

def test_poll(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.poll(timeout=0)
            yield from asyncio.sleep(0)
            self.assertEqual(f.result(), 0)

            f = b.poll(timeout=1)
            assert not f.done()
            evt = yield from f

            self.assertEqual(evt, 0)

            f = b.poll(timeout=1000)
            assert not f.done()
            yield from a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, zmq.POLLIN)
            recvd = yield from b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def do_backoff(self, err_msg=" "):
        if self.num_retries == self.max_retries:
            LOGGER.warning("Failed sending message to the Validator. No more "
                           "retries left. Backoff terminated: %s",
                           err_msg)
            raise self.error

        self.num_retries += 1
        LOGGER.warning("Sleeping for %s ms after failed attempt %s of %s to "
                       "send message to the Validator: %s",
                       str(self.num_retries),
                       str(self.max_retries),
                       str(self.interval / 1000),
                       err_msg)

        await asyncio.sleep(self.interval / 1000)
        self.interval *= 2 

def shutdown(self):
        self._dispatcher.remove_send_message(self._connection)
        self._dispatcher.remove_send_last_message(self._connection)
        if self._event_loop is None:
            return
        if self._event_loop.is_closed():
            return

        if self._event_loop.is_running():
            if self._auth is not None:
                self._event_loop.call_soon_threadsafe(self._auth.stop)
        else:
            # event loop was never started, so the only Task that is running
            # is the Auth Task.
            self._event_loop.run_until_complete(self._stop_auth())

        asyncio.ensure_future(self._stop(), loop=self._event_loop) 

def test_custom_serialize_error(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            with pytest.raises(TypeError):
                yield from a.send_serialized(json, json.dumps)

            yield from a.send(b'not json')
            with pytest.raises(TypeError):
                recvd = yield from b.recv_serialized(json.loads)
        self.loop.run_until_complete(test()) 

def test_recv_json_cancelled(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield from asyncio.sleep(0)
            obj = dict(a=5)
            yield from a.send_json(obj)
            with pytest.raises(CancelledError):
                recvd = yield from f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield from b.poll(timeout=5)
            assert events
            yield from asyncio.sleep(0)
            # make sure cancelled recv didn't eat up event
            f = b.recv_json()
            recvd = yield from asyncio.wait_for(f, timeout=5)
            assert recvd == obj
        self.loop.run_until_complete(test()) 

def launch(self, websocket_port, message_max_size):
        async def receive_from_engine_worker_loop():
            await self.wait_for_start()
            while self.is_running():
                await self._receive_from_engine_worker_helper()

        async def heartbeat_loop():
            await self.wait_for_start()
            while self.is_running():
                await asyncio.sleep(self._timeout)
                await self._heartbeat_helper()

        asyncio.ensure_future(receive_from_engine_worker_loop())
        asyncio.ensure_future(heartbeat_loop())

        super().launch(websocket_port, message_max_size) 

def can_connect(self, server, client):
        """Check if client can connect to server using tcp transport"""
        @asyncio.coroutine
        def go():
            result = False
            iface = 'tcp://127.0.0.1'
            port = server.bind_to_random_port(iface)
            client.connect("%s:%i" % (iface, port))
            msg = [b"Hello World"]
            yield from server.send_multipart(msg)
            if (yield from client.poll(1000)):
                rcvd_msg = yield from client.recv_multipart()
                self.assertEqual(rcvd_msg, msg)
                result = True
            return result
        return self.loop.run_until_complete(go()) 

def test_poll_base_socket(self):
        @asyncio.coroutine
        def test():
            ctx = zmq.Context()
            url = 'inproc://test'
            a = ctx.socket(zmq.PUSH)
            b = ctx.socket(zmq.PULL)
            self.sockets.extend([a, b])
            a.bind(url)
            b.connect(url)

            poller = zaio.Poller()
            poller.register(b, zmq.POLLIN)

            f = poller.poll(timeout=1000)
            assert not f.done()
            a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, [(b, zmq.POLLIN)])
            recvd = b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def test_poll(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.poll(timeout=0)
            yield from asyncio.sleep(0)
            self.assertEqual(f.result(), 0)

            f = b.poll(timeout=1)
            assert not f.done()
            evt = yield from f

            self.assertEqual(evt, 0)

            f = b.poll(timeout=1000)
            assert not f.done()
            yield from a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, zmq.POLLIN)
            recvd = yield from b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def test_custom_serialize_error(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            with pytest.raises(TypeError):
                yield from a.send_serialized(json, json.dumps)

            yield from a.send(b'not json')
            with pytest.raises(TypeError):
                recvd = yield from b.recv_serialized(json.loads)
        self.loop.run_until_complete(test()) 

def stop(self):
        self.active = False
        self.log.warning("New block processing restricted")
        self.log.warning("Stopping node connector ...")
        try:
            for i in self.block_loader.worker:
                self.block_loader.worker[i].terminate()
        except:
            pass
        [task.cancel() for task in self.tasks]
        if self.tasks:
            await asyncio.wait(self.tasks)
        try:
            self.zeromq_task.cancel()
            await asyncio.wait([self.zeromq_task])
        except:
            pass
        if not self.active_block.done():
            self.log.warning("Waiting active block task ...")
            await self.active_block
        if self.rpc: await self.rpc.close()
        if self.zmqContext:
            self.zmqContext.destroy()
        self.log.warning('Node connector terminated') 

def test_recv_json_cancelled(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield from asyncio.sleep(0)
            obj = dict(a=5)
            yield from a.send_json(obj)
            with pytest.raises(CancelledError):
                recvd = yield from f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield from b.poll(timeout=5)
            assert events
            yield from asyncio.sleep(0)
            # make sure cancelled recv didn't eat up event
            f = b.recv_json()
            recvd = yield from asyncio.wait_for(f, timeout=5)
            assert recvd == obj
        self.loop.run_until_complete(test()) 

def run(self):
        self._logger.info("Backend started")
        self._agent_socket.bind(self._agent_addr)
        self._client_socket.bind(self._client_addr)
        self._loop.call_later(1, self._create_safe_task, self._do_ping())

        try:
            while True:
                socks = await self._poller.poll()
                socks = dict(socks)

                # New message from agent
                if self._agent_socket in socks:
                    agent_addr, message = await ZMQUtils.recv_with_addr(self._agent_socket)
                    await self.handle_agent_message(agent_addr, message)

                # New message from client
                if self._client_socket in socks:
                    client_addr, message = await ZMQUtils.recv_with_addr(self._client_socket)
                    await self.handle_client_message(client_addr, message)

        except asyncio.CancelledError:
            return
        except KeyboardInterrupt:
            return 

def start(self):
        """Starts receiving messages on the underlying socket and passes them
        to the message router.
        """
        self._is_running = True

        while self._is_running:
            try:
                zmq_msg = await self._socket.recv_multipart()

                message = Message()
                message.ParseFromString(zmq_msg[-1])

                await self._msg_router.route_msg(message)
            except DecodeError as e:
                LOGGER.warning('Unable to decode: %s', e)
            except zmq.ZMQError as e:
                LOGGER.warning('Unable to receive: %s', e)
                return
            except asyncio.CancelledError:
                self._is_running = False 

def __init__(self, endpoint: str, topic: bytes, context: zmq.asyncio.Context):
        self._endpoint = endpoint
        self._topic = topic
        self.context = context
        self.setup_socket() 

def shortDescription(self):
        """Rewrite doc strings from TestThreadAuthentication from
        'threaded' to 'asyncio'.
        """
        doc = self._testMethodDoc
        if doc:
            doc = doc.split("\n")[0].strip()
            if doc.startswith('threaded auth'):
                doc = doc.replace('threaded auth', 'asyncio auth')
        return doc 

def on_event(self, data: bytes, retries=0):
        try:
            await self.socket.send_multipart([self._topic, data])
        except Exception as e:
            if retries:
                raise
            Logger.zmq.warning('Error: %s' % e)
            await asyncio.sleep(0.5)
            try:
                self.socket.close()
            except:
                pass
            setattr(_SOCKETS, self._endpoint, None)
            await self.on_event(data, retries=retries+1) 

def start(self):
        """Start ZAP authentication"""
        super().start()
        self.__poller = zmq.asyncio.Poller()
        self.__poller.register(self.zap_socket, zmq.POLLIN)
        self.__task = asyncio.ensure_future(self.__handle_zap()) 

def setUp(self):
        if asyncio is None:
            raise SkipTest()
        self.loop = zaio.ZMQEventLoop()
        asyncio.set_event_loop(self.loop)
        super().setUp() 

def start(self):
        """Start ZAP authentication"""
        super().start()
        self.__poller = zmq.asyncio.Poller()
        self.__poller.register(self.zap_socket, zmq.POLLIN)
        self.__task = asyncio.ensure_future(self.__handle_zap()) 

def test_poll_raw(self):
        @asyncio.coroutine
        def test():
            p = zaio.Poller()
            # make a pipe
            r, w = os.pipe()
            r = os.fdopen(r, 'rb')
            w = os.fdopen(w, 'wb')

            # POLLOUT
            p.register(r, zmq.POLLIN)
            p.register(w, zmq.POLLOUT)
            evts = yield from p.poll(timeout=1)
            evts = dict(evts)
            assert r.fileno() not in evts
            assert w.fileno() in evts
            assert evts[w.fileno()] == zmq.POLLOUT

            # POLLIN
            p.unregister(w)
            w.write(b'x')
            w.flush()
            evts = yield from p.poll(timeout=1000)
            evts = dict(evts)
            assert r.fileno() in evts
            assert evts[r.fileno()] == zmq.POLLIN
            assert r.read(1) == b'x'
            r.close()
            w.close()

        loop = asyncio.get_event_loop()
        loop.run_until_complete(test()) 

def __init__(self, context=None, loop=None):
        super().__init__(context)
        self.loop = loop or asyncio.get_event_loop()
        self.__poller = None
        self.__task = None

    # TODO: Remove this commented method later
    # @asyncio.coroutine
    # def __handle_zap(self):
    #     while True:
    #         events = yield from self.__poller.poll()
    #         if self.zap_socket in dict(events):
    #             msg = yield from self.zap_socket.recv_multipart()
    #             self.handle_zap_message(msg) 

def retry_get_next_block(self):
        await asyncio.sleep(1)
        self.get_next_block_mutex = True
        self.loop.create_task(self.get_next_block()) 

def __init__(self, context=None, loop=None):
        super().__init__(context)
        self.loop = loop or asyncio.get_event_loop()
        self.__poller = None
        self.__task = None

    # TODO: Remove this commented method later
    # @asyncio.coroutine
    # def __handle_zap(self):
    #     while True:
    #         events = yield from self.__poller.poll()
    #         if self.zap_socket in dict(events):
    #             msg = yield from self.zap_socket.recv_multipart()
    #             self.handle_zap_message(msg) 

def test_zmq(self):
        block = self._get_block_with_tx()
        self.setup_zmq_urls()
        self._build()
        _tasks = [
            self._subscribe_topic(BitcoindZMQTopics.TX.value, self.context.zmqpubrawtx, 1),
            self._subscribe_topic(BitcoindZMQTopics.TXHASH.value, self.context.zmqpubhashtx, 2),
            self._subscribe_topic(BitcoindZMQTopics.BLOCK.value, self.context.zmqpubrawblock, 1),
            self._subscribe_topic(BitcoindZMQTopics.BLOCKHASH.value, self.context.zmqpubhashblock, 1),
            self._test_zmq(block)
        ]
        self._loop.run_until_complete(asyncio.gather(*_tasks))