Python zmq.Again() Examples

def backend_status_datasets() -> BackendStatusDatasetsSchema:
    """
    Returns a combined list of datasets from all agents.

    Caveat: In case of collision of dataset ids when there are multiple agents,
    this API will only return one dataset per colliding ID. Collision is
    extremally unlikely though and it shouldn't be a problem in the real world.

    This endpoint is not stable and may be subject to change in the future.
    """
    datasets: Dict[str, int] = {}
    for agent_spec in db.get_active_agents().values():
        try:
            ursa = UrsaDb(agent_spec.ursadb_url)
            datasets.update(ursa.topology()["result"]["datasets"])
        except Again:
            pass

    return BackendStatusDatasetsSchema(datasets=datasets) 

def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{} got error {} while sending through listener to {}' \
                .format(self, ex, ident)
            print(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 

def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        try:
            ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK)
            if msg:
                # Router probing sends empty message on connection
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
        except zmq.Again as e:
            return i
        except zmq.ZMQError as e:
            print("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            print('{} got {} messages through listener'.
                  format(self, i))
        return i 

def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            return False, err_str
        socket = remote.socket
        if not socket:
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            print('{} transmitting message {} to {} by socket {} {}'
                  .format(self, msg, uid, socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            return True, err_str
        except zmq.Again:
            print('{} could not transmit message to {}'.format(self, uid))
        return False, err_str 

def recv_array(socket, flags=0, copy=False, track=False, block=True):
    """recv a numpy array"""
    if block:
        md = socket.recv_json(flags=flags)
        msg = socket.recv(flags=flags, copy=copy, track=track)
        buf = bytearray(msg)
        A = np.frombuffer(buf, dtype=md['dtype'])
        return A.reshape(md['shape'])
    else:
        try:
            md = socket.recv_json(flags=flags | zmq.NOBLOCK)
            msg = socket.recv(flags=flags | zmq.NOBLOCK, copy=copy, track=track)
            buf = bytearray(msg)
            A = np.frombuffer(buf, dtype=md['dtype'])
            return A.reshape(md['shape'])
        except zmq.Again:
            return False 

def test_disconnection(self):
        """ Test the disconnection of subscribers. """
        from supvisors.utils import InternalEventHeaders
        # get the local address
        local_address = self.supvisors.address_mapper.local_address
        # test remote disconnection
        address = next(address
                       for address in self.supvisors.address_mapper.addresses
                       if address != local_address)
        self.subscriber.disconnect([address])
        # send a tick event from the local publisher
        payload = {'date': 1000}
        self.publisher.send_tick_event(payload)
        # check the reception of the tick event
        msg = self.receive('Tick')
        self.assertTupleEqual((InternalEventHeaders.TICK,
                               local_address, payload), msg)
        # test local disconnection
        self.subscriber.disconnect([local_address])
        # send a tick event from the local publisher
        self.publisher.send_tick_event(payload)
        # check the non-reception of the tick event
        with self.assertRaises(zmq.Again):
            self.subscriber.receive() 

def generator_from_zmq_pull(context, host):
    socket = context.socket(zmq.PULL)
    # TODO: Configure socket with clean properties to avoid message overload.
    if host.endswith('/'):
        host = host[:-1]
    print_item("+", "Binding ZMQ pull socket : " + colorama.Fore.CYAN + "{0}".format(host) + colorama.Style.RESET_ALL)
    socket.bind(host)

    while True:
        try:
            message = socket.recv(flags=zmq.NOBLOCK)
        except zmq.Again as e:
            message = None
        if message is None:
            yield None # NOTE: We have to make the generator non blocking.
        else:
            task = json.loads(message)
            yield task 

def recv_params_from_learner(self):
        """Get new params and sync. return True if success, False otherwise."""
        received = False
        try:
            new_params_id = self.sub_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            # Although learner doesn't send params, don't wait
            pass

        if received:
            new_param_info = pa.deserialize(new_params_id)
            update_step, new_params = new_param_info
            self.update_step = update_step
            self.worker.synchronize(new_params)

            # Add new entry for scores dict
            self.scores[self.update_step] = [] 

def recv_worker_data(self):
        """Receive replay data from worker and incorporate to buffer."""
        received = False
        try:
            new_replay_data_id = self.pull_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            pass

        if received:
            new_replay_data = pa.deserialize(new_replay_data_id)
            experience, priorities = new_replay_data
            for idx in range(len(experience["states"])):
                transition = (
                    experience["states"][idx],
                    experience["actions"][idx],
                    experience["rewards"][idx],
                    experience["next_states"][idx],
                    experience["dones"][idx],
                )
                self.buffer.add(transition)
                self.buffer.update_priorities([len(self.buffer) - 1], priorities[idx]) 

def receive_replies(self, context=None):
        result_sock = context.socket(zmq.PULL)
        self.receive_port = result_sock.bind_to_random_port("tcp://*")
        logger.debug("Result port is %s" % self.receive_port)
        set_timeouts_on_socket(result_sock)

        while True:
            try:
                message_reply = result_sock.recv_json()
                if message_reply['status'] == 'success':
                    self.success(message_reply['id'])
                elif message_reply['status'] == 'fail':
                    self.failed(message_reply['id'])
                else:
                    logger.warn("Received unknown status feedback %s" % message_reply['status'])
            except zmq.Again:
                time.sleep(1) 

def run(self):
        if self.should_stop:  # early exit
            return 1

        context = zmq.Context()
        socket = context.socket(zmq.SUB)

        logger.info(f"subscribing to events from {self.socket_uri}…")
        socket.connect(self.socket_uri)
        for event in self.events:
            logger.debug(f".. {event}")
            socket.setsockopt_string(zmq.SUBSCRIBE, event)

        while not self.should_stop:
            try:
                received_string = socket.recv_string(zmq.DONTWAIT)
                self.handle_broadcast_event(received_string)
            except zmq.Again:
                pass

            if self.should_poll:
                self.sync_tasks_and_containers()
                self.poll()
            else:
                self.sleep() 

def receive_message(socket, blocking=True):
        flags = 0 if blocking else zmq.NOBLOCK
        try:
            cmd, data = socket.recv_multipart(flags=flags)
            return cmd, data
        except zmq.Again:
            return None, None
        except zmq.ContextTerminated:
            print("Context terminated ..")
            return None, None
        except KeyboardInterrupt:
            return None, None 

def backend_status() -> BackendStatusSchema:
    """
    Returns the current status of backend services, and returns it. Intended to
    be used by the webpage.

    This endpoint is not stable and may be subject to change in the future.
    """
    agents = []
    components = {
        "mquery": mquery_version(),
    }
    for name, agent_spec in db.get_active_agents().items():
        try:
            ursa = UrsaDb(agent_spec.ursadb_url)
            status = ursa.status()
            tasks = status["result"]["tasks"]
            ursadb_version = status["result"]["ursadb_version"]
            agents.append(
                AgentSchema(
                    name=name, alive=True, tasks=tasks, spec=agent_spec
                )
            )
            components[f"ursadb ({name})"] = ursadb_version
        except Again:
            agents.append(
                AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec)
            )
            components[f"ursadb ({name})"] = "unknown"

    return BackendStatusSchema(agents=agents, components=components,) 

def recv_all(self, socket, flags=0, validate=True):
        '''Receive all messages currently available from the given socket.'''
        messages = []
        while True:
            try:
                messages.append(self.recv_message(socket, flags | zmq.NOBLOCK, validate))
            except zmq.Again:
                return messages 

def _add_timeout(self, future, timeout):
        """Add a timeout for a send or recv Future"""
        def future_timeout():
            if future.done():
                # future already resolved, do nothing
                return

            # raise EAGAIN
            future.set_exception(_zmq.Again())
        self._call_later(timeout, future_timeout) 

def test_again(self):
        s = self.context.socket(zmq.REP)
        self.assertRaises(Again, s.recv, zmq.NOBLOCK)
        self.assertRaisesErrno(zmq.EAGAIN, s.recv, zmq.NOBLOCK)
        s.close() 

def test_recv_timeout(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with self.assertRaises(zmq.Again):
                yield from f1
            yield from a.send_multipart([b'hi', b'there'])
            recvd = yield from f2
            assert f2.done()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def test_send_timeout(self):
        @asyncio.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with self.assertRaises(zmq.Again):
                yield from s.send(b'not going anywhere')
        self.loop.run_until_complete(test()) 

def test_recv_dontwait(self):
        @asyncio.coroutine
        def test():
            push, pull = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = pull.recv(zmq.DONTWAIT)
            with self.assertRaises(zmq.Again):
                yield from f
            yield from push.send(b'ping')
            yield from pull.poll() # ensure message will be waiting
            f = pull.recv(zmq.DONTWAIT)
            assert f.done()
            msg = yield from f
            self.assertEqual(msg, b'ping')
        self.loop.run_until_complete(test()) 

def test_retry_recv(self):
        pull = self.socket(zmq.PULL)
        pull.rcvtimeo = self.timeout_ms
        self.alarm()
        self.assertRaises(zmq.Again, pull.recv)
        assert self.timer_fired 

def test_retry_send(self):
        push = self.socket(zmq.PUSH)
        push.sndtimeo = self.timeout_ms
        self.alarm()
        self.assertRaises(zmq.Again, push.send, b('buf'))
        assert self.timer_fired 

def test_recv_timeout(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with pytest.raises(zmq.Again):
                yield f1
            yield  a.send_multipart([b'hi', b'there'])
            recvd = yield f2
            assert f2.done()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_sync(test) 

def test_send_timeout(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere')
        self.loop.run_sync(test) 

def test_send_noblock(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere', flags=zmq.NOBLOCK)
        self.loop.run_sync(test) 

def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        while i < quota.count and incoming_size < quota.size:
            try:
                received = self.listener.recv_multipart(flags=zmq.NOBLOCK)
                # If client was connected in DEALER mode then is expected
                # to get exactly 2 values from recv_multipart function
                if len(received) > 2:
                    logger.debug("Got too many values for unpack. "
                                 "It seems that client's socket is open not in DEALER mode")
                    continue
                ident, msg = received
                if not msg:
                    # Router probing sends empty message on connection
                    continue
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
            except zmq.Again as e:
                break
            except zmq.ZMQError as e:
                logger.debug("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            logger.trace('{} got {} messages through listener'.
                         format(self, i))
        return i 

def _receiveFromRemotes(self, quotaPerRemote) -> int:
        """
        Receives messages from remotes
        :param quotaPerRemote: number of messages to receive from one remote
        :return: number of received messages
        """

        assert quotaPerRemote
        totalReceived = 0
        for ident, remote in self.remotesByKeys.items():
            if not remote.socket:
                continue
            i = 0
            sock = remote.socket
            while i < quotaPerRemote:
                try:
                    msg, = sock.recv_multipart(flags=zmq.NOBLOCK)
                    if not msg:
                        # Router probing sends empty message on connection
                        continue
                    i += 1
                    logger.trace("{} received a message from remote {} by socket {} {}", self,
                                 z85_to_friendly(ident), sock.FD, sock.underlying)
                    self._verifyAndAppend(msg, ident)
                except zmq.Again as e:
                    break
                except zmq.ZMQError as e:
                    logger.debug(
                        "Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
            if i > 0:
                logger.trace('{} got {} messages through remote {}'.
                             format(self, i, remote))
            totalReceived += i
        return totalReceived 

def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid)))
            return False, err_str
        socket = remote.socket
        if not socket:
            logger.debug('{} has uninitialised socket '
                         'for remote {}'.format(self, z85_to_friendly(uid)))
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            logger.trace('{} transmitting message {} to {} by socket {} {}'
                         .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            if remote.isConnected or msg in self.healthMessages:
                self.metrics.add_event(self.mt_outgoing_size, len(msg))
            else:
                logger.debug('Remote {} is not connected - message will not be sent immediately.'
                             'If this problem does not resolve itself - check your firewall settings'
                             .format(z85_to_friendly(uid)))

            return True, err_str
        except zmq.Again:
            logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid)))
        except InvalidMessageExceedingSizeException as ex:
            err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
            logger.warning(err_str)
        return False, err_str 

def get_monitor_events(monitor_socket, non_block=True):
        events = []
        # noinspection PyUnresolvedReferences
        flags = zmq.NOBLOCK if non_block else 0
        while True:
            try:
                # noinspection PyUnresolvedReferences
                message = recv_monitor_message(monitor_socket, flags)
                events.append(message)
            except zmq.Again:
                break
        return events 

def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{}{} got error {} while sending through listener to {}' \
                .format(CONNECTION_PREFIX, self, ex, ident)
            logger.debug(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            logger.trace('{} transmitting {} to {} through listener socket'.
                         format(self, msg, ident))
            self.metrics.add_event(self._mt_outgoing_size, len(msg))
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except InvalidMessageExceedingSizeException as ex:
            err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
            logger.debug(err_str)
            return False, err_str, need_to_resend
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 

def test_resending_for_old_stash_msgs(tdir, tconf, looper, stacks,
                                      alpha_handler, monkeypatch):
    alpha, beta = stacks
    msg1 = {'msg': 'msg1'}
    pending_client_messages = beta._client_message_provider._pending_client_messages

    alpha.connect(name=beta.name, ha=beta.ha,
                  verKeyRaw=beta.verKeyRaw, publicKeyRaw=beta.publicKeyRaw)
    looper.runFor(0.25)

    def fake_send_multipart(msg_parts, flags=0, copy=True, track=False, **kwargs):
        raise zmq.Again

    monkeypatch.setattr(beta.listener, 'send_multipart',
                        fake_send_multipart)
    alpha.send(msg1, beta.name)
    looper.run(eventually(
        lambda messages: assertExp(messages[alpha.listener.IDENTITY] == [(0, msg1)]),
        pending_client_messages))
    monkeypatch.undo()

    beta._client_message_provider._timer.set_time(tconf.RESEND_CLIENT_MSG_TIMEOUT + 2)
    looper.run(eventually(
        lambda msg_handler: assertExp(msg_handler.received_messages == [msg1]),
        alpha_handler))
    assert not pending_client_messages