Python zmq.Again() Examples

def test_disconnection(self):
        """ Test the disconnection of subscribers. """
        from supvisors.utils import InternalEventHeaders
        # get the local address
        local_address = self.supvisors.address_mapper.local_address
        # test remote disconnection
        address = next(address
                       for address in self.supvisors.address_mapper.addresses
                       if address != local_address)
        self.subscriber.disconnect([address])
        # send a tick event from the local publisher
        payload = {'date': 1000}
        self.publisher.send_tick_event(payload)
        # check the reception of the tick event
        msg = self.receive('Tick')
        self.assertTupleEqual((InternalEventHeaders.TICK,
                               local_address, payload), msg)
        # test local disconnection
        self.subscriber.disconnect([local_address])
        # send a tick event from the local publisher
        self.publisher.send_tick_event(payload)
        # check the non-reception of the tick event
        with self.assertRaises(zmq.Again):
            self.subscriber.receive() 

def recv_array(socket, flags=0, copy=False, track=False, block=True):
    """recv a numpy array"""
    if block:
        md = socket.recv_json(flags=flags)
        msg = socket.recv(flags=flags, copy=copy, track=track)
        buf = bytearray(msg)
        A = np.frombuffer(buf, dtype=md['dtype'])
        return A.reshape(md['shape'])
    else:
        try:
            md = socket.recv_json(flags=flags | zmq.NOBLOCK)
            msg = socket.recv(flags=flags | zmq.NOBLOCK, copy=copy, track=track)
            buf = bytearray(msg)
            A = np.frombuffer(buf, dtype=md['dtype'])
            return A.reshape(md['shape'])
        except zmq.Again:
            return False 

def generator_from_zmq_pull(context, host):
    socket = context.socket(zmq.PULL)
    # TODO: Configure socket with clean properties to avoid message overload.
    if host.endswith('/'):
        host = host[:-1]
    print_item("+", "Binding ZMQ pull socket : " + colorama.Fore.CYAN + "{0}".format(host) + colorama.Style.RESET_ALL)
    socket.bind(host)

    while True:
        try:
            message = socket.recv(flags=zmq.NOBLOCK)
        except zmq.Again as e:
            message = None
        if message is None:
            yield None # NOTE: We have to make the generator non blocking.
        else:
            task = json.loads(message)
            yield task 

def recv_params_from_learner(self):
        """Get new params and sync. return True if success, False otherwise."""
        received = False
        try:
            new_params_id = self.sub_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            # Although learner doesn't send params, don't wait
            pass

        if received:
            new_param_info = pa.deserialize(new_params_id)
            update_step, new_params = new_param_info
            self.update_step = update_step
            self.worker.synchronize(new_params)

            # Add new entry for scores dict
            self.scores[self.update_step] = [] 

def recv_worker_data(self):
        """Receive replay data from worker and incorporate to buffer."""
        received = False
        try:
            new_replay_data_id = self.pull_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            pass

        if received:
            new_replay_data = pa.deserialize(new_replay_data_id)
            experience, priorities = new_replay_data
            for idx in range(len(experience["states"])):
                transition = (
                    experience["states"][idx],
                    experience["actions"][idx],
                    experience["rewards"][idx],
                    experience["next_states"][idx],
                    experience["dones"][idx],
                )
                self.buffer.add(transition)
                self.buffer.update_priorities([len(self.buffer) - 1], priorities[idx]) 

def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            return False, err_str
        socket = remote.socket
        if not socket:
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            print('{} transmitting message {} to {} by socket {} {}'
                  .format(self, msg, uid, socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            return True, err_str
        except zmq.Again:
            print('{} could not transmit message to {}'.format(self, uid))
        return False, err_str 

def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        try:
            ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK)
            if msg:
                # Router probing sends empty message on connection
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
        except zmq.Again as e:
            return i
        except zmq.ZMQError as e:
            print("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            print('{} got {} messages through listener'.
                  format(self, i))
        return i 

def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{} got error {} while sending through listener to {}' \
                .format(self, ex, ident)
            print(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 

def run(self):
        if self.should_stop:  # early exit
            return 1

        context = zmq.Context()
        socket = context.socket(zmq.SUB)

        logger.info(f"subscribing to events from {self.socket_uri}…")
        socket.connect(self.socket_uri)
        for event in self.events:
            logger.debug(f".. {event}")
            socket.setsockopt_string(zmq.SUBSCRIBE, event)

        while not self.should_stop:
            try:
                received_string = socket.recv_string(zmq.DONTWAIT)
                self.handle_broadcast_event(received_string)
            except zmq.Again:
                pass

            if self.should_poll:
                self.sync_tasks_and_containers()
                self.poll()
            else:
                self.sleep() 

def receive_replies(self, context=None):
        result_sock = context.socket(zmq.PULL)
        self.receive_port = result_sock.bind_to_random_port("tcp://*")
        logger.debug("Result port is %s" % self.receive_port)
        set_timeouts_on_socket(result_sock)

        while True:
            try:
                message_reply = result_sock.recv_json()
                if message_reply['status'] == 'success':
                    self.success(message_reply['id'])
                elif message_reply['status'] == 'fail':
                    self.failed(message_reply['id'])
                else:
                    logger.warn("Received unknown status feedback %s" % message_reply['status'])
            except zmq.Again:
                time.sleep(1) 

def backend_status_datasets() -> BackendStatusDatasetsSchema:
    """
    Returns a combined list of datasets from all agents.

    Caveat: In case of collision of dataset ids when there are multiple agents,
    this API will only return one dataset per colliding ID. Collision is
    extremally unlikely though and it shouldn't be a problem in the real world.

    This endpoint is not stable and may be subject to change in the future.
    """
    datasets: Dict[str, int] = {}
    for agent_spec in db.get_active_agents().values():
        try:
            ursa = UrsaDb(agent_spec.ursadb_url)
            datasets.update(ursa.topology()["result"]["datasets"])
        except Again:
            pass

    return BackendStatusDatasetsSchema(datasets=datasets) 

def send(self, mtype, message):
        self.logger.debug("Sending message {}, {}".format(mtype, message))
        try:
            self._dfk_channel.send_pyobj((mtype, message))
        except zmq.Again:
            self.logger.exception(
                "[MONITORING] The monitoring message sent from DFK to Hub timeouts after {}ms".format(self.dfk_channel_timeout)) 

def send_message(self, message, process_id, time_consumed=None, sender=None, control_message=True, retries=100):
        """

        :param message: A ::pipeline.messages.Message:: instance
        :param process_id:
        :param time_consumed:
        :param sender:
        :param control_message:
        :param retries:
        :type retries: int
        :raises: SocketBlockedException, GroupingValueMissing
        :return:
        """
        try:
            socket_addresses = self.grouper_instance.get_destinations_for(message.grouping_value)
        except GroupingValueMissing:
            raise GroupingValueMissing("Message '%s' provided an invalid grouping_value: '%s'" % (message.content, message.grouping_value))
        for destination in socket_addresses:
            message.send(
                self.send_socks[destination],
                process_id
            )
            if self.controller_sock and self.send_control_messages and control_message:
                for index in xrange(0, retries):
                    try:
                        message.send_control_message(
                            self.controller_sock,
                            time_consumed=time_consumed,
                            process_name=process_id,
                            destination_uuid=self.process_address_to_uuid[destination],
                            destination_endpoint=destination,
                            sender=sender
                        )
                        break
                    except KeyError:  # If destination is not known, lookup fails
                        time.sleep(random.randrange(1, 3))  # random to avoid peak loads when running multiple processes
                    except zmq.Again:  # If connection fails, retry
                        time.sleep(random.randrange(1, 3))
                else:  # if for loop exited cleanly (no break)
                    raise SocketBlockedException("Control Message for process %s could not be sent after %s attempts"
                                                 % (process_id, retries)) 

def start_listener(self):
        print('ZMQ listener started')
        while True:
            try:
                self.s.recv(zmq.NOBLOCK)  # note NOBLOCK here
            except zmq.Again:
                # no message to recv, do other things
                time.sleep(0.05)
            else:
                self.on_q.put(ON_SIGNAL) 

def remote_pull(socket):
    
    try:
        msg = socket.recv(flags=zmq.NOBLOCK)
        print msg
        
    except zmq.Again as e:
        print "Waiting for PUSH from MetaTrader 4.."
    
# Run Tests 

def test_send_timeout(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere')
        self.loop.run_sync(test) 

def test_send_noblock(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere', flags=zmq.NOBLOCK)
        self.loop.run_sync(test) 

def remote_send(socket, data):
    
    try:
        socket.send(data)
        msg = socket.recv_string()
        print msg
        
    except zmq.Again as e:
        print "Waiting for PUSH from MetaTrader 4.."
    
# Function to retrieve data from ZeroMQ MT4 EA 

def test_recv_timeout(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with pytest.raises(zmq.Again):
                yield f1
            yield  a.send_multipart([b'hi', b'there'])
            recvd = yield f2
            assert f2.done()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_sync(test) 

def get_connections_block(queue, refresh_connection_socket, limit=100, existing_connections=None):
    i = 0
    connections = existing_connections if existing_connections else {}
    while queue not in connections and i < limit:
        try:
            connections = refresh_connection_socket.recv_json()
        except zmq.Again:
            pass
        i += 1
    return connections 

def _add_timeout(self, future, timeout):
        """Add a timeout for a send or recv Future"""
        def future_timeout():
            if future.done():
                # future already resolved, do nothing
                return

            # raise EAGAIN
            future.set_exception(_zmq.Again())
        self._call_later(timeout, future_timeout) 

def start(self, priority_msgs, node_msgs, resource_msgs):

        while True:
            try:
                data, addr = self.sock.recvfrom(2048)
                msg = pickle.loads(data)
                resource_msgs.put((msg, addr))
                self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
            except socket.timeout:
                pass

            try:
                msg = self.dfk_channel.recv_pyobj()
                self.logger.debug("Got ZMQ Message from DFK: {}".format(msg))
                priority_msgs.put((msg, 0))
                if msg[0].value == MessageType.WORKFLOW_INFO.value and 'python_version' not in msg[1]:
                    break
            except zmq.Again:
                pass

            try:
                msg = self.ic_channel.recv_pyobj()
                msg[1]['run_id'] = self.run_id
                msg = (msg[0], msg[1])
                self.logger.debug("Got ZMQ Message from interchange: {}".format(msg))
                node_msgs.put((msg, 0))
            except zmq.Again:
                pass

        last_msg_received_time = time.time()
        while time.time() - last_msg_received_time < self.atexit_timeout:
            try:
                data, addr = self.sock.recvfrom(2048)
                msg = pickle.loads(data)
                resource_msgs.put((msg, addr))
                last_msg_received_time = time.time()
                self.logger.debug("Got UDP Message from {}: {}".format(addr, msg))
            except socket.timeout:
                pass

        self.logger.info("Monitoring router finished") 

def recv_log_info(self):
        """Receive info from learner."""
        received = False
        try:
            log_info_id = self.pull_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            pass

        if received:
            self.log_info_queue.append(log_info_id) 

def migrate_tasks_to_internal(self, kill_event):
        """Pull tasks from the incoming tasks 0mq pipe onto the internal
        pending task queue

        Parameters:
        -----------
        kill_event : threading.Event
              Event to let the thread know when it is time to die.
        """
        logger.info("[TASK_PULL_THREAD] Starting")
        task_counter = 0
        poller = zmq.Poller()
        poller.register(self.task_incoming, zmq.POLLIN)

        while not kill_event.is_set():
            try:
                msg = self.task_incoming.recv_pyobj()
            except zmq.Again:
                # We just timed out while attempting to receive
                logger.debug("[TASK_PULL_THREAD] {} tasks in internal queue".format(self.pending_task_queue.qsize()))
                continue

            if msg == 'STOP':
                kill_event.set()
                break
            else:
                self.pending_task_queue.put(msg)
                task_counter += 1
                logger.debug("[TASK_PULL_THREAD] Fetched task:{}".format(task_counter)) 

def _add_timeout(self, future, timeout):
        """Add a timeout for a send or recv Future"""
        def future_timeout():
            if future.done():
                # future already resolved, do nothing
                return

            # raise EAGAIN
            future.set_exception(_zmq.Again())
        self._call_later(timeout, future_timeout) 

def run(self):
        while True:
            try:
                message = self.pull.recv(flags=zmq.NOBLOCK)
            except zmq.Again as e:
                message = None
            if message is not None:
                task = json.loads(message)
                self.redis.setex(
                    task['transaction'],
                    self.result_expiration,
                    json.dumps(task['data'])
                ) 

def check_reception(self, header=None, data=None):
        """ The method tests that the message is received correctly
        or not received at all. """
        if header and data:
            # check that subscriber receives the message
            try:
                msg = self.subscriber.receive()
            except zmq.Again:
                self.fail('Failed to get {} status'.format(header))
            self.assertTupleEqual((header, data), msg)
        else:
            # check the non-reception of the Supvisors status
            with self.assertRaises(zmq.Again):
                self.subscriber.receive() 

def receive(self, event_type):
        """ This method performs a checked reception on the puller. """
        try:
            return self.puller.receive()
        except zmq.Again:
            self.fail('Failed to get {} request'. format(event_type)) 

def receive(self, event_type):
        """ This method performs a checked reception on the subscriber. """
        try:
            self.subscriber.socket.poll(1000)
            return self.subscriber.receive()
        except zmq.Again:
            self.fail('Failed to get {} event'.format(event_type)) 

def test_again(self):
        s = self.context.socket(zmq.REP)
        self.assertRaises(Again, s.recv, zmq.NOBLOCK)
        self.assertRaisesErrno(zmq.EAGAIN, s.recv, zmq.NOBLOCK)
        s.close()