Python zmq.Again() Examples

The following are 30 code examples of zmq.Again(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module zmq , or try the search function .
Example #1
Source Project: mquery   Author: CERT-Polska   File: app.py    License: GNU Affero General Public License v3.0 6 votes vote down vote up
def backend_status_datasets() -> BackendStatusDatasetsSchema:
    """
    Returns a combined list of datasets from all agents.

    Caveat: In case of collision of dataset ids when there are multiple agents,
    this API will only return one dataset per colliding ID. Collision is
    extremally unlikely though and it shouldn't be a problem in the real world.

    This endpoint is not stable and may be subject to change in the future.
    """
    datasets: Dict[str, int] = {}
    for agent_spec in db.get_active_agents().values():
        try:
            ursa = UrsaDb(agent_spec.ursadb_url)
            datasets.update(ursa.topology()["result"]["datasets"])
        except Again:
            pass

    return BackendStatusDatasetsSchema(datasets=datasets) 
Example #2
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 6 votes vote down vote up
def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{} got error {} while sending through listener to {}' \
                .format(self, ex, ident)
            print(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 
Example #3
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 6 votes vote down vote up
def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        try:
            ident, msg = self.listener.recv_multipart(flags=zmq.NOBLOCK)
            if msg:
                # Router probing sends empty message on connection
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
        except zmq.Again as e:
            return i
        except zmq.ZMQError as e:
            print("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            print('{} got {} messages through listener'.
                  format(self, i))
        return i 
Example #4
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 6 votes vote down vote up
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            return False, err_str
        socket = remote.socket
        if not socket:
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            print('{} transmitting message {} to {} by socket {} {}'
                  .format(self, msg, uid, socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            return True, err_str
        except zmq.Again:
            print('{} could not transmit message to {}'.format(self, uid))
        return False, err_str 
Example #5
Source Project: cryptotrader   Author: naripok   File: utils.py    License: MIT License 6 votes vote down vote up
def recv_array(socket, flags=0, copy=False, track=False, block=True):
    """recv a numpy array"""
    if block:
        md = socket.recv_json(flags=flags)
        msg = socket.recv(flags=flags, copy=copy, track=track)
        buf = bytearray(msg)
        A = np.frombuffer(buf, dtype=md['dtype'])
        return A.reshape(md['shape'])
    else:
        try:
            md = socket.recv_json(flags=flags | zmq.NOBLOCK)
            msg = socket.recv(flags=flags | zmq.NOBLOCK, copy=copy, track=track)
            buf = bytearray(msg)
            A = np.frombuffer(buf, dtype=md['dtype'])
            return A.reshape(md['shape'])
        except zmq.Again:
            return False 
Example #6
Source Project: supvisors   Author: julien6387   File: test_supvisorszmq.py    License: Apache License 2.0 6 votes vote down vote up
def test_disconnection(self):
        """ Test the disconnection of subscribers. """
        from supvisors.utils import InternalEventHeaders
        # get the local address
        local_address = self.supvisors.address_mapper.local_address
        # test remote disconnection
        address = next(address
                       for address in self.supvisors.address_mapper.addresses
                       if address != local_address)
        self.subscriber.disconnect([address])
        # send a tick event from the local publisher
        payload = {'date': 1000}
        self.publisher.send_tick_event(payload)
        # check the reception of the tick event
        msg = self.receive('Tick')
        self.assertTupleEqual((InternalEventHeaders.TICK,
                               local_address, payload), msg)
        # test local disconnection
        self.subscriber.disconnect([local_address])
        # send a tick event from the local publisher
        self.publisher.send_tick_event(payload)
        # check the non-reception of the tick event
        with self.assertRaises(zmq.Again):
            self.subscriber.receive() 
Example #7
Source Project: og-miner   Author: opendns   File: miner.py    License: MIT License 6 votes vote down vote up
def generator_from_zmq_pull(context, host):
    socket = context.socket(zmq.PULL)
    # TODO: Configure socket with clean properties to avoid message overload.
    if host.endswith('/'):
        host = host[:-1]
    print_item("+", "Binding ZMQ pull socket : " + colorama.Fore.CYAN + "{0}".format(host) + colorama.Style.RESET_ALL)
    socket.bind(host)

    while True:
        try:
            message = socket.recv(flags=zmq.NOBLOCK)
        except zmq.Again as e:
            message = None
        if message is None:
            yield None # NOTE: We have to make the generator non blocking.
        else:
            task = json.loads(message)
            yield task 
Example #8
Source Project: rl_algorithms   Author: medipixel   File: worker.py    License: MIT License 6 votes vote down vote up
def recv_params_from_learner(self):
        """Get new params and sync. return True if success, False otherwise."""
        received = False
        try:
            new_params_id = self.sub_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            # Although learner doesn't send params, don't wait
            pass

        if received:
            new_param_info = pa.deserialize(new_params_id)
            update_step, new_params = new_param_info
            self.update_step = update_step
            self.worker.synchronize(new_params)

            # Add new entry for scores dict
            self.scores[self.update_step] = [] 
Example #9
Source Project: rl_algorithms   Author: medipixel   File: wrapper.py    License: MIT License 6 votes vote down vote up
def recv_worker_data(self):
        """Receive replay data from worker and incorporate to buffer."""
        received = False
        try:
            new_replay_data_id = self.pull_socket.recv(zmq.DONTWAIT)
            received = True
        except zmq.Again:
            pass

        if received:
            new_replay_data = pa.deserialize(new_replay_data_id)
            experience, priorities = new_replay_data
            for idx in range(len(experience["states"])):
                transition = (
                    experience["states"][idx],
                    experience["actions"][idx],
                    experience["rewards"][idx],
                    experience["next_states"][idx],
                    experience["dones"][idx],
                )
                self.buffer.add(transition)
                self.buffer.update_priorities([len(self.buffer) - 1], priorities[idx]) 
Example #10
Source Project: motorway   Author: plecto   File: ramp.py    License: Apache License 2.0 6 votes vote down vote up
def receive_replies(self, context=None):
        result_sock = context.socket(zmq.PULL)
        self.receive_port = result_sock.bind_to_random_port("tcp://*")
        logger.debug("Result port is %s" % self.receive_port)
        set_timeouts_on_socket(result_sock)

        while True:
            try:
                message_reply = result_sock.recv_json()
                if message_reply['status'] == 'success':
                    self.success(message_reply['id'])
                elif message_reply['status'] == 'fail':
                    self.failed(message_reply['id'])
                else:
                    logger.warn("Received unknown status feedback %s" % message_reply['status'])
            except zmq.Again:
                time.sleep(1) 
Example #11
Source Project: zimfarm   Author: openzim   File: worker.py    License: GNU General Public License v3.0 6 votes vote down vote up
def run(self):
        if self.should_stop:  # early exit
            return 1

        context = zmq.Context()
        socket = context.socket(zmq.SUB)

        logger.info(f"subscribing to events from {self.socket_uri}…")
        socket.connect(self.socket_uri)
        for event in self.events:
            logger.debug(f".. {event}")
            socket.setsockopt_string(zmq.SUBSCRIBE, event)

        while not self.should_stop:
            try:
                received_string = socket.recv_string(zmq.DONTWAIT)
                self.handle_broadcast_event(received_string)
            except zmq.Again:
                pass

            if self.should_poll:
                self.sync_tasks_and_containers()
                self.poll()
            else:
                self.sleep() 
Example #12
Source Project: idasec   Author: RobinDavid   File: broker.py    License: GNU Lesser General Public License v2.1 5 votes vote down vote up
def receive_message(socket, blocking=True):
        flags = 0 if blocking else zmq.NOBLOCK
        try:
            cmd, data = socket.recv_multipart(flags=flags)
            return cmd, data
        except zmq.Again:
            return None, None
        except zmq.ContextTerminated:
            print("Context terminated ..")
            return None, None
        except KeyboardInterrupt:
            return None, None 
Example #13
Source Project: mquery   Author: CERT-Polska   File: app.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def backend_status() -> BackendStatusSchema:
    """
    Returns the current status of backend services, and returns it. Intended to
    be used by the webpage.

    This endpoint is not stable and may be subject to change in the future.
    """
    agents = []
    components = {
        "mquery": mquery_version(),
    }
    for name, agent_spec in db.get_active_agents().items():
        try:
            ursa = UrsaDb(agent_spec.ursadb_url)
            status = ursa.status()
            tasks = status["result"]["tasks"]
            ursadb_version = status["result"]["ursadb_version"]
            agents.append(
                AgentSchema(
                    name=name, alive=True, tasks=tasks, spec=agent_spec
                )
            )
            components[f"ursadb ({name})"] = ursadb_version
        except Again:
            agents.append(
                AgentSchema(name=name, alive=False, tasks=[], spec=agent_spec)
            )
            components[f"ursadb ({name})"] = "unknown"

    return BackendStatusSchema(agents=agents, components=components,) 
Example #14
Source Project: westpa   Author: westpa   File: core.py    License: MIT License 5 votes vote down vote up
def recv_all(self, socket, flags=0, validate=True):
        '''Receive all messages currently available from the given socket.'''
        messages = []
        while True:
            try:
                messages.append(self.recv_message(socket, flags | zmq.NOBLOCK, validate))
            except zmq.Again:
                return messages 
Example #15
Source Project: vnpy_crypto   Author: birforce   File: _future.py    License: MIT License 5 votes vote down vote up
def _add_timeout(self, future, timeout):
        """Add a timeout for a send or recv Future"""
        def future_timeout():
            if future.done():
                # future already resolved, do nothing
                return

            # raise EAGAIN
            future.set_exception(_zmq.Again())
        self._call_later(timeout, future_timeout) 
Example #16
Source Project: vnpy_crypto   Author: birforce   File: test_error.py    License: MIT License 5 votes vote down vote up
def test_again(self):
        s = self.context.socket(zmq.REP)
        self.assertRaises(Again, s.recv, zmq.NOBLOCK)
        self.assertRaisesErrno(zmq.EAGAIN, s.recv, zmq.NOBLOCK)
        s.close() 
Example #17
Source Project: vnpy_crypto   Author: birforce   File: _test_asyncio.py    License: MIT License 5 votes vote down vote up
def test_recv_timeout(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with self.assertRaises(zmq.Again):
                yield from f1
            yield from a.send_multipart([b'hi', b'there'])
            recvd = yield from f2
            assert f2.done()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 
Example #18
Source Project: vnpy_crypto   Author: birforce   File: _test_asyncio.py    License: MIT License 5 votes vote down vote up
def test_send_timeout(self):
        @asyncio.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with self.assertRaises(zmq.Again):
                yield from s.send(b'not going anywhere')
        self.loop.run_until_complete(test()) 
Example #19
Source Project: vnpy_crypto   Author: birforce   File: _test_asyncio.py    License: MIT License 5 votes vote down vote up
def test_recv_dontwait(self):
        @asyncio.coroutine
        def test():
            push, pull = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = pull.recv(zmq.DONTWAIT)
            with self.assertRaises(zmq.Again):
                yield from f
            yield from push.send(b'ping')
            yield from pull.poll() # ensure message will be waiting
            f = pull.recv(zmq.DONTWAIT)
            assert f.done()
            msg = yield from f
            self.assertEqual(msg, b'ping')
        self.loop.run_until_complete(test()) 
Example #20
Source Project: vnpy_crypto   Author: birforce   File: test_retry_eintr.py    License: MIT License 5 votes vote down vote up
def test_retry_recv(self):
        pull = self.socket(zmq.PULL)
        pull.rcvtimeo = self.timeout_ms
        self.alarm()
        self.assertRaises(zmq.Again, pull.recv)
        assert self.timer_fired 
Example #21
Source Project: vnpy_crypto   Author: birforce   File: test_retry_eintr.py    License: MIT License 5 votes vote down vote up
def test_retry_send(self):
        push = self.socket(zmq.PUSH)
        push.sndtimeo = self.timeout_ms
        self.alarm()
        self.assertRaises(zmq.Again, push.send, b('buf'))
        assert self.timer_fired 
Example #22
Source Project: vnpy_crypto   Author: birforce   File: test_future.py    License: MIT License 5 votes vote down vote up
def test_recv_timeout(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            b.rcvtimeo = 100
            f1 = b.recv()
            b.rcvtimeo = 1000
            f2 = b.recv_multipart()
            with pytest.raises(zmq.Again):
                yield f1
            yield  a.send_multipart([b'hi', b'there'])
            recvd = yield f2
            assert f2.done()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_sync(test) 
Example #23
Source Project: vnpy_crypto   Author: birforce   File: test_future.py    License: MIT License 5 votes vote down vote up
def test_send_timeout(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            s.sndtimeo = 100
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere')
        self.loop.run_sync(test) 
Example #24
Source Project: vnpy_crypto   Author: birforce   File: test_future.py    License: MIT License 5 votes vote down vote up
def test_send_noblock(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere', flags=zmq.NOBLOCK)
        self.loop.run_sync(test) 
Example #25
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 5 votes vote down vote up
def _receiveFromListener(self, quota: Quota) -> int:
        """
        Receives messages from listener
        :param quota: number of messages to receive
        :return: number of received messages
        """
        i = 0
        incoming_size = 0
        while i < quota.count and incoming_size < quota.size:
            try:
                received = self.listener.recv_multipart(flags=zmq.NOBLOCK)
                # If client was connected in DEALER mode then is expected
                # to get exactly 2 values from recv_multipart function
                if len(received) > 2:
                    logger.debug("Got too many values for unpack. "
                                 "It seems that client's socket is open not in DEALER mode")
                    continue
                ident, msg = received
                if not msg:
                    # Router probing sends empty message on connection
                    continue
                incoming_size += len(msg)
                i += 1
                self._verifyAndAppend(msg, ident)
            except zmq.Again as e:
                break
            except zmq.ZMQError as e:
                logger.debug("Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
        if i > 0:
            logger.trace('{} got {} messages through listener'.
                         format(self, i))
        return i 
Example #26
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 5 votes vote down vote up
def _receiveFromRemotes(self, quotaPerRemote) -> int:
        """
        Receives messages from remotes
        :param quotaPerRemote: number of messages to receive from one remote
        :return: number of received messages
        """

        assert quotaPerRemote
        totalReceived = 0
        for ident, remote in self.remotesByKeys.items():
            if not remote.socket:
                continue
            i = 0
            sock = remote.socket
            while i < quotaPerRemote:
                try:
                    msg, = sock.recv_multipart(flags=zmq.NOBLOCK)
                    if not msg:
                        # Router probing sends empty message on connection
                        continue
                    i += 1
                    logger.trace("{} received a message from remote {} by socket {} {}", self,
                                 z85_to_friendly(ident), sock.FD, sock.underlying)
                    self._verifyAndAppend(msg, ident)
                except zmq.Again as e:
                    break
                except zmq.ZMQError as e:
                    logger.debug(
                        "Strange ZMQ behaviour during node-to-node message receiving, experienced {}".format(e))
            if i > 0:
                logger.trace('{} got {} messages through remote {}'.
                             format(self, i, remote))
            totalReceived += i
        return totalReceived 
Example #27
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 5 votes vote down vote up
def transmit(self, msg, uid, timeout=None, serialized=False, is_batch=False):
        remote = self.remotes.get(uid)
        err_str = None
        if not remote:
            logger.debug("Remote {} does not exist!".format(z85_to_friendly(uid)))
            return False, err_str
        socket = remote.socket
        if not socket:
            logger.debug('{} has uninitialised socket '
                         'for remote {}'.format(self, z85_to_friendly(uid)))
            return False, err_str
        try:
            if not serialized:
                msg = self.prepare_to_send(msg)

            logger.trace('{} transmitting message {} to {} by socket {} {}'
                         .format(self, msg, z85_to_friendly(uid), socket.FD, socket.underlying))
            socket.send(msg, flags=zmq.NOBLOCK)

            if remote.isConnected or msg in self.healthMessages:
                self.metrics.add_event(self.mt_outgoing_size, len(msg))
            else:
                logger.debug('Remote {} is not connected - message will not be sent immediately.'
                             'If this problem does not resolve itself - check your firewall settings'
                             .format(z85_to_friendly(uid)))

            return True, err_str
        except zmq.Again:
            logger.warning('{} could not transmit message to {}'.format(self, z85_to_friendly(uid)))
        except InvalidMessageExceedingSizeException as ex:
            err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
            logger.warning(err_str)
        return False, err_str 
Example #28
Source Project: indy-plenum   Author: hyperledger   File: zstack.py    License: Apache License 2.0 5 votes vote down vote up
def get_monitor_events(monitor_socket, non_block=True):
        events = []
        # noinspection PyUnresolvedReferences
        flags = zmq.NOBLOCK if non_block else 0
        while True:
            try:
                # noinspection PyUnresolvedReferences
                message = recv_monitor_message(monitor_socket, flags)
                events.append(message)
            except zmq.Again:
                break
        return events 
Example #29
Source Project: indy-plenum   Author: hyperledger   File: client_message_provider.py    License: Apache License 2.0 5 votes vote down vote up
def _transmit_one_msg_throughlistener(self, msg, ident) -> Tuple[bool, Optional[str], bool]:

        def prepare_error_msg(ex):
            err_str = '{}{} got error {} while sending through listener to {}' \
                .format(CONNECTION_PREFIX, self, ex, ident)
            logger.debug(err_str)
            return err_str

        need_to_resend = False
        if isinstance(ident, str):
            ident = ident.encode()
        try:
            msg = self._prepare_to_send(msg)
            logger.trace('{} transmitting {} to {} through listener socket'.
                         format(self, msg, ident))
            self.metrics.add_event(self._mt_outgoing_size, len(msg))
            self.listener.send_multipart([ident, msg], flags=zmq.NOBLOCK)
        except InvalidMessageExceedingSizeException as ex:
            err_str = '{}Cannot transmit message. Error {}'.format(CONNECTION_PREFIX, ex)
            logger.debug(err_str)
            return False, err_str, need_to_resend
        except zmq.Again as ex:
            need_to_resend = True
            return False, prepare_error_msg(ex), need_to_resend
        except zmq.ZMQError as ex:
            need_to_resend = (ex.errno == 113)
            return False, prepare_error_msg(ex), need_to_resend
        except Exception as ex:
            return False, prepare_error_msg(ex), need_to_resend
        return True, None, need_to_resend 
Example #30
Source Project: indy-plenum   Author: hyperledger   File: test_stashed_client_messages.py    License: Apache License 2.0 5 votes vote down vote up
def test_resending_for_old_stash_msgs(tdir, tconf, looper, stacks,
                                      alpha_handler, monkeypatch):
    alpha, beta = stacks
    msg1 = {'msg': 'msg1'}
    pending_client_messages = beta._client_message_provider._pending_client_messages

    alpha.connect(name=beta.name, ha=beta.ha,
                  verKeyRaw=beta.verKeyRaw, publicKeyRaw=beta.publicKeyRaw)
    looper.runFor(0.25)

    def fake_send_multipart(msg_parts, flags=0, copy=True, track=False, **kwargs):
        raise zmq.Again

    monkeypatch.setattr(beta.listener, 'send_multipart',
                        fake_send_multipart)
    alpha.send(msg1, beta.name)
    looper.run(eventually(
        lambda messages: assertExp(messages[alpha.listener.IDENTITY] == [(0, msg1)]),
        pending_client_messages))
    monkeypatch.undo()

    beta._client_message_provider._timer.set_time(tconf.RESEND_CLIENT_MSG_TIMEOUT + 2)
    looper.run(eventually(
        lambda msg_handler: assertExp(msg_handler.received_messages == [msg1]),
        alpha_handler))
    assert not pending_client_messages