Python zmq.DEALER Examples

def test_hwm(self):
        zmq3 = zmq.zmq_version_info()[0] >= 3
        for stype in (zmq.PUB, zmq.ROUTER, zmq.SUB, zmq.REQ, zmq.DEALER):
            s = self.context.socket(stype)
            s.hwm = 100
            self.assertEqual(s.hwm, 100)
            if zmq3:
                try:
                    self.assertEqual(s.sndhwm, 100)
                except AttributeError:
                    pass
                try:
                    self.assertEqual(s.rcvhwm, 100)
                except AttributeError:
                    pass
            s.close() 

def run(self):
        worker = self.context.socket(zmq.DEALER)
        worker.connect('inproc://backend')
        tprint('Worker started')
        while True:
            ident, task = worker.recv_multipart()
            task = pickle.loads(task)
            tprint('Worker received %s from %s' % (task, ident))
            ok = True
            try:
                if task.encode:
                    self.process_encode(task.infile, task.outfile)
                else:
                    self.process_decode(task.infile, task.outfile)
                worker.send_multipart([ident, str(True)])
            except Exception as e:
                outf = io.open(task.outfile, 'w', encoding='utf-8')
                outf.write("%s\n" % unicode(e))
                outf.close()
                worker.send_multipart([ident, str(False)])

        worker.close() 

def reset_state(self):
        _MultiProcessZMQDataFlow.reset_state(self)
        _ParallelMapData.reset_state(self)
        self._guard = DataFlowReentrantGuard()

        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.set_hwm(self._buffer_size * 2)
        pipename = _get_pipe_name('dataflow-map')
        _bind_guard(self.socket, pipename)

        self._proc_ids = [u'{}'.format(k).encode('utf-8') for k in range(self.num_proc)]
        worker_hwm = int(self._buffer_size * 2 // self.num_proc)
        self._procs = [self._create_worker(self._proc_ids[k], pipename, worker_hwm)
                       for k in range(self.num_proc)]

        self._start_processes()
        self._fill_buffer()     # pre-fill the bufer 

def __init__(self, actnode_topics=b''):
        ctx = zmq.Context.instance()
        self.event_io = ctx.socket(zmq.DEALER)
        self.stream_in = ctx.socket(zmq.SUB)
        self.poller = zmq.Poller()
        self.host_id = b''
        self.client_id = b'\x00' + os.urandom(4)
        self.sender_id = b''
        self.servers = dict()
        self.act = b''
        self.actroute = []
        self.acttopics = actnode_topics
        self.discovery = None

        # Signals
        self.nodes_changed = Signal()
        self.server_discovered = Signal()
        self.signal_quit = Signal()
        self.event_received = Signal()
        self.stream_received = Signal()

        # Tell bluesky that this client will manage the network I/O
        bluesky.net = self 

def reset_state(self):
        _MultiProcessZMQDataFlow.reset_state(self)
        _ParallelMapData.reset_state(self)

        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.set_hwm(self._buffer_size * 2)
        pipename = _get_pipe_name('dataflow-map')
        _bind_guard(self.socket, pipename)

        self._proc_ids = [u'{}'.format(k).encode('utf-8') for k in range(self.nr_proc)]
        worker_hwm = int(self._buffer_size * 2 // self.nr_proc)
        self._procs = [MultiProcessMapDataZMQ._Worker(
            self._proc_ids[k], self.map_func, pipename, worker_hwm)
            for k in range(self.nr_proc)]

        self._start_processes()
        self._fill_buffer()     # pre-fill the bufer 

def __init__(self, ip_address, port_range):
        """
        Parameters
        ----------

        ip_address: str
           IP address of the client (where Parsl runs)
        port_range: tuple(int, int)
           Port range for the comms between client and interchange

        """
        self.context = zmq.Context()
        self.zmq_socket = self.context.socket(zmq.DEALER)
        self.zmq_socket.set_hwm(0)
        self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
                                                        min_port=port_range[0],
                                                        max_port=port_range[1]) 

def __init__(self, ip_address, port_range):
        """
        Parameters
        ----------

        ip_address: str
           IP address of the client (where Parsl runs)
        port_range: tuple(int, int)
           Port range for the comms between client and interchange

        """
        self.context = zmq.Context()
        self.zmq_socket = self.context.socket(zmq.DEALER)
        self.zmq_socket.set_hwm(0)
        self.port = self.zmq_socket.bind_to_random_port("tcp://{}".format(ip_address),
                                                        min_port=port_range[0],
                                                        max_port=port_range[1])
        self.poller = zmq.Poller()
        self.poller.register(self.zmq_socket, zmq.POLLOUT) 

def __init__(self, ip_address, port_range):
        """
        Parameters
        ----------

        ip_address: str
           IP address of the client (where Parsl runs)
        port_range: tuple(int, int)
           Port range for the comms between client and interchange

        """
        self.context = zmq.Context()
        self.results_receiver = self.context.socket(zmq.DEALER)
        self.results_receiver.set_hwm(0)
        self.port = self.results_receiver.bind_to_random_port("tcp://{}".format(ip_address),
                                                              min_port=port_range[0],
                                                              max_port=port_range[1]) 

def reconnect_to_broker(self):
        """Connect or reconnect to broker"""
        if self.worker:
            self.poller.unregister(self.worker)
            self.worker.close()
        self.worker = self.ctx.socket(zmq.DEALER)
        self.worker.linger = 0
        self.worker.connect(self.broker)
        self.poller.register(self.worker, zmq.POLLIN)
        if self.verbose:
            logging.info("I: connecting to broker at %s…", self.broker)

        # Register service with broker
        self.send_to_broker(MDP.W_READY, pickle.dumps(self.service), [])

        # If liveness hits zero, queue is considered disconnected
        self.liveness = self.HEARTBEAT_LIVENESS
        self.heartbeat_at = time.time() + 1e-3 * self.heartbeat 

def __init__(self, url):
        self._url = url

        self._ctx = Context.instance()
        self._socket = self._ctx.socket(zmq.DEALER)
        self._socket.identity = uuid.uuid4().hex.encode()[0:16]

        self._msg_router = _MessageRouter()
        self._receiver = _Receiver(self._socket, self._msg_router)
        self._sender = _Sender(self._socket, self._msg_router)

        self._connection_state_listeners = {}

        self._recv_task = None

        # Monitoring properties
        self._monitor_sock = None
        self._monitor_fd = None
        self._monitor_task = None 

def init_networking(self):
        """
        Creates socket, connects to specified port on localhost,
        and starts the :meth:`~Net_Node.threaded_loop` as a daemon thread.
        """
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.identity = self.id
        #self.sock.probe_router = 1

        # net nodes are local only
        self.sock.connect('tcp://localhost:{}'.format(self.port))

        # wrap in zmqstreams and start loop thread
        self.sock = ZMQStream(self.sock, self.loop)
        self.sock.on_recv(self.handle_listen)

        self.loop_thread = threading.Thread(target=self.threaded_loop)
        self.loop_thread.daemon = True
        self.loop_thread.start()

        self.repeat_thread = threading.Thread(target=self.repeat)
        self.repeat_thread.daemon = True
        self.repeat_thread.start()

        #self.connected = True 

def run(self):
        """
        Main execution.
        Returns:
        """
        # Socket to communicate with front facing server.
        socket = self.zmq_context.socket(zmq.DEALER)
        socket.connect('inproc://backend')

        while True:
            # First string recieved is socket ID of client
            client_id = socket.recv()
            request = socket.recv()
            # print('Worker ID - %s. Recieved computation request.' % (self.worker_id))

            result = self.compute(request)

            # print('Worker ID - %s. Sending computed result back.' % (self.worker_id))

            # For successful routing of result to correct client, the socket ID of client should be sent first.
            socket.send(client_id, zmq.SNDMORE)
            socket.send_string(result) 

def _do_heartbeat(self):
        while True:
            try:
                if self._socket.getsockopt(zmq.TYPE) == zmq.ROUTER:
                    yield from self._do_router_heartbeat()
                elif self._socket.getsockopt(zmq.TYPE) == zmq.DEALER:
                    yield from self._do_dealer_heartbeat()
                yield from asyncio.sleep(self._heartbeat_interval,
                                         loop=self._event_loop)
            except CancelledError:  # pylint: disable=try-except-raise
                # The concurrent.futures.CancelledError is caught by asyncio
                # when the Task associated with the coroutine is cancelled.
                # The raise is required to stop this component.
                raise
            except Exception as e:  # pylint: disable=broad-except
                LOGGER.exception(
                    "An error occurred while sending heartbeat: %s", e) 

def reset_state(self):
        _MultiProcessZMQDataFlow.reset_state(self)
        _ParallelMapData.reset_state(self)
        self._guard = DataFlowReentrantGuard()

        self.context = zmq.Context()
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.set_hwm(self._buffer_size * 2)
        pipename = _get_pipe_name('dataflow-map')
        _bind_guard(self.socket, pipename)

        self._proc_ids = [u'{}'.format(k).encode('utf-8') for k in range(self.num_proc)]
        worker_hwm = int(self._buffer_size * 2 // self.num_proc)
        self._procs = [self._create_worker(self._proc_ids[k], pipename, worker_hwm)
                       for k in range(self.num_proc)]

        self._start_processes()
        self._fill_buffer()     # pre-fill the bufer 

def run(self):
        player = self._build_player()
        context = zmq.Context()
        c2s_socket = context.socket(zmq.PUSH)
        c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
        c2s_socket.set_hwm(2)
        c2s_socket.connect(self.c2s)

        s2c_socket = context.socket(zmq.DEALER)
        s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
        #s2c_socket.set_hwm(5)
        s2c_socket.connect(self.s2c)

        state = player.current_state()
        reward, isOver = 0, False
        while True:
            c2s_socket.send(dumps(
                (self.identity, state, reward, isOver)),
                copy=False)
            action = loads(s2c_socket.recv(copy=False).bytes)
            reward, isOver = player.action(action)
            state = player.current_state()

# compatibility 

def test_custom_serialize_error(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            with pytest.raises(TypeError):
                yield a.send_serialized(json, json.dumps)

            yield a.send(b'not json')
            with pytest.raises(TypeError):
                recvd = yield b.recv_serialized(json.loads)
        self.loop.run_sync(test) 

def bounce(self, server, client, test_metadata=True):
        msg = [os.urandom(64), os.urandom(64)]
        client.send_multipart(msg)
        frames = self.recv_multipart(server, copy=False)
        recvd = list(map(lambda x: x.bytes, frames))

        try:
            if test_metadata and not PYPY:
                for frame in frames:
                    self.assertEqual(frame.get('User-Id'), 'anonymous')
                    self.assertEqual(frame.get('Hello'), 'World')
                    self.assertEqual(frame['Socket-Type'], 'DEALER')
        except zmq.ZMQVersionError:
            pass

        self.assertEqual(recvd, msg)
        server.send_multipart(recvd)
        msg2 = self.recv_multipart(client)
        self.assertEqual(msg2, msg) 

def test_plain(self):
        """test PLAIN authentication"""
        server = self.socket(zmq.DEALER)
        server.identity = b'IDENT'
        client = self.socket(zmq.DEALER)
        self.assertEqual(client.plain_username, b'')
        self.assertEqual(client.plain_password, b'')
        client.plain_username = USER
        client.plain_password = PASS
        self.assertEqual(client.getsockopt(zmq.PLAIN_USERNAME), USER)
        self.assertEqual(client.getsockopt(zmq.PLAIN_PASSWORD), PASS)
        self.assertEqual(client.plain_server, 0)
        self.assertEqual(server.plain_server, 0)
        server.plain_server = True
        self.assertEqual(server.mechanism, zmq.PLAIN)
        self.assertEqual(client.mechanism, zmq.PLAIN)
        
        assert not client.plain_server
        assert server.plain_server

        with self.zap():
            iface = 'tcp://127.0.0.1'
            port = server.bind_to_random_port(iface)
            client.connect("%s:%i" % (iface, port))
            self.bounce(server, client) 

def bounce(self, server, client, test_metadata=True):
        msg = [os.urandom(64), os.urandom(64)]
        client.send_multipart(msg)
        frames = self.recv_multipart(server, copy=False)
        recvd = list(map(lambda x: x.bytes, frames))

        try:
            if test_metadata and not PYPY:
                for frame in frames:
                    self.assertEqual(frame.get('User-Id'), 'anonymous')
                    self.assertEqual(frame.get('Hello'), 'World')
                    self.assertEqual(frame['Socket-Type'], 'DEALER')
        except zmq.ZMQVersionError:
            pass

        self.assertEqual(recvd, msg)
        server.send_multipart(recvd)
        msg2 = self.recv_multipart(client)
        self.assertEqual(msg2, msg) 

def test_custom_serialize_error(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.DEALER, zmq.ROUTER)

            msg = {
                'content': {
                    'a': 5,
                    'b': 'bee',
                }
            }
            with pytest.raises(TypeError):
                yield a.send_serialized(json, json.dumps)

            yield a.send(b'not json')
            with pytest.raises(TypeError):
                recvd = yield b.recv_serialized(json.loads)
        self.loop.run_sync(test) 

def register(self):
        """send the registration_request"""

        self.log.info("Registering with controller at %s"%self.url)
        ctx = self.context
        connect,maybe_tunnel = self.init_connector()
        reg = ctx.socket(zmq.DEALER)
        reg.setsockopt(zmq.IDENTITY, self.bident)
        connect(reg, self.url)
        self.registrar = zmqstream.ZMQStream(reg, self.loop)


        content = dict(uuid=self.ident)
        self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
        # print (self.session.key)
        self.session.send(self.registrar, "registration_request", content=content) 

def __init__(self, in_addr, out_addr, mon_addr=None, in_type=zmq.SUB, out_type=zmq.DEALER, mon_type=zmq.PUB, heart_id=None):
        if mon_addr is None:
            self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        else:
            self.device = ThreadMonitoredQueue(in_type, out_type, mon_type, in_prefix=b"", out_prefix=b"")
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context
        
        self.device.daemon=True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if mon_addr is not None:
            self.device.connect_mon(mon_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id 

def initialize_client_thread(jobiface):
        """
        Creates a ZMQ object in this thread. This talks to background processes.
        """
        print = partial(ut.colorprint, color='blue')
        if jobiface.verbose:
            print('Initializing JobInterface')
        jobiface.engine_deal_sock = ctx.socket(zmq.DEALER)
        jobiface.engine_deal_sock.setsockopt_string(zmq.IDENTITY,
                                                    'client%s.engine.DEALER' %
                                                    (jobiface.id_,))
        jobiface.engine_deal_sock.connect(jobiface.port_dict['engine_url1'])
        if jobiface.verbose:
            print('connect engine_url1 = %r' % (jobiface.port_dict['engine_url1'],))

        jobiface.collect_deal_sock = ctx.socket(zmq.DEALER)
        jobiface.collect_deal_sock.setsockopt_string(zmq.IDENTITY,
                                                     'client%s.collect.DEALER'
                                                     % (jobiface.id_,))
        jobiface.collect_deal_sock.connect(jobiface.port_dict['collect_url1'])
        if jobiface.verbose:
            print('connect collect_url1 = %r' % (jobiface.port_dict['collect_url1'],)) 

def __init__(self, port, pipeline=100, host='localhost', log_file=None):
        """Create a new ZMQDealer object.
        """
        context = zmq.Context.instance()
        # noinspection PyUnresolvedReferences
        self.socket = context.socket(zmq.DEALER)
        self.socket.hwm = pipeline
        self.socket.connect('tcp://%s:%d' % (host, port))
        self._log_file = log_file
        self.poller = zmq.Poller()
        # noinspection PyUnresolvedReferences
        self.poller.register(self.socket, zmq.POLLIN)

        if self._log_file is not None:
            self._log_file = Path(self._log_file).resolve()
            # If log file directory does not exists, create it
            log_dir: Path = self._log_file.parent
            log_dir.mkdir(parents=True, exist_ok=True)
            # time stamp the file
            now = datetime.now()
            time_stamp = now.strftime('%Y%m%d_%H%M%S%f')
            ext = self._log_file.suffix
            self._log_file = str(log_dir / f'{self._log_file.stem}_{time_stamp}{ext}') 

def sendConnect(self, data):
        """Send a CONNECT command to the broker
            :param data: List of other broker main socket URL"""
        # Imported dynamically - Not used if only one broker
        if self.backend == 'ZMQ':
            import zmq
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.DEALER)
            self.socket.setsockopt(zmq.IDENTITY, b'launcher')
            self.socket.connect(
                "tcp://127.0.0.1:{port}".format(
                    port=self.brokerPort,
                )
            )
            self.socket.send_multipart([b"CONNECT",
                                        pickle.dumps(data,
                                                     pickle.HIGHEST_PROTOCOL)])
        else:
            # TODO
            pass 

def sendConnect(self, data):
        """Send a CONNECT command to the broker
            :param data: List of other broker main socket URL"""
        # Imported dynamically - Not used if only one broker
        if self.backend == 'ZMQ':
            import zmq
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.DEALER)
            if sys.version_info < (3,):
                self.socket.setsockopt_string(zmq.IDENTITY, unicode('launcher'))
            else:
                self.socket.setsockopt_string(zmq.IDENTITY, 'launcher')
            self.socket.connect(
                "tcp://{hostname}:{port}".format(
                    port=self.brokerPort,
                    hostname = self.hostname
                )
            )
            self.socket.send_multipart([b"CONNECT",
                                        pickle.dumps(data,
                                                     pickle.HIGHEST_PROTOCOL)])
        else:
            # TODO
            pass 

def _do_connect(self):

        client_id = str(random.randint(1000000, 100000000))

        socket = self._ctx.socket(zmq.DEALER)
        identity = (client_id) + '$' + str(random.randint(1000000, 1000000000))
        identity = identity.encode('utf-8')
        socket.setsockopt(zmq.IDENTITY, identity)
        socket.setsockopt(zmq.RCVTIMEO, 500)
        socket.setsockopt(zmq.SNDTIMEO, 500)
        socket.setsockopt(zmq.LINGER, 0)
        socket.connect(self._addr)

        return socket 

def test_hwm(self):
        zmq3 = zmq.zmq_version_info()[0] >= 3
        for stype in (zmq.PUB, zmq.ROUTER, zmq.SUB, zmq.REQ, zmq.DEALER):
            s = self.context.socket(stype)
            s.hwm = 100
            self.assertEqual(s.hwm, 100)
            if zmq3:
                try:
                    self.assertEqual(s.sndhwm, 100)
                except AttributeError:
                    pass
                try:
                    self.assertEqual(s.rcvhwm, 100)
                except AttributeError:
                    pass
            s.close() 

def test_plain(self):
        """test PLAIN authentication"""
        server = self.socket(zmq.DEALER)
        server.identity = b'IDENT'
        client = self.socket(zmq.DEALER)
        self.assertEqual(client.plain_username, b'')
        self.assertEqual(client.plain_password, b'')
        client.plain_username = USER
        client.plain_password = PASS
        self.assertEqual(client.getsockopt(zmq.PLAIN_USERNAME), USER)
        self.assertEqual(client.getsockopt(zmq.PLAIN_PASSWORD), PASS)
        self.assertEqual(client.plain_server, 0)
        self.assertEqual(server.plain_server, 0)
        server.plain_server = True
        self.assertEqual(server.mechanism, zmq.PLAIN)
        self.assertEqual(client.mechanism, zmq.PLAIN)
        
        assert not client.plain_server
        assert server.plain_server

        with self.zap():
            iface = 'tcp://127.0.0.1'
            port = server.bind_to_random_port(iface)
            client.connect("%s:%i" % (iface, port))
            self.bounce(server, client) 

def run(self):
        ''' Implementation of the I/O loop. '''
        ctx = zmq.Context.instance()
        fe_event = ctx.socket(zmq.DEALER)
        fe_stream = ctx.socket(zmq.PUB)
        be_event = ctx.socket(zmq.PAIR)
        be_stream = ctx.socket(zmq.PAIR)
        fe_event.connect('tcp://localhost:10000')
        fe_stream.connect('tcp://localhost:10001')

        be_event.connect('inproc://event')
        be_stream.connect('inproc://stream')
        poller = zmq.Poller()
        poller.register(fe_event, zmq.POLLIN)
        poller.register(be_event, zmq.POLLIN)
        poller.register(be_stream, zmq.POLLIN)

        while True:
            try:
                poll_socks = dict(poller.poll(None))
            except zmq.ZMQError:
                break  # interrupted

            if poll_socks.get(fe_event) == zmq.POLLIN:
                be_event.send_multipart(fe_event.recv_multipart())
            if poll_socks.get(be_event) == zmq.POLLIN:
                msg = be_event.recv_multipart()
                if msg[0] == b'QUIT':
                    break
                fe_event.send_multipart(msg)
            if poll_socks.get(be_stream) == zmq.POLLIN:
                fe_stream.send_multipart(be_stream.recv_multipart())