Python zmq.device() Examples

def _init_device(self):
        try:
            context = zmq.Context(1)
            # Socket facing clients
            frontend = context.socket(zmq.XREP)
            frontend.bind("tcp://*:%s" % self.port_frontend)
            # Socket facing services
            backend = context.socket(zmq.XREQ)
            backend.bind("tcp://*:%s" % self.port_backend)

            zmq.device(zmq.QUEUE, frontend, backend)
        except:
            print("bringing down zmq device")
        finally:
            pass
            frontend.close()
            backend.close()
            context.term() 

def run(self):
    dprint(1, "Config:", json.dumps(self._config))
    dprint(1, "Starting loops...")

    def heartbeat_loop():
      dprint(2, "Starting loop for 'Heartbeat'...")
      while not self._exiting:
        dprint(3, ".", end="")
        try:
          zmq.device(zmq.FORWARDER, self._heartbeat_socket, self._heartbeat_socket)
        except zmq.ZMQError as e:
          if e.errno == errno.EINTR:
            continue
          else:
            raise
        else:
          break
    hb_thread = threading.Thread(target=heartbeat_loop)
    hb_thread.daemon = True
    hb_thread.start()

    dprint(1, "Ready! Listening...")

    ioloop.IOLoop.instance().start() 

def main(N, inport, outport):
  try:
    context = zmq.Context(1)
    # Socket facing clients
    frontend = context.socket(zmq.PULL)
    frontend.sndhwm = N    
    frontend.rcvhwm = N
    #frontend.bind("tcp://127.0.0.1:5579")
    print "IN PORT: %s" % inport
    frontend.bind("tcp://127.0.0.1:" + inport)
    
    # Socket facing services
    backend = context.socket(zmq.PUSH)
    backend.sndhwm = N
    backend.rcvhwm = N
    #backend.bind("tcp://127.0.0.1:5580")
    print "OUT PORT: %s" % outport
    backend.bind("tcp://127.0.0.1:" + outport)

    zmq.device(zmq.STREAMER, frontend, backend)
  except Exception, e:
    print e
    print "bringing down zmq device" 

def test_core(self):
        """test core imports"""
        from zmq import Context
        from zmq import Socket
        from zmq import Poller
        from zmq import Frame
        from zmq import constants
        from zmq import device, proxy
        from zmq import (
            zmq_version,
            zmq_version_info,
            pyzmq_version,
            pyzmq_version_info,
        ) 

def test_devices(self):
        """test device imports"""
        import zmq.devices
        from zmq.devices import basedevice
        from zmq.devices import monitoredqueue
        from zmq.devices import monitoredqueuedevice 

def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 

def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 

def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 

def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 

def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 

def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 

def run(self):
        self.socket = self.context.socket(zmq.REP)
        c = ':' if self.transport == 'tcp' else '-'
        self.socket.bind('%s://%s' % (self.transport, self.ip) + c + str(self.port))
        while True:
            try:
                zmq.device(zmq.FORWARDER, self.socket, self.socket)
            except zmq.ZMQError as e:
                if e.errno == errno.EINTR:
                    continue
                else:
                    raise
            else:
                break 

def start_broker():
    def worker():
        context = zmq.Context(1)
        frontend = context.socket(zmq.SUB)
        frontend.bind("ipc:///tmp/keylime.verifier.ipc")

        frontend.setsockopt(zmq.SUBSCRIBE, b'')

        # Socket facing services
        backend = context.socket(zmq.PUB)
        backend.bind("tcp://*:%s" %
                     config.getint('cloud_verifier', 'revocation_notifier_port'))

        zmq.device(zmq.FORWARDER, frontend, backend)

    global broker_proc
    broker_proc = Process(target=worker)
    broker_proc.start() 

def test_core(self):
        """test core imports"""
        from zmq import Context
        from zmq import Socket
        from zmq import Poller
        from zmq import Frame
        from zmq import constants
        from zmq import device, proxy
        from zmq import (
            zmq_version,
            zmq_version_info,
            pyzmq_version,
            pyzmq_version_info,
        ) 

def test_devices(self):
        """test device imports"""
        import zmq.devices
        from zmq.devices import basedevice
        from zmq.devices import monitoredqueue
        from zmq.devices import monitoredqueuedevice 

def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 

def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 

def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 

def start(self):
        """
        Main execution.
        Instantiate workers, Accept client connections,
        distribute computation requests among workers and route computed results back to clients.
        """

        # Front facing socket to accept client connections.
        socket_front = self.zmq_context.socket(zmq.ROUTER)
        socket_front.bind(f'tcp://0.0.0.0:{self.port}')

        # Backend socket to distribute work.
        socket_back = self.zmq_context.socket(zmq.DEALER)
        socket_back.bind('inproc://backend')

        # Start workers.
        for i in range(0, self.num_workers):
            worker = Worker(self.zmq_context, self.encoder, i)
            worker.start()
            logger.info(f'[WORKER-{i}]: ready and listening!')

        # Use built in queue device to distribute requests among workers.
        # What queue device does internally is,
        #   1. Read a client's socket ID and request.
        #   2. Send socket ID and request to a worker.
        #   3. Read a client's socket ID and result from a worker.
        #   4. Route result back to the client using socket ID.
        zmq.device(zmq.QUEUE, socket_front, socket_back) 

def seq_to_emb(self, seq):
        """Helper function to calculate the embedding (molecular descriptor) for input sequnce(s)

        Args:
            seq: Single sequnces or list of sequnces to encode.
        Returns:
            Embedding of the input sequnce(s).
        """
        if isinstance(seq, str):
            seq = [seq]
        if self.use_gpu:
            emb = sequence2embedding(self.encode_model, self.hparams, seq)
        else:
            with tf.device("/cpu:0"):
                emb = sequence2embedding(self.encode_model, self.hparams, seq)
        return emb 

def emb_to_seq(self, embedding):
        """Helper function to calculate the sequnce(s) for one or multiple (concatinated)
        embedding.

        Args:
            embedding: array with n_samples x num_features.
        Returns:
            sequnce(s).
        """
        if embedding.ndim == 1:
            embedding = np.expand_dims(embedding, 0)
        if self.use_gpu:
            seq = embedding2sequence(self.decode_model, self.hparams, embedding, self.num_top, self.maximum_iterations)
        else:
            with tf.device("/cpu:0"):
                seq = embedding2sequence(self.decode_model, self.hparams, embedding, self.num_top, self.maximum_iterations)
        if len(seq) == 1:
            seq = seq[0]
        if len(seq) == 1:
            seq = seq[0]
        return seq 

def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 

def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 

def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 

def make_queue_loop(name='collect'):
    """
    Standard queue loop

    Args:
        name (None): (default = None)
    """
    assert name is not None, 'must name queue'
    queue_name = name + '_queue'
    loop_name = queue_name + '_loop'
    def queue_loop(port_dict):
        iface1, iface2 = port_dict['collect_url1'], port_dict['collect_url2']
        print = partial(ut.colorprint, color='green')
        update_proctitle(queue_name)

        with ut.Indenter('[%s] ' % (queue_name,)):
            if VERBOSE_JOBS:
                print('Init make_queue_loop: name=%r' % (name,))
            # bind the client dealer to the queue router
            rout_sock = ctx.socket(zmq.ROUTER)
            rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER')
            rout_sock.bind(iface1)
            if VERBOSE_JOBS:
                print('bind %s_url1 = %r' % (name, iface1,))
            # bind the server router to the queue dealer
            deal_sock = ctx.socket(zmq.DEALER)
            deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER')
            deal_sock.bind(iface2)
            if VERBOSE_JOBS:
                print('bind %s_url2 = %r' % (name, iface2,))
            try:
                if 1:
                    # the remainder of this function can be entirely replaced with
                    zmq.device(zmq.QUEUE, rout_sock, deal_sock)
                else:
                    # but this shows what is really going on:
                    poller = zmq.Poller()
                    poller.register(rout_sock, zmq.POLLIN)
                    poller.register(deal_sock, zmq.POLLIN)
                    while True:
                        evts = dict(poller.poll())
                        # poll() returns a list of tuples [(socket, evt), (socket, evt)]
                        # dict(poll()) turns this into {socket:evt, socket:evt}
                        if rout_sock in evts:
                            msg = rout_sock.recv_multipart()
                            # ROUTER sockets prepend the identity of the jobiface,
                            # for routing replies
                            if VERBOSE_JOBS:
                                print('ROUTER relayed %r via DEALER' % (msg,))
                            deal_sock.send_multipart(msg)
                        if deal_sock in evts:
                            msg = deal_sock.recv_multipart()
                            if VERBOSE_JOBS:
                                print('DEALER relayed %r via ROUTER' % (msg,))
                            rout_sock.send_multipart(msg)
            except KeyboardInterrupt:
                print('Caught ctrl+c in collector loop. Gracefully exiting')
            if VERBOSE_JOBS:
                print('Exiting %s' % (loop_name,))
    ut.set_funcname(queue_loop, loop_name)
    return queue_loop 

def run(self):
        '''
        Build a QUEUE device

        http://zguide.zeromq.org/py:msgqueue
        '''
        appendproctitle(self.__class__.__name__)
        context = zmq.Context()

        # Set up a router sock to receive results from minions
        router_uri = 'tcp://{0}:{1}'.format(self.opts['interface'],
                                            self.opts['ret_port'])
        router_sock = context.socket(zmq.ROUTER)
        router_sock = set_tcp_keepalive(router_sock,
                                        opts=self.opts)
        log.info(
            'Starting set up a broker ROUTER sock on {0}'.format(router_uri))
        router_sock.bind(router_uri)

        # Set up a dealer sock to send results to master ret interface
        dealer_sock = context.socket(zmq.DEALER)
        dealer_sock = set_tcp_keepalive(dealer_sock,
                                        opts=self.opts)
        self.opts['master_uri'] = 'tcp://{0}:{1}'.format(self.opts['master_ip'],
                                                      self.opts['ret_port'])
        log.info(
            'Starting set up a broker DEALER sock on {0}'.format(
                self.opts['master_uri']))
        dealer_sock.connect(self.opts['master_uri'])

        try:
            # Forward all results
            zmq.device(zmq.QUEUE, router_sock, dealer_sock)
        except KeyboardInterrupt:
            log.warn('Stopping the RetBroker')
            if router_sock.closed is False:
                router_sock.setsockopt(zmq.LINGER, 1)
                router_sock.close()
            if dealer_sock.closed is False:
                dealer_sock.setsockopt(zmq.LINGER, 1)
                dealer_sock.close()
            if context.closed is False:
                context.term()