Python zmq.device() Examples

The following are 26 code examples of zmq.device(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module zmq , or try the search function .
Example #1
Source Project: cddd   Author: jrwnter   File: inference.py    License: MIT License 6 votes vote down vote up
def _init_device(self):
        try:
            context = zmq.Context(1)
            # Socket facing clients
            frontend = context.socket(zmq.XREP)
            frontend.bind("tcp://*:%s" % self.port_frontend)
            # Socket facing services
            backend = context.socket(zmq.XREQ)
            backend.bind("tcp://*:%s" % self.port_backend)

            zmq.device(zmq.QUEUE, frontend, backend)
        except:
            print("bringing down zmq device")
        finally:
            pass
            frontend.close()
            backend.close()
            context.term() 
Example #2
Source Project: tensorlang   Author: tensorlang   File: jupyter_kernel.py    License: Apache License 2.0 6 votes vote down vote up
def run(self):
    dprint(1, "Config:", json.dumps(self._config))
    dprint(1, "Starting loops...")

    def heartbeat_loop():
      dprint(2, "Starting loop for 'Heartbeat'...")
      while not self._exiting:
        dprint(3, ".", end="")
        try:
          zmq.device(zmq.FORWARDER, self._heartbeat_socket, self._heartbeat_socket)
        except zmq.ZMQError as e:
          if e.errno == errno.EINTR:
            continue
          else:
            raise
        else:
          break
    hb_thread = threading.Thread(target=heartbeat_loop)
    hb_thread.daemon = True
    hb_thread.start()

    dprint(1, "Ready! Listening...")

    ioloop.IOLoop.instance().start() 
Example #3
Source Project: ShapeNet   Author: jonathanmasci   File: streamer_device.py    License: GNU General Public License v3.0 6 votes vote down vote up
def main(N, inport, outport):
  try:
    context = zmq.Context(1)
    # Socket facing clients
    frontend = context.socket(zmq.PULL)
    frontend.sndhwm = N    
    frontend.rcvhwm = N
    #frontend.bind("tcp://127.0.0.1:5579")
    print "IN PORT: %s" % inport
    frontend.bind("tcp://127.0.0.1:" + inport)
    
    # Socket facing services
    backend = context.socket(zmq.PUSH)
    backend.sndhwm = N
    backend.rcvhwm = N
    #backend.bind("tcp://127.0.0.1:5580")
    print "OUT PORT: %s" % outport
    backend.bind("tcp://127.0.0.1:" + outport)

    zmq.device(zmq.STREAMER, frontend, backend)
  except Exception, e:
    print e
    print "bringing down zmq device" 
Example #4
Source Project: vnpy_crypto   Author: birforce   File: test_imports.py    License: MIT License 5 votes vote down vote up
def test_core(self):
        """test core imports"""
        from zmq import Context
        from zmq import Socket
        from zmq import Poller
        from zmq import Frame
        from zmq import constants
        from zmq import device, proxy
        from zmq import (
            zmq_version,
            zmq_version_info,
            pyzmq_version,
            pyzmq_version_info,
        ) 
Example #5
Source Project: vnpy_crypto   Author: birforce   File: test_imports.py    License: MIT License 5 votes vote down vote up
def test_devices(self):
        """test device imports"""
        import zmq.devices
        from zmq.devices import basedevice
        from zmq.devices import monitoredqueue
        from zmq.devices import monitoredqueuedevice 
Example #6
Source Project: vnpy_crypto   Author: birforce   File: device.py    License: MIT License 5 votes vote down vote up
def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 
Example #7
Source Project: vnpy_crypto   Author: birforce   File: basedevice.py    License: MIT License 5 votes vote down vote up
def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 
Example #8
Source Project: vnpy_crypto   Author: birforce   File: basedevice.py    License: MIT License 5 votes vote down vote up
def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 
Example #9
Source Project: Computable   Author: ktraunmueller   File: device.py    License: MIT License 5 votes vote down vote up
def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 
Example #10
Source Project: Computable   Author: ktraunmueller   File: basedevice.py    License: MIT License 5 votes vote down vote up
def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 
Example #11
Source Project: Computable   Author: ktraunmueller   File: basedevice.py    License: MIT License 5 votes vote down vote up
def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 
Example #12
Source Project: Computable   Author: ktraunmueller   File: heartbeat.py    License: MIT License 5 votes vote down vote up
def run(self):
        self.socket = self.context.socket(zmq.REP)
        c = ':' if self.transport == 'tcp' else '-'
        self.socket.bind('%s://%s' % (self.transport, self.ip) + c + str(self.port))
        while True:
            try:
                zmq.device(zmq.FORWARDER, self.socket, self.socket)
            except zmq.ZMQError as e:
                if e.errno == errno.EINTR:
                    continue
                else:
                    raise
            else:
                break 
Example #13
Source Project: keylime   Author: keylime   File: revocation_notifier.py    License: BSD 2-Clause "Simplified" License 5 votes vote down vote up
def start_broker():
    def worker():
        context = zmq.Context(1)
        frontend = context.socket(zmq.SUB)
        frontend.bind("ipc:///tmp/keylime.verifier.ipc")

        frontend.setsockopt(zmq.SUBSCRIBE, b'')

        # Socket facing services
        backend = context.socket(zmq.PUB)
        backend.bind("tcp://*:%s" %
                     config.getint('cloud_verifier', 'revocation_notifier_port'))

        zmq.device(zmq.FORWARDER, frontend, backend)

    global broker_proc
    broker_proc = Process(target=worker)
    broker_proc.start() 
Example #14
Source Project: pySINDy   Author: luckystarufo   File: test_imports.py    License: MIT License 5 votes vote down vote up
def test_core(self):
        """test core imports"""
        from zmq import Context
        from zmq import Socket
        from zmq import Poller
        from zmq import Frame
        from zmq import constants
        from zmq import device, proxy
        from zmq import (
            zmq_version,
            zmq_version_info,
            pyzmq_version,
            pyzmq_version_info,
        ) 
Example #15
Source Project: pySINDy   Author: luckystarufo   File: test_imports.py    License: MIT License 5 votes vote down vote up
def test_devices(self):
        """test device imports"""
        import zmq.devices
        from zmq.devices import basedevice
        from zmq.devices import monitoredqueue
        from zmq.devices import monitoredqueuedevice 
Example #16
Source Project: pySINDy   Author: luckystarufo   File: device.py    License: MIT License 5 votes vote down vote up
def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 
Example #17
Source Project: pySINDy   Author: luckystarufo   File: basedevice.py    License: MIT License 5 votes vote down vote up
def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 
Example #18
Source Project: pySINDy   Author: luckystarufo   File: basedevice.py    License: MIT License 5 votes vote down vote up
def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 
Example #19
Source Project: embedding-as-service   Author: amansrivastava17   File: __init__.py    License: MIT License 5 votes vote down vote up
def start(self):
        """
        Main execution.
        Instantiate workers, Accept client connections,
        distribute computation requests among workers and route computed results back to clients.
        """

        # Front facing socket to accept client connections.
        socket_front = self.zmq_context.socket(zmq.ROUTER)
        socket_front.bind(f'tcp://0.0.0.0:{self.port}')

        # Backend socket to distribute work.
        socket_back = self.zmq_context.socket(zmq.DEALER)
        socket_back.bind('inproc://backend')

        # Start workers.
        for i in range(0, self.num_workers):
            worker = Worker(self.zmq_context, self.encoder, i)
            worker.start()
            logger.info(f'[WORKER-{i}]: ready and listening!')

        # Use built in queue device to distribute requests among workers.
        # What queue device does internally is,
        #   1. Read a client's socket ID and request.
        #   2. Send socket ID and request to a worker.
        #   3. Read a client's socket ID and result from a worker.
        #   4. Route result back to the client using socket ID.
        zmq.device(zmq.QUEUE, socket_front, socket_back) 
Example #20
Source Project: cddd   Author: jrwnter   File: inference.py    License: MIT License 5 votes vote down vote up
def seq_to_emb(self, seq):
        """Helper function to calculate the embedding (molecular descriptor) for input sequnce(s)

        Args:
            seq: Single sequnces or list of sequnces to encode.
        Returns:
            Embedding of the input sequnce(s).
        """
        if isinstance(seq, str):
            seq = [seq]
        if self.use_gpu:
            emb = sequence2embedding(self.encode_model, self.hparams, seq)
        else:
            with tf.device("/cpu:0"):
                emb = sequence2embedding(self.encode_model, self.hparams, seq)
        return emb 
Example #21
Source Project: cddd   Author: jrwnter   File: inference.py    License: MIT License 5 votes vote down vote up
def emb_to_seq(self, embedding):
        """Helper function to calculate the sequnce(s) for one or multiple (concatinated)
        embedding.

        Args:
            embedding: array with n_samples x num_features.
        Returns:
            sequnce(s).
        """
        if embedding.ndim == 1:
            embedding = np.expand_dims(embedding, 0)
        if self.use_gpu:
            seq = embedding2sequence(self.decode_model, self.hparams, embedding, self.num_top, self.maximum_iterations)
        else:
            with tf.device("/cpu:0"):
                seq = embedding2sequence(self.decode_model, self.hparams, embedding, self.num_top, self.maximum_iterations)
        if len(seq) == 1:
            seq = seq[0]
        if len(seq) == 1:
            seq = seq[0]
        return seq 
Example #22
Source Project: Carnets   Author: holzschu   File: device.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def device(device_type, isocket, osocket):
    """Start a zeromq device (gevent-compatible).
    
    Unlike the true zmq.device, this does not release the GIL.

    Parameters
    ----------
    device_type : (QUEUE, FORWARDER, STREAMER)
        The type of device to start (ignored).
    isocket : Socket
        The Socket instance for the incoming traffic.
    osocket : Socket
        The Socket instance for the outbound traffic.
    """
    p = Poller()
    if osocket == -1:
        osocket = isocket
    p.register(isocket, zmq.POLLIN)
    p.register(osocket, zmq.POLLIN)
    
    while True:
        events = dict(p.poll())
        if isocket in events:
            osocket.send_multipart(isocket.recv_multipart())
        if osocket in events:
            isocket.send_multipart(osocket.recv_multipart()) 
Example #23
Source Project: Carnets   Author: holzschu   File: basedevice.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def run_device(self):
        """The runner method.

        Do not call me directly, instead call ``self.start()``, just like a Thread.
        """
        ins,outs = self._setup_sockets()
        device(self.device_type, ins, outs) 
Example #24
Source Project: Carnets   Author: holzschu   File: basedevice.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def start(self):
        """Start the device. Override me in subclass for other launchers."""
        return self.run() 
Example #25
Source Project: ibeis   Author: Erotemic   File: job_engine.py    License: Apache License 2.0 4 votes vote down vote up
def make_queue_loop(name='collect'):
    """
    Standard queue loop

    Args:
        name (None): (default = None)
    """
    assert name is not None, 'must name queue'
    queue_name = name + '_queue'
    loop_name = queue_name + '_loop'
    def queue_loop(port_dict):
        iface1, iface2 = port_dict['collect_url1'], port_dict['collect_url2']
        print = partial(ut.colorprint, color='green')
        update_proctitle(queue_name)

        with ut.Indenter('[%s] ' % (queue_name,)):
            if VERBOSE_JOBS:
                print('Init make_queue_loop: name=%r' % (name,))
            # bind the client dealer to the queue router
            rout_sock = ctx.socket(zmq.ROUTER)
            rout_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'ROUTER')
            rout_sock.bind(iface1)
            if VERBOSE_JOBS:
                print('bind %s_url1 = %r' % (name, iface1,))
            # bind the server router to the queue dealer
            deal_sock = ctx.socket(zmq.DEALER)
            deal_sock.setsockopt_string(zmq.IDENTITY, 'queue.' + name + '.' + 'DEALER')
            deal_sock.bind(iface2)
            if VERBOSE_JOBS:
                print('bind %s_url2 = %r' % (name, iface2,))
            try:
                if 1:
                    # the remainder of this function can be entirely replaced with
                    zmq.device(zmq.QUEUE, rout_sock, deal_sock)
                else:
                    # but this shows what is really going on:
                    poller = zmq.Poller()
                    poller.register(rout_sock, zmq.POLLIN)
                    poller.register(deal_sock, zmq.POLLIN)
                    while True:
                        evts = dict(poller.poll())
                        # poll() returns a list of tuples [(socket, evt), (socket, evt)]
                        # dict(poll()) turns this into {socket:evt, socket:evt}
                        if rout_sock in evts:
                            msg = rout_sock.recv_multipart()
                            # ROUTER sockets prepend the identity of the jobiface,
                            # for routing replies
                            if VERBOSE_JOBS:
                                print('ROUTER relayed %r via DEALER' % (msg,))
                            deal_sock.send_multipart(msg)
                        if deal_sock in evts:
                            msg = deal_sock.recv_multipart()
                            if VERBOSE_JOBS:
                                print('DEALER relayed %r via ROUTER' % (msg,))
                            rout_sock.send_multipart(msg)
            except KeyboardInterrupt:
                print('Caught ctrl+c in collector loop. Gracefully exiting')
            if VERBOSE_JOBS:
                print('Exiting %s' % (loop_name,))
    ut.set_funcname(queue_loop, loop_name)
    return queue_loop 
Example #26
Source Project: salt-broker   Author: pengyao   File: broker.py    License: Apache License 2.0 4 votes vote down vote up
def run(self):
        '''
        Build a QUEUE device

        http://zguide.zeromq.org/py:msgqueue
        '''
        appendproctitle(self.__class__.__name__)
        context = zmq.Context()

        # Set up a router sock to receive results from minions
        router_uri = 'tcp://{0}:{1}'.format(self.opts['interface'],
                                            self.opts['ret_port'])
        router_sock = context.socket(zmq.ROUTER)
        router_sock = set_tcp_keepalive(router_sock,
                                        opts=self.opts)
        log.info(
            'Starting set up a broker ROUTER sock on {0}'.format(router_uri))
        router_sock.bind(router_uri)

        # Set up a dealer sock to send results to master ret interface
        dealer_sock = context.socket(zmq.DEALER)
        dealer_sock = set_tcp_keepalive(dealer_sock,
                                        opts=self.opts)
        self.opts['master_uri'] = 'tcp://{0}:{1}'.format(self.opts['master_ip'],
                                                      self.opts['ret_port'])
        log.info(
            'Starting set up a broker DEALER sock on {0}'.format(
                self.opts['master_uri']))
        dealer_sock.connect(self.opts['master_uri'])

        try:
            # Forward all results
            zmq.device(zmq.QUEUE, router_sock, dealer_sock)
        except KeyboardInterrupt:
            log.warn('Stopping the RetBroker')
            if router_sock.closed is False:
                router_sock.setsockopt(zmq.LINGER, 1)
                router_sock.close()
            if dealer_sock.closed is False:
                dealer_sock.setsockopt(zmq.LINGER, 1)
                dealer_sock.close()
            if context.closed is False:
                context.term()