Python zmq.PUSH Examples

The following are 30 code examples of zmq.PUSH(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module zmq , or try the search function .
Example #1
Source Project: dataflow   Author: tensorpack   File: parallel_map.py    License: Apache License 2.0 8 votes vote down vote up
def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()

            # recv jobs
            socket = ctx.socket(zmq.PULL)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm * self.batch_size)
            socket.connect(self.input_pipe)

            # send results
            out_socket = ctx.socket(zmq.PUSH)
            out_socket.set_hwm(max(self.hwm, 5))
            out_socket.connect(self.result_pipe)

            batch = []
            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                if dp is not None:
                    batch.append(dp)
                    if len(batch) == self.batch_size:
                        dp = BatchData.aggregate_batch(batch)
                        out_socket.send(dumps(dp), copy=False)
                        del batch[:] 
Example #2
Source Project: dataflow   Author: tensorpack   File: parallel.py    License: Apache License 2.0 6 votes vote down vote up
def run(self):
            enable_death_signal(_warn=self.idx == 0)
            self.ds.reset_state()
            itr = _repeat_iter(lambda: self.ds)

            context = zmq.Context()
            socket = context.socket(zmq.PUSH)
            socket.set_hwm(self.hwm)
            socket.connect(self.conn_name)
            try:
                while True:
                    try:
                        dp = next(itr)
                        socket.send(dumps(dp), copy=False)
                    except Exception:
                        dp = _ExceptionWrapper(sys.exc_info()).pack()
                        socket.send(dumps(dp), copy=False)
                        raise
            # sigint could still propagate here, e.g. when nested
            except KeyboardInterrupt:
                pass
            finally:
                socket.close(0)
                context.destroy(0) 
Example #3
Source Project: Bert-TextClassification   Author: a414351664   File: server.py    License: MIT License 6 votes vote down vote up
def run(self):
        context = zmq.Context()
        receiver = context.socket(zmq.PULL)
        receiver.connect(self.worker_address)

        sink = context.socket(zmq.PUSH)
        sink.connect(self.sink_address)

        input_fn = self.input_fn_builder(receiver)

        self.logger.info('ready and listening')
        start_t = time.perf_counter()
        for r in self.estimator.predict(input_fn, yield_single_examples=False):
            # logger.info('new result!')
            send_ndarray(sink, r['client_id'], r['encodes'])
            time_used = time.perf_counter() - start_t
            start_t = time.perf_counter()
            self.logger.info('job %s\tsamples: %4d\tdone: %.2fs' %
                             (r['client_id'], r['encodes'].shape[0], time_used))

        receiver.close()
        sink.close()
        context.term()
        self.logger.info('terminated!') 
Example #4
Source Project: vnpy_crypto   Author: birforce   File: decorators.py    License: MIT License 6 votes vote down vote up
def socket(*args, **kwargs):
    '''Decorator for adding a socket to a function.
    
    Usage::
    
        @socket(zmq.PUSH)
        def foo(push):
            ...
    
    .. versionadded:: 15.3

    :param str name: the keyword argument passed to decorated function
    :param str context_name: the keyword only argument to identify context
                             object
    '''
    return _SocketDecorator()(*args, **kwargs) 
Example #5
Source Project: vnpy_crypto   Author: birforce   File: test_socket.py    License: MIT License 6 votes vote down vote up
def test_shadow(self):
        p = self.socket(zmq.PUSH)
        p.bind("tcp://127.0.0.1:5555")
        p2 = zmq.Socket.shadow(p.underlying)
        self.assertEqual(p.underlying, p2.underlying)
        s = self.socket(zmq.PULL)
        s2 = zmq.Socket.shadow(s.underlying)
        self.assertNotEqual(s.underlying, p.underlying)
        self.assertEqual(s.underlying, s2.underlying)
        s2.connect("tcp://127.0.0.1:5555")
        sent = b'hi'
        p2.send(sent)
        rcvd = self.recv(s2)
        self.assertEqual(rcvd, sent) 
Example #6
Source Project: vnpy_crypto   Author: birforce   File: _test_asyncio.py    License: MIT License 6 votes vote down vote up
def test_recv_json_cancelled(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield from asyncio.sleep(0)
            obj = dict(a=5)
            yield from a.send_json(obj)
            with pytest.raises(CancelledError):
                recvd = yield from f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield from b.poll(timeout=5)
            assert events
            yield from asyncio.sleep(0)
            # make sure cancelled recv didn't eat up event
            f = b.recv_json()
            recvd = yield from asyncio.wait_for(f, timeout=5)
            assert recvd == obj
        self.loop.run_until_complete(test()) 
Example #7
Source Project: vnpy_crypto   Author: birforce   File: _test_asyncio.py    License: MIT License 6 votes vote down vote up
def test_poll_base_socket(self):
        @asyncio.coroutine
        def test():
            ctx = zmq.Context()
            url = 'inproc://test'
            a = ctx.socket(zmq.PUSH)
            b = ctx.socket(zmq.PULL)
            self.sockets.extend([a, b])
            a.bind(url)
            b.connect(url)

            poller = zaio.Poller()
            poller.register(b, zmq.POLLIN)

            f = poller.poll(timeout=1000)
            assert not f.done()
            a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, [(b, zmq.POLLIN)])
            recvd = b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 
Example #8
Source Project: vnpy_crypto   Author: birforce   File: test_context.py    License: MIT License 6 votes vote down vote up
def test_gc(self):
        """test close&term by garbage collection alone"""
        if PYPY:
            raise SkipTest("GC doesn't work ")
            
        # test credit @dln (GH #137):
        def gcf():
            def inner():
                ctx = self.Context()
                s = ctx.socket(zmq.PUSH)
            inner()
            gc.collect()
        t = Thread(target=gcf)
        t.start()
        t.join(timeout=1)
        self.assertFalse(t.is_alive(), "Garbage collection should have cleaned up context") 
Example #9
Source Project: vnpy_crypto   Author: birforce   File: test_context.py    License: MIT License 6 votes vote down vote up
def test_shadow_pyczmq(self):
        try:
            from pyczmq import zctx, zsocket, zstr
        except Exception:
            raise SkipTest("Requires pyczmq")
        
        ctx = zctx.new()
        a = zsocket.new(ctx, zmq.PUSH)
        zsocket.bind(a, "inproc://a")
        ctx2 = self.Context.shadow_pyczmq(ctx)
        b = ctx2.socket(zmq.PULL)
        b.connect("inproc://a")
        zstr.send(a, b'hi')
        rcvd = self.recv(b)
        self.assertEqual(rcvd, b'hi')
        b.close() 
Example #10
Source Project: vnpy_crypto   Author: birforce   File: test_decorators.py    License: MIT License 6 votes vote down vote up
def test_multi_skts():
    @socket(zmq.PUB)
    @socket(zmq.SUB)
    @socket(zmq.PUSH)
    def test(pub, sub, push):
        assert isinstance(pub, zmq.Socket), pub
        assert isinstance(sub, zmq.Socket), sub
        assert isinstance(push, zmq.Socket), push

        assert pub.context is zmq.Context.instance()
        assert sub.context is zmq.Context.instance()
        assert push.context is zmq.Context.instance()

        assert pub.type == zmq.PUB
        assert sub.type == zmq.SUB
        assert push.type == zmq.PUSH
    test() 
Example #11
Source Project: vnpy_crypto   Author: birforce   File: test_decorators.py    License: MIT License 6 votes vote down vote up
def test_multi_skts_with_name():
    @socket('foo', zmq.PUSH)
    @socket('bar', zmq.SUB)
    @socket('baz', zmq.PUB)
    def test(foo, bar, baz):
        assert isinstance(foo, zmq.Socket), foo
        assert isinstance(bar, zmq.Socket), bar
        assert isinstance(baz, zmq.Socket), baz

        assert foo.context is zmq.Context.instance()
        assert bar.context is zmq.Context.instance()
        assert baz.context is zmq.Context.instance()

        assert foo.type == zmq.PUSH
        assert bar.type == zmq.SUB
        assert baz.type == zmq.PUB
    test() 
Example #12
Source Project: vnpy_crypto   Author: birforce   File: test_decorators.py    License: MIT License 6 votes vote down vote up
def test_skt_multi_thread():
    @socket(zmq.PUB)
    @socket(zmq.SUB)
    @socket(zmq.PUSH)
    def f(pub, sub, push):
        assert isinstance(pub, zmq.Socket), pub
        assert isinstance(sub, zmq.Socket), sub
        assert isinstance(push, zmq.Socket), push

        assert pub.context is zmq.Context.instance()
        assert sub.context is zmq.Context.instance()
        assert push.context is zmq.Context.instance()

        assert pub.type == zmq.PUB
        assert sub.type == zmq.SUB
        assert push.type == zmq.PUSH

        assert len(set(map(id, [pub, sub, push]))) == 3

    threads = [threading.Thread(target=f) for i in range(8)]
    [t.start() for t in threads]
    [t.join() for t in threads] 
Example #13
Source Project: vnpy_crypto   Author: birforce   File: test_future.py    License: MIT License 6 votes vote down vote up
def test_recv_json_cancelled(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield gen.sleep(0)
            obj = dict(a=5)
            yield a.send_json(obj)
            with pytest.raises(future.CancelledError):
                recvd = yield f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield b.poll(timeout=5)
            assert events
            yield gen.sleep(0)
            # make sure cancelled recv didn't eat up event
            recvd = yield gen.with_timeout(timedelta(seconds=5), b.recv_json())
            assert recvd == obj
        self.loop.run_sync(test) 
Example #14
Source Project: vnpy_crypto   Author: birforce   File: test_future.py    License: MIT License 6 votes vote down vote up
def test_poll(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.poll(timeout=0)
            assert f.done()
            self.assertEqual(f.result(), 0)

            f = b.poll(timeout=1)
            assert not f.done()
            evt = yield f
            self.assertEqual(evt, 0)

            f = b.poll(timeout=1000)
            assert not f.done()
            yield a.send_multipart([b'hi', b'there'])
            evt = yield f
            self.assertEqual(evt, zmq.POLLIN)
            recvd = yield b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_sync(test) 
Example #15
def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):
        
        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)
    
    
    ########################################################################## 
Example #16
def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):
        
        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)
    
    
    ########################################################################## 
Example #17
Source Project: Computable   Author: ktraunmueller   File: win32support.py    License: MIT License 6 votes vote down vote up
def forward_read_events(fd, context=None):
    """Forward read events from an FD over a socket.

    This method wraps a file in a socket pair, so it can
    be polled for read events by select (specifically zmq.eventloop.ioloop)
    """
    if context is None:
        context = zmq.Context.instance()
    push = context.socket(zmq.PUSH)
    push.setsockopt(zmq.LINGER, -1)
    pull = context.socket(zmq.PULL)
    addr='inproc://%s'%uuid.uuid4()
    push.bind(addr)
    pull.connect(addr)
    forwarder = ForwarderThread(push, fd)
    forwarder.start()
    return pull 
Example #18
Source Project: DDRL   Author: anonymous-author1   File: remote.py    License: Apache License 2.0 6 votes vote down vote up
def serve_data(ds, addr):
    ctx = zmq.Context()
    socket = ctx.socket(zmq.PUSH)
    socket.set_hwm(10)
    socket.bind(addr)
    ds = RepeatedData(ds, -1)
    try:
        ds.reset_state()
        logger.info("Serving data at {}".format(addr))
        while True:
            for dp in ds.get_data():
                socket.send(dumps(dp), copy=False)
    finally:
        socket.setsockopt(zmq.LINGER, 0)
        socket.close()
        if not ctx.closed:
            ctx.destroy(0) 
Example #19
Source Project: attention-lvcsr   Author: rizar   File: parallel.py    License: MIT License 6 votes vote down vote up
def _producer_wrapper(f, port, addr='tcp://127.0.0.1'):
    """A shim that sets up a socket and starts the producer callable.

    Parameters
    ----------
    f : callable
        Callable that takes a single argument, a handle
        for a ZeroMQ PUSH socket. Must be picklable.
    port : int
        The port on which the socket should connect.
    addr : str, optional
        Address to which the socket should connect. Defaults
        to localhost ('tcp://127.0.0.1').

    """
    try:
        context = zmq.Context()
        socket = context.socket(zmq.PUSH)
        socket.connect(':'.join([addr, str(port)]))
        f(socket)
    finally:
        # Works around a Python 3.x bug.
        context.destroy() 
Example #20
Source Project: attention-lvcsr   Author: rizar   File: parallel.py    License: MIT License 6 votes vote down vote up
def _spawn_producer(f, port, addr='tcp://127.0.0.1'):
    """Start a process that sends results on a PUSH socket.

    Parameters
    ----------
    f : callable
        Callable that takes a single argument, a handle
        for a ZeroMQ PUSH socket. Must be picklable.

    Returns
    -------
    process : multiprocessing.Process
        The process handle of the created producer process.

    """
    process = Process(target=_producer_wrapper, args=(f, port, addr))
    process.start()
    return process 
Example #21
Source Project: dataflow   Author: tensorpack   File: parallel_map.py    License: Apache License 2.0 5 votes vote down vote up
def run(self):
            enable_death_signal()
            ctx = zmq.Context()
            socket = ctx.socket(zmq.PUSH)
            socket.set_hwm(self.hwm)
            socket.bind(self.pipename)
            self.ds.reset_state()
            for dp in self.ds:
                socket.send(dumps(dp), copy=False) 
Example #22
Source Project: westpa   Author: westpa   File: test_work_manager.py    License: MIT License 5 votes vote down vote up
def result_socket(self):
        socket = self.test_context.socket(zmq.PUSH)
        socket.connect(self.test_wm.wm_result_endpoint)
        
        yield socket
        
        socket.close(linger=1) 
Example #23
Source Project: westpa   Author: westpa   File: worker.py    License: MIT License 5 votes vote down vote up
def shutdown_executor(self):
        if self.context is not None:
            try:
                self.log.debug('sending shutdown task to executor')
                task_socket = self.context.socket(zmq.PUSH)
                task_socket.connect(self.task_endpoint)
                self.send_message(task_socket, Message.SHUTDOWN)
                task_socket.close(linger=1)
            except:
                pass

        pid = self.executor_process.pid
        self.executor_process.join(self.shutdown_timeout)
        # is_alive() is prone to a race condition so catch the case that the PID is already dead
        if self.executor_process.is_alive():
            self.log.debug('sending SIGTERM to worker process {:d}'.format(pid))
            self.executor_process.terminate()
            # try:
            #     os.kill(self.executor_process.pid, signal.SIGINT)
            # except ProcessLookupError:
            #     self.log.debug('worker process {:d} already dead'.format(pid))
            self.executor_process.join(self.shutdown_timeout)
            if self.executor_process.is_alive():
                self.executor_process.kill()
                self.log.warning('sending SIGKILL to worker process {:d}'.format(pid))
                # try:
                #     os.kill(self.executor_process.pid, signal.SIGKILL)
                # except ProcessLookupError:
                #     self.log.debug('worker process {:d} already dead'.format(pid))
            self.executor_process.join()
            self.log.debug('worker process {:d} terminated'.format(pid))
        else:
            self.log.debug('worker process {:d} terminated gracefully with code {:d}'.format(pid, self.executor_process.exitcode)) 
Example #24
Source Project: westpa   Author: westpa   File: worker.py    License: MIT License 5 votes vote down vote up
def comm_loop(self):
        
        task_socket = self.context.socket(zmq.PULL)
        result_socket = self.context.socket(zmq.PUSH)
        
        task_socket.bind(self.task_endpoint)
        result_socket.connect(self.result_endpoint)
        
        self.log.info('This is {}'.format(self.node_description))
        
        try:
            while True:
                try:
                    msg = self.recv_message(task_socket,timeout=100)
                except KeyboardInterrupt:
                    break
                except ZMQWMTimeout:
                    continue
                else:
                    if msg.message == Message.TASK:
                        task = msg.payload
                        result = task.execute()
                        self.send_message(result_socket, Message.RESULT, result)
                    elif msg.message == Message.SHUTDOWN:
                        break
        finally:
            self.context.destroy(linger=0)
            self.context = None 
Example #25
Source Project: cloud-asr   Author: UFAL-DSG   File: lib.py    License: Apache License 2.0 5 votes vote down vote up
def create_master(worker_address, frontend_address, monitor_address):
    poller = create_poller(worker_address, frontend_address)
    context = zmq.Context()
    monitor = context.socket(zmq.PUSH)
    monitor.connect(monitor_address)
    run_forever = lambda: True

    return Master(poller, monitor, run_forever) 
Example #26
Source Project: cloud-asr   Author: UFAL-DSG   File: lib.py    License: Apache License 2.0 5 votes vote down vote up
def create_recordings_saver_socket(address):
    context = zmq.Context()
    socket = context.socket(zmq.PUSH)
    socket.connect(address)

    return socket 
Example #27
Source Project: cloud-asr   Author: UFAL-DSG   File: lib.py    License: Apache License 2.0 5 votes vote down vote up
def create_heartbeat(model, address, master_address):
    context = zmq.Context()
    master_socket = context.socket(zmq.PUSH)
    master_socket.connect(master_address)

    return Heartbeat(model, address, master_socket) 
Example #28
Source Project: heralding   Author: johnnykv   File: curiosum_integration.py    License: GNU General Public License v3.0 5 votes vote down vote up
def __init__(self, port):
    super().__init__()

    zmq_socket = 'tcp://127.0.0.1:{0}'.format(port)

    context = heralding.misc.zmq_context
    self.socket = context.socket(zmq.PUSH)
    self.socket.bind(zmq_socket)
    self.listen_ports = []
    self.last_listen_ports_transmit = datetime.now()

    logger.info('Curiosum logger started using files: %s', zmq_socket) 
Example #29
Source Project: Bert-TextClassification   Author: a414351664   File: client.py    License: MIT License 5 votes vote down vote up
def __init__(self, ip='localhost', port=5555, port_out=5556,
                 output_fmt='ndarray', show_server_config=False,
                 identity=None):
        self.context = zmq.Context()
        self.sender = self.context.socket(zmq.PUSH)
        self.identity = identity or str(uuid.uuid4()).encode('ascii')
        self.sender.connect('tcp://%s:%d' % (ip, port))

        self.receiver = self.context.socket(zmq.SUB)
        self.receiver.setsockopt(zmq.SUBSCRIBE, self.identity)
        self.receiver.connect('tcp://%s:%d' % (ip, port_out))

        if output_fmt == 'ndarray':
            self.formatter = lambda x: x
        elif output_fmt == 'list':
            self.formatter = lambda x: x.tolist()
        else:
            raise AttributeError('"output_fmt" must be "ndarray" or "list"')

        if show_server_config:
            print('server returns the following config:')
            for k, v in self.get_server_config().items():
                print('%30s\t=\t%-30s' % (k, v))
        print('you should NOT see this message multiple times! '
              'if you see it appears repeatedly, '
              'consider moving "BertClient()" out of the loop.') 
Example #30
Source Project: vnpy_crypto   Author: birforce   File: garbage.py    License: MIT License 5 votes vote down vote up
def _stop(self):
        push = self.context.socket(zmq.PUSH)
        push.connect(self.url)
        push.send(b'DIE')
        push.close()
        if self._push:
            self._push.close()
            self._push = None
        self._push_mutex = None
        self.thread.join()
        self.context.term()
        self.refs.clear()
        self.context = None