Python zmq.PUSH Examples

def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()

            # recv jobs
            socket = ctx.socket(zmq.PULL)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm * self.batch_size)
            socket.connect(self.input_pipe)

            # send results
            out_socket = ctx.socket(zmq.PUSH)
            out_socket.set_hwm(max(self.hwm, 5))
            out_socket.connect(self.result_pipe)

            batch = []
            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                if dp is not None:
                    batch.append(dp)
                    if len(batch) == self.batch_size:
                        dp = BatchData.aggregate_batch(batch)
                        out_socket.send(dumps(dp), copy=False)
                        del batch[:] 

def serve_data(ds, addr):
    ctx = zmq.Context()
    socket = ctx.socket(zmq.PUSH)
    socket.set_hwm(10)
    socket.bind(addr)
    ds = RepeatedData(ds, -1)
    try:
        ds.reset_state()
        logger.info("Serving data at {}".format(addr))
        while True:
            for dp in ds.get_data():
                socket.send(dumps(dp), copy=False)
    finally:
        socket.setsockopt(zmq.LINGER, 0)
        socket.close()
        if not ctx.closed:
            ctx.destroy(0) 

def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):
        
        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)
    
    
    ########################################################################## 

def _DWX_MTX_SEND_MARKETDATA_REQUEST_(self,
                                 _symbol='EURUSD',
                                 _timeframe=1,
                                 _start='2019.01.04 17:00:00',
                                 _end=Timestamp.now().strftime('%Y.%m.%d %H:%M:00')):
                                 #_end='2019.01.04 17:05:00'):
        
        _msg = "{};{};{};{};{}".format('DATA',
                                     _symbol,
                                     _timeframe,
                                     _start,
                                     _end)
        # Send via PUSH Socket
        self.remote_send(self._PUSH_SOCKET, _msg)
    
    
    ########################################################################## 

def test_recv_json_cancelled(self):
        @asyncio.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield from asyncio.sleep(0)
            obj = dict(a=5)
            yield from a.send_json(obj)
            with pytest.raises(CancelledError):
                recvd = yield from f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield from b.poll(timeout=5)
            assert events
            yield from asyncio.sleep(0)
            # make sure cancelled recv didn't eat up event
            f = b.recv_json()
            recvd = yield from asyncio.wait_for(f, timeout=5)
            assert recvd == obj
        self.loop.run_until_complete(test()) 

def test_poll_base_socket(self):
        @asyncio.coroutine
        def test():
            ctx = zmq.Context()
            url = 'inproc://test'
            a = ctx.socket(zmq.PUSH)
            b = ctx.socket(zmq.PULL)
            self.sockets.extend([a, b])
            a.bind(url)
            b.connect(url)

            poller = zaio.Poller()
            poller.register(b, zmq.POLLIN)

            f = poller.poll(timeout=1000)
            assert not f.done()
            a.send_multipart([b'hi', b'there'])
            evt = yield from f
            self.assertEqual(evt, [(b, zmq.POLLIN)])
            recvd = b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_until_complete(test()) 

def test_gc(self):
        """test close&term by garbage collection alone"""
        if PYPY:
            raise SkipTest("GC doesn't work ")
            
        # test credit @dln (GH #137):
        def gcf():
            def inner():
                ctx = self.Context()
                s = ctx.socket(zmq.PUSH)
            inner()
            gc.collect()
        t = Thread(target=gcf)
        t.start()
        t.join(timeout=1)
        self.assertFalse(t.is_alive(), "Garbage collection should have cleaned up context") 

def forward_read_events(fd, context=None):
    """Forward read events from an FD over a socket.

    This method wraps a file in a socket pair, so it can
    be polled for read events by select (specifically zmq.eventloop.ioloop)
    """
    if context is None:
        context = zmq.Context.instance()
    push = context.socket(zmq.PUSH)
    push.setsockopt(zmq.LINGER, -1)
    pull = context.socket(zmq.PULL)
    addr='inproc://%s'%uuid.uuid4()
    push.bind(addr)
    pull.connect(addr)
    forwarder = ForwarderThread(push, fd)
    forwarder.start()
    return pull 

def test_shadow_pyczmq(self):
        try:
            from pyczmq import zctx, zsocket, zstr
        except Exception:
            raise SkipTest("Requires pyczmq")
        
        ctx = zctx.new()
        a = zsocket.new(ctx, zmq.PUSH)
        zsocket.bind(a, "inproc://a")
        ctx2 = self.Context.shadow_pyczmq(ctx)
        b = ctx2.socket(zmq.PULL)
        b.connect("inproc://a")
        zstr.send(a, b'hi')
        rcvd = self.recv(b)
        self.assertEqual(rcvd, b'hi')
        b.close() 

def test_shadow(self):
        p = self.socket(zmq.PUSH)
        p.bind("tcp://127.0.0.1:5555")
        p2 = zmq.Socket.shadow(p.underlying)
        self.assertEqual(p.underlying, p2.underlying)
        s = self.socket(zmq.PULL)
        s2 = zmq.Socket.shadow(s.underlying)
        self.assertNotEqual(s.underlying, p.underlying)
        self.assertEqual(s.underlying, s2.underlying)
        s2.connect("tcp://127.0.0.1:5555")
        sent = b'hi'
        p2.send(sent)
        rcvd = self.recv(s2)
        self.assertEqual(rcvd, sent) 

def test_multi_skts():
    @socket(zmq.PUB)
    @socket(zmq.SUB)
    @socket(zmq.PUSH)
    def test(pub, sub, push):
        assert isinstance(pub, zmq.Socket), pub
        assert isinstance(sub, zmq.Socket), sub
        assert isinstance(push, zmq.Socket), push

        assert pub.context is zmq.Context.instance()
        assert sub.context is zmq.Context.instance()
        assert push.context is zmq.Context.instance()

        assert pub.type == zmq.PUB
        assert sub.type == zmq.SUB
        assert push.type == zmq.PUSH
    test() 

def test_multi_skts_with_name():
    @socket('foo', zmq.PUSH)
    @socket('bar', zmq.SUB)
    @socket('baz', zmq.PUB)
    def test(foo, bar, baz):
        assert isinstance(foo, zmq.Socket), foo
        assert isinstance(bar, zmq.Socket), bar
        assert isinstance(baz, zmq.Socket), baz

        assert foo.context is zmq.Context.instance()
        assert bar.context is zmq.Context.instance()
        assert baz.context is zmq.Context.instance()

        assert foo.type == zmq.PUSH
        assert bar.type == zmq.SUB
        assert baz.type == zmq.PUB
    test() 

def socket(*args, **kwargs):
    '''Decorator for adding a socket to a function.
    
    Usage::
    
        @socket(zmq.PUSH)
        def foo(push):
            ...
    
    .. versionadded:: 15.3

    :param str name: the keyword argument passed to decorated function
    :param str context_name: the keyword only argument to identify context
                             object
    '''
    return _SocketDecorator()(*args, **kwargs) 

def run(self):
        context = zmq.Context()
        receiver = context.socket(zmq.PULL)
        receiver.connect(self.worker_address)

        sink = context.socket(zmq.PUSH)
        sink.connect(self.sink_address)

        input_fn = self.input_fn_builder(receiver)

        self.logger.info('ready and listening')
        start_t = time.perf_counter()
        for r in self.estimator.predict(input_fn, yield_single_examples=False):
            # logger.info('new result!')
            send_ndarray(sink, r['client_id'], r['encodes'])
            time_used = time.perf_counter() - start_t
            start_t = time.perf_counter()
            self.logger.info('job %s\tsamples: %4d\tdone: %.2fs' %
                             (r['client_id'], r['encodes'].shape[0], time_used))

        receiver.close()
        sink.close()
        context.term()
        self.logger.info('terminated!') 

def test_skt_multi_thread():
    @socket(zmq.PUB)
    @socket(zmq.SUB)
    @socket(zmq.PUSH)
    def f(pub, sub, push):
        assert isinstance(pub, zmq.Socket), pub
        assert isinstance(sub, zmq.Socket), sub
        assert isinstance(push, zmq.Socket), push

        assert pub.context is zmq.Context.instance()
        assert sub.context is zmq.Context.instance()
        assert push.context is zmq.Context.instance()

        assert pub.type == zmq.PUB
        assert sub.type == zmq.SUB
        assert push.type == zmq.PUSH

        assert len(set(map(id, [pub, sub, push]))) == 3

    threads = [threading.Thread(target=f) for i in range(8)]
    [t.start() for t in threads]
    [t.join() for t in threads] 

def test_poll(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.poll(timeout=0)
            assert f.done()
            self.assertEqual(f.result(), 0)

            f = b.poll(timeout=1)
            assert not f.done()
            evt = yield f
            self.assertEqual(evt, 0)

            f = b.poll(timeout=1000)
            assert not f.done()
            yield a.send_multipart([b'hi', b'there'])
            evt = yield f
            self.assertEqual(evt, zmq.POLLIN)
            recvd = yield b.recv_multipart()
            self.assertEqual(recvd, [b'hi', b'there'])
        self.loop.run_sync(test) 

def test_recv_json_cancelled(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            f.cancel()
            # cycle eventloop to allow cancel events to fire
            yield gen.sleep(0)
            obj = dict(a=5)
            yield a.send_json(obj)
            with pytest.raises(future.CancelledError):
                recvd = yield f
            assert f.done()
            # give it a chance to incorrectly consume the event
            events = yield b.poll(timeout=5)
            assert events
            yield gen.sleep(0)
            # make sure cancelled recv didn't eat up event
            recvd = yield gen.with_timeout(timedelta(seconds=5), b.recv_json())
            assert recvd == obj
        self.loop.run_sync(test) 

def run(self):
            enable_death_signal(_warn=self.idx == 0)
            self.ds.reset_state()
            itr = _repeat_iter(lambda: self.ds)

            context = zmq.Context()
            socket = context.socket(zmq.PUSH)
            socket.set_hwm(self.hwm)
            socket.connect(self.conn_name)
            try:
                while True:
                    try:
                        dp = next(itr)
                        socket.send(dumps(dp), copy=False)
                    except Exception:
                        dp = _ExceptionWrapper(sys.exc_info()).pack()
                        socket.send(dumps(dp), copy=False)
                        raise
            # sigint could still propagate here, e.g. when nested
            except KeyboardInterrupt:
                pass
            finally:
                socket.close(0)
                context.destroy(0) 

def _spawn_producer(f, port, addr='tcp://127.0.0.1'):
    """Start a process that sends results on a PUSH socket.

    Parameters
    ----------
    f : callable
        Callable that takes a single argument, a handle
        for a ZeroMQ PUSH socket. Must be picklable.

    Returns
    -------
    process : multiprocessing.Process
        The process handle of the created producer process.

    """
    process = Process(target=_producer_wrapper, args=(f, port, addr))
    process.start()
    return process 

def _producer_wrapper(f, port, addr='tcp://127.0.0.1'):
    """A shim that sets up a socket and starts the producer callable.

    Parameters
    ----------
    f : callable
        Callable that takes a single argument, a handle
        for a ZeroMQ PUSH socket. Must be picklable.
    port : int
        The port on which the socket should connect.
    addr : str, optional
        Address to which the socket should connect. Defaults
        to localhost ('tcp://127.0.0.1').

    """
    try:
        context = zmq.Context()
        socket = context.socket(zmq.PUSH)
        socket.connect(':'.join([addr, str(port)]))
        f(socket)
    finally:
        # Works around a Python 3.x bug.
        context.destroy() 

def test_message_pattern_type():
    """Test schema errors are raised for incorrect pattern types.."""
    server_args = {"name": "server", "host": "localhost", "port": 0}

    with pytest.raises(SchemaError):
        ZMQServer(message_pattern=zmq.REQ, **server_args)

    with pytest.raises(SchemaError):
        ZMQServer(message_pattern=zmq.SUB, **server_args)

    with pytest.raises(SchemaError):
        ZMQServer(message_pattern=zmq.PULL, **server_args)

    client_args = {
        "name": "client",
        "hosts": ["localhost"],
        "ports": [0],
        "connect_at_start": False,
    }

    with pytest.raises(SchemaError):
        ZMQClient(message_pattern=zmq.REP, **client_args)

    with pytest.raises(SchemaError):
        ZMQClient(message_pattern=zmq.PUB, **client_args)

    with pytest.raises(SchemaError):
        ZMQClient(message_pattern=zmq.PUSH, **client_args) 

def test_many_push_one_pull(runpath):
    """Test connecting one pull client to two push servers."""
    server1 = ZMQServer(
        name="server1",
        host="127.0.0.1",
        port=0,
        message_pattern=zmq.PUSH,
        runpath=os.path.join(runpath, "server1"),
    )
    server2 = ZMQServer(
        name="server2",
        host="127.0.0.1",
        port=0,
        message_pattern=zmq.PUSH,
        runpath=os.path.join(runpath, "server2"),
    )

    with server1, server2:
        client = ZMQClient(
            name="client",
            hosts=[server1.host, server2.host],
            ports=[server1.port, server2.port],
            message_pattern=zmq.PULL,
            runpath=os.path.join(runpath, "client"),
        )

        with client:
            time.sleep(1)

            data1 = b"Hello"
            data2 = b"World"
            server1.send(data=data1, timeout=TIMEOUT)
            recv1 = client.receive(timeout=TIMEOUT)
            server2.send(data=data2, timeout=TIMEOUT)
            recv2 = client.receive(timeout=TIMEOUT)
            assert data1 == recv1
            assert data2 == recv2 

def __init__(self, socket_type, to_recv=()):
        self.socket_type = socket_type
        if self.socket_type not in (zmq.PUSH, zmq.PULL):
            raise NotImplementedError('only PUSH and PULL currently supported')
        self.sent = deque()
        self.to_recv = deque(to_recv) 

def remote_send(self, _socket, _data):
        
        if self._PUSH_SOCKET_STATUS['state'] == True:
            try:
                _socket.send_string(_data, zmq.DONTWAIT)
            except zmq.error.Again:
                print("\nResource timeout.. please try again.")
                sleep(self._sleep_delay)
        else:
            print('\n[KERNEL] NO HANDSHAKE ON PUSH SOCKET.. Cannot SEND data')
      
    ########################################################################## 

def test_proxy(self):
        if zmq.zmq_version_info() < (3,2):
            raise SkipTest("Proxies only in libzmq >= 3")
        dev = devices.ThreadProxy(zmq.PULL, zmq.PUSH, zmq.PUSH)
        binder = self.context.socket(zmq.REQ)
        iface = 'tcp://127.0.0.1'
        port = binder.bind_to_random_port(iface)
        port2 = binder.bind_to_random_port(iface)
        port3 = binder.bind_to_random_port(iface)
        binder.close()
        time.sleep(0.1)
        dev.bind_in("%s:%i" % (iface, port))
        dev.bind_out("%s:%i" % (iface, port2))
        dev.bind_mon("%s:%i" % (iface, port3))
        dev.start()
        time.sleep(0.25)
        msg = b'hello'
        push = self.context.socket(zmq.PUSH)
        push.connect("%s:%i" % (iface, port))
        pull = self.context.socket(zmq.PULL)
        pull.connect("%s:%i" % (iface, port2))
        mon = self.context.socket(zmq.PULL)
        mon.connect("%s:%i" % (iface, port3))
        push.send(msg)
        self.sockets.extend([push, pull, mon])
        self.assertEqual(msg, self.recv(pull))
        self.assertEqual(msg, self.recv(mon)) 

def test_recv_string(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_string()
            assert not f.done()
            msg = u('πøøπ')
            yield a.send_string(msg)
            recvd = yield f
            assert f.done()
            self.assertEqual(f.result(), msg)
            self.assertEqual(recvd, msg)
        self.loop.run_sync(test) 

def test_recv_json(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_json()
            assert not f.done()
            obj = dict(a=5)
            yield a.send_json(obj)
            recvd = yield f
            assert f.done()
            self.assertEqual(f.result(), obj)
            self.assertEqual(recvd, obj)
        self.loop.run_sync(test) 

def test_recv_pyobj(self):
        @gen.coroutine
        def test():
            a, b = self.create_bound_pair(zmq.PUSH, zmq.PULL)
            f = b.recv_pyobj()
            assert not f.done()
            obj = dict(a=5)
            yield a.send_pyobj(obj)
            recvd = yield f
            assert f.done()
            self.assertEqual(f.result(), obj)
            self.assertEqual(recvd, obj)
        self.loop.run_sync(test) 

def setUp(self):
        if tornado is None:
            pytest.skip()
        self.context = zmq.Context()
        self.loop = ioloop.IOLoop.instance()
        self.push = zmqstream.ZMQStream(self.context.socket(zmq.PUSH))
        self.pull = zmqstream.ZMQStream(self.context.socket(zmq.PULL))
        port = self.push.bind_to_random_port('tcp://127.0.0.1')
        self.pull.connect('tcp://127.0.0.1:%i' % port)
        self.stream = self.push 

def test_send_noblock(self):
        @gen.coroutine
        def test():
            s = self.socket(zmq.PUSH)
            with pytest.raises(zmq.Again):
                yield s.send(b'not going anywhere', flags=zmq.NOBLOCK)
        self.loop.run_sync(test)