Python zmq.IDENTITY Examples

def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()

            # recv jobs
            socket = ctx.socket(zmq.PULL)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm * self.batch_size)
            socket.connect(self.input_pipe)

            # send results
            out_socket = ctx.socket(zmq.PUSH)
            out_socket.set_hwm(max(self.hwm, 5))
            out_socket.connect(self.result_pipe)

            batch = []
            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                if dp is not None:
                    batch.append(dp)
                    if len(batch) == self.batch_size:
                        dp = BatchData.aggregate_batch(batch)
                        out_socket.send(dumps(dp), copy=False)
                        del batch[:] 

def test_unicode_sockopts(self):
        """test setting/getting sockopts with unicode strings"""
        topic = "tést"
        if str is not unicode:
            topic = topic.decode('utf8')
        p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
        self.assertEqual(s.send_unicode, s.send_unicode)
        self.assertEqual(p.recv_unicode, p.recv_unicode)
        self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
        s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
        self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
        s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
        self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE)
        
        identb = s.getsockopt(zmq.IDENTITY)
        identu = identb.decode('utf16')
        identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16')
        self.assertEqual(identu, identu2)
        time.sleep(0.1) # wait for connection/subscription
        p.send_unicode(topic,zmq.SNDMORE)
        p.send_unicode(topic*2, encoding='latin-1')
        self.assertEqual(topic, s.recv_unicode())
        self.assertEqual(topic*2, s.recv_unicode(encoding='latin-1')) 

def get(self, option):
        c_data = new_pointer_from_opt(option, length=255)

        c_value_pointer = c_data[0]
        c_sizet_pointer = c_data[1]

        _retry_sys_call(C.zmq_getsockopt,
                        self._zmq_socket,
                        option,
                        c_value_pointer,
                        c_sizet_pointer)
        
        sz = c_sizet_pointer[0]
        v = value_from_opt_pointer(option, c_value_pointer, sz)
        if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
            v = v[:-1]
        return v 

def _setup_sockets(self):
        ins,outs = Device._setup_sockets(self)
        ctx = self._context
        mons = ctx.socket(self.mon_type)
        
        # set sockopts (must be done first, in case of zmq.IDENTITY)
        for opt,value in self._mon_sockopts:
            mons.setsockopt(opt, value)
        
        for iface in self._mon_binds:
            mons.bind(iface)
        
        for iface in self._mon_connects:
            mons.connect(iface)
        
        return ins,outs,mons 

def get(self, option):
        low_level_data = new_pointer_from_opt(option, length=255)

        low_level_value_pointer = low_level_data[0]
        low_level_sizet_pointer = low_level_data[1]

        rc = C.zmq_getsockopt(self._zmq_socket,
                               option,
                               low_level_value_pointer,
                               low_level_sizet_pointer)
        _check_rc(rc)
        
        sz = low_level_sizet_pointer[0]
        v = value_from_opt_pointer(option, low_level_value_pointer, sz)
        if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
            v = v[:-1]
        return v 

def _setup_sockets(self):
        ins,outs = Device._setup_sockets(self)
        ctx = self._context
        mons = ctx.socket(self.mon_type)
        
        # set sockopts (must be done first, in case of zmq.IDENTITY)
        for opt,value in self._mon_sockopts:
            mons.setsockopt(opt, value)
        
        for iface in self._mon_binds:
            mons.bind(iface)
        
        for iface in self._mon_connects:
            mons.connect(iface)
        
        return ins,outs,mons 

def register(self):
        """send the registration_request"""

        self.log.info("Registering with controller at %s"%self.url)
        ctx = self.context
        connect,maybe_tunnel = self.init_connector()
        reg = ctx.socket(zmq.DEALER)
        reg.setsockopt(zmq.IDENTITY, self.bident)
        connect(reg, self.url)
        self.registrar = zmqstream.ZMQStream(reg, self.loop)


        content = dict(uuid=self.ident)
        self.registrar.on_recv(lambda msg: self.complete_registration(msg, connect, maybe_tunnel))
        # print (self.session.key)
        self.session.send(self.registrar, "registration_request", content=content) 

def __init__(self, in_addr, out_addr, mon_addr=None, in_type=zmq.SUB, out_type=zmq.DEALER, mon_type=zmq.PUB, heart_id=None):
        if mon_addr is None:
            self.device = ThreadDevice(zmq.FORWARDER, in_type, out_type)
        else:
            self.device = ThreadMonitoredQueue(in_type, out_type, mon_type, in_prefix=b"", out_prefix=b"")
        # do not allow the device to share global Context.instance,
        # which is the default behavior in pyzmq > 2.1.10
        self.device.context_factory = zmq.Context
        
        self.device.daemon=True
        self.device.connect_in(in_addr)
        self.device.connect_out(out_addr)
        if mon_addr is not None:
            self.device.connect_mon(mon_addr)
        if in_type == zmq.SUB:
            self.device.setsockopt_in(zmq.SUBSCRIBE, b"")
        if heart_id is None:
            heart_id = uuid.uuid4().bytes
        self.device.setsockopt_out(zmq.IDENTITY, heart_id)
        self.id = heart_id 

def initialize_client_thread(jobiface):
        """
        Creates a ZMQ object in this thread. This talks to background processes.
        """
        print = partial(ut.colorprint, color='blue')
        if jobiface.verbose:
            print('Initializing JobInterface')
        jobiface.engine_deal_sock = ctx.socket(zmq.DEALER)
        jobiface.engine_deal_sock.setsockopt_string(zmq.IDENTITY,
                                                    'client%s.engine.DEALER' %
                                                    (jobiface.id_,))
        jobiface.engine_deal_sock.connect(jobiface.port_dict['engine_url1'])
        if jobiface.verbose:
            print('connect engine_url1 = %r' % (jobiface.port_dict['engine_url1'],))

        jobiface.collect_deal_sock = ctx.socket(zmq.DEALER)
        jobiface.collect_deal_sock.setsockopt_string(zmq.IDENTITY,
                                                     'client%s.collect.DEALER'
                                                     % (jobiface.id_,))
        jobiface.collect_deal_sock.connect(jobiface.port_dict['collect_url1'])
        if jobiface.verbose:
            print('connect collect_url1 = %r' % (jobiface.port_dict['collect_url1'],)) 

def sendConnect(self, data):
        """Send a CONNECT command to the broker
            :param data: List of other broker main socket URL"""
        # Imported dynamically - Not used if only one broker
        if self.backend == 'ZMQ':
            import zmq
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.DEALER)
            self.socket.setsockopt(zmq.IDENTITY, b'launcher')
            self.socket.connect(
                "tcp://127.0.0.1:{port}".format(
                    port=self.brokerPort,
                )
            )
            self.socket.send_multipart([b"CONNECT",
                                        pickle.dumps(data,
                                                     pickle.HIGHEST_PROTOCOL)])
        else:
            # TODO
            pass 

def sendConnect(self, data):
        """Send a CONNECT command to the broker
            :param data: List of other broker main socket URL"""
        # Imported dynamically - Not used if only one broker
        if self.backend == 'ZMQ':
            import zmq
            self.context = zmq.Context()
            self.socket = self.context.socket(zmq.DEALER)
            if sys.version_info < (3,):
                self.socket.setsockopt_string(zmq.IDENTITY, unicode('launcher'))
            else:
                self.socket.setsockopt_string(zmq.IDENTITY, 'launcher')
            self.socket.connect(
                "tcp://{hostname}:{port}".format(
                    port=self.brokerPort,
                    hostname = self.hostname
                )
            )
            self.socket.send_multipart([b"CONNECT",
                                        pickle.dumps(data,
                                                     pickle.HIGHEST_PROTOCOL)])
        else:
            # TODO
            pass 

def test_unicode_sockopts(self):
        """test setting/getting sockopts with unicode strings"""
        topic = "tést"
        if str is not unicode:
            topic = topic.decode('utf8')
        p,s = self.create_bound_pair(zmq.PUB, zmq.SUB)
        self.assertEqual(s.send_unicode, s.send_unicode)
        self.assertEqual(p.recv_unicode, p.recv_unicode)
        self.assertRaises(TypeError, s.setsockopt, zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.setsockopt, zmq.IDENTITY, topic)
        s.setsockopt_unicode(zmq.IDENTITY, topic, 'utf16')
        self.assertRaises(TypeError, s.setsockopt, zmq.AFFINITY, topic)
        s.setsockopt_unicode(zmq.SUBSCRIBE, topic)
        self.assertRaises(TypeError, s.getsockopt_unicode, zmq.AFFINITY)
        self.assertRaisesErrno(zmq.EINVAL, s.getsockopt_unicode, zmq.SUBSCRIBE)
        
        identb = s.getsockopt(zmq.IDENTITY)
        identu = identb.decode('utf16')
        identu2 = s.getsockopt_unicode(zmq.IDENTITY, 'utf16')
        self.assertEqual(identu, identu2)
        time.sleep(0.1) # wait for connection/subscription
        p.send_unicode(topic,zmq.SNDMORE)
        p.send_unicode(topic*2, encoding='latin-1')
        self.assertEqual(topic, s.recv_unicode())
        self.assertEqual(topic*2, s.recv_unicode(encoding='latin-1')) 

def get(self, option):
        c_data = new_pointer_from_opt(option, length=255)

        c_value_pointer = c_data[0]
        c_sizet_pointer = c_data[1]

        _retry_sys_call(C.zmq_getsockopt,
                        self._zmq_socket,
                        option,
                        c_value_pointer,
                        c_sizet_pointer)
        
        sz = c_sizet_pointer[0]
        v = value_from_opt_pointer(option, c_value_pointer, sz)
        if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
            v = v[:-1]
        return v 

def _setup_sockets(self):
        ins,outs = Device._setup_sockets(self)
        ctx = self._context
        mons = ctx.socket(self.mon_type)
        
        # set sockopts (must be done first, in case of zmq.IDENTITY)
        for opt,value in self._mon_sockopts:
            mons.setsockopt(opt, value)
        
        for iface in self._mon_binds:
            mons.bind(iface)
        
        for iface in self._mon_connects:
            mons.connect(iface)
        
        return ins,outs,mons 

def run(self):
        player = self._build_player()
        context = zmq.Context()
        c2s_socket = context.socket(zmq.PUSH)
        c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
        c2s_socket.set_hwm(2)
        c2s_socket.connect(self.c2s)

        s2c_socket = context.socket(zmq.DEALER)
        s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
        #s2c_socket.set_hwm(5)
        s2c_socket.connect(self.s2c)

        state = player.current_state()
        reward, isOver = 0, False
        while True:
            c2s_socket.send(dumps(
                (self.identity, state, reward, isOver)),
                copy=False)
            action = loads(s2c_socket.recv(copy=False).bytes)
            reward, isOver = player.action(action)
            state = player.current_state()

# compatibility 

def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()

            socket = ctx.socket(zmq.PULL)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm)
            socket.connect(self.input_pipe)

            out_socket = ctx.socket(zmq.PUSH)
            out_socket.set_hwm(max(self.hwm // self.batch_size, 5))
            out_socket.connect(self.result_pipe)

            batch = []
            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                if dp is not None:
                    batch.append(dp)
                    if len(batch) == self.batch_size:
                        dp = BatchData.aggregate_batch(batch)
                        out_socket.send(dumps(dp), copy=False)
                        del batch[:] 

def run(self):
        player = self._build_player()
        context = zmq.Context()
        c2s_socket = context.socket(zmq.PUSH)
        c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
        c2s_socket.set_hwm(2)
        c2s_socket.connect(self.c2s)

        s2c_socket = context.socket(zmq.DEALER)
        s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
        #s2c_socket.set_hwm(5)
        s2c_socket.connect(self.s2c)

        state = player.current_state()
        reward, isOver = 0, False
        while True:
            c2s_socket.send(dumps(
                (self.identity, state, reward, isOver)),
                copy=False)
            action = loads(s2c_socket.recv(copy=False).bytes)
            reward, isOver = player.action(action)
            state = player.current_state()

# compatibility 

def get(self, option):
        c_data = new_pointer_from_opt(option, length=255)

        c_value_pointer = c_data[0]
        c_sizet_pointer = c_data[1]

        _retry_sys_call(C.zmq_getsockopt,
                        self._zmq_socket,
                        option,
                        c_value_pointer,
                        c_sizet_pointer)
        
        sz = c_sizet_pointer[0]
        v = value_from_opt_pointer(option, c_value_pointer, sz)
        if option != zmq.IDENTITY and option in zmq.constants.bytes_sockopts and v.endswith(b'\0'):
            v = v[:-1]
        return v 

def _setup_sockets(self):
        ins,outs = Device._setup_sockets(self)
        ctx = self._context
        mons = ctx.socket(self.mon_type)
        
        # set sockopts (must be done first, in case of zmq.IDENTITY)
        for opt,value in self._mon_sockopts:
            mons.setsockopt(opt, value)
        
        for iface in self._mon_binds:
            mons.bind(iface)
        
        for iface in self._mon_connects:
            mons.connect(iface)
        
        return ins,outs,mons 

def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()

            # recv jobs
            socket = ctx.socket(zmq.PULL)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm * self.batch_size)
            socket.connect(self.input_pipe)

            # send results
            out_socket = ctx.socket(zmq.PUSH)
            out_socket.set_hwm(max(self.hwm, 5))
            out_socket.connect(self.result_pipe)

            batch = []
            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                if dp is not None:
                    batch.append(dp)
                    if len(batch) == self.batch_size:
                        dp = BatchData.aggregate_batch(batch)
                        out_socket.send(dumps(dp), copy=False)
                        del batch[:] 

def run(self):
            enable_death_signal(_warn=self.identity == b'0')
            ctx = zmq.Context()
            socket = ctx.socket(zmq.REP)
            socket.setsockopt(zmq.IDENTITY, self.identity)
            socket.set_hwm(self.hwm)
            socket.connect(self.pipename)

            while True:
                dp = loads(socket.recv(copy=False))
                dp = self.map_func(dp)
                socket.send(dumps(dp), copy=False) 

def __init__(self, worker_id, address, port, logdir, debug=False, worker_type='RAW'):

        self.worker_id = worker_id
        self.address = address
        self.port = port
        self.logdir = logdir
        self.debug = debug
        self.worker_type = worker_type
        self.serializer = FuncXSerializer()
        self.serialize = self.serializer.serialize
        self.deserialize = self.serializer.deserialize

        global logger
        logger = set_file_logger('{}/funcx_worker_{}.log'.format(logdir, worker_id),
                                 name="worker_log",
                                 level=logging.DEBUG if debug else logging.INFO)

        logger.info('Initializing worker {}'.format(worker_id))
        logger.info('Worker is of type: {}'.format(worker_type))

        if debug:
            logger.debug('Debug logging enabled')

        self.context = zmq.Context()
        self.poller = zmq.Poller()
        self.identity = worker_id.encode()

        self.task_socket = self.context.socket(zmq.DEALER)
        self.task_socket.setsockopt(zmq.IDENTITY, self.identity)

        logger.info('Trying to connect to : tcp://{}:{}'.format(self.address, self.port))
        self.task_socket.connect('tcp://{}:{}'.format(self.address, self.port))
        self.poller.register(self.task_socket, zmq.POLLIN) 

def set_id(zsocket):
    """Set simple random printable identity on socket"""
    identity = u"%04x-%04x" % (randint(0, 0x10000), randint(0, 0x10000))
    zsocket.setsockopt_string(zmq.IDENTITY, identity) 

def test_dir(self):
        ctx = self.Context()
        s = ctx.socket(zmq.PUB)
        self.assertTrue('send' in dir(s))
        self.assertTrue('IDENTITY' in dir(s))
        self.assertTrue('AFFINITY' in dir(s))
        self.assertTrue('FD' in dir(s))
        s.close()
        ctx.term() 

def test_identity(self):
        s = self.context.socket(zmq.PULL)
        self.sockets.append(s)
        ident = b'identity\0\0'
        s.identity = ident
        self.assertEqual(s.get(zmq.IDENTITY), ident) 

def create_zmq_connection(node, sock_type):
    clientstack = node.clientstack
    sock = zmq.Context().socket(sock_type)
    l_pub_key, l_sec_key = zmq.curve_keypair()
    sock.setsockopt(zmq.IDENTITY, base64.encodebytes(l_pub_key))
    sock.setsockopt(zmq.CURVE_PUBLICKEY, l_pub_key)
    sock.setsockopt(zmq.CURVE_SECRETKEY, l_sec_key)
    sock.setsockopt(zmq.TCP_KEEPALIVE, 1)

    sock.setsockopt(zmq.CURVE_SERVERKEY, clientstack.publicKey)
    sock.connect("tcp://{}:{}".format(clientstack.ha.host, clientstack.ha.port))

    return sock 

def check_zmq_connection(addr_port):
    ctx = zmq.Context()
    sock = ctx.socket(zmq.DEALER)
    l_pub_key, l_sec_key = zmq.curve_keypair()
    sock.setsockopt(zmq.IDENTITY, base64.encodebytes(l_pub_key))
    sock.setsockopt(zmq.CURVE_PUBLICKEY, l_pub_key)
    sock.setsockopt(zmq.CURVE_SECRETKEY, l_sec_key)
    dest = get_dest(SERVER_SEED)
    sock.setsockopt(zmq.CURVE_SERVERKEY, z85.encode(ed_25519_pk_to_curve_25519(base58.b58decode(dest))))

    try:
        sock.connect("{}://{}".format(ZMQ_NETWORK_PROTOCOL, addr_port))
        sock.send_string(TEST_MSG)
    except OSError as e:
        print("ZMQ CHECK PHASE::: Cannot connect to {} because\n {}".format(addr_port, e))
        return False

    print("ZMQ CHECK PHASE:::Waiting {} seconds for response from server".format(WAIT_TIMEOUT))

    ready = zmq.select([sock], [], [], WAIT_TIMEOUT)
    if ready[0]:
        reply = sock.recv(flags=zmq.NOBLOCK)
        reply = reply.decode()
        print("ZMQ CHECK PHASE::: Got reply {}".format(reply))

        if reply != EXPECTED_ZMQ_REPLY:
            print("ZMQ CHECK PHASE:::ZMQ connection test failed. \nGot {} \nbut expected reply {}\n".format(reply, EXPECTED_ZMQ_REPLY))
        print("ZMQ CHECK PHASE:::Got expected response")
        return True
    return False 

def run(self):
        """The thread's main activity.  Call start() instead."""
        self.socket = self.context.socket(zmq.SUB)
        self.socket.setsockopt(zmq.SUBSCRIBE,b'')
        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
        self.socket.connect(self.address)
        self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
        self.stream.on_recv(self._handle_recv)
        self._run_loop()
        try:
            self.socket.close()
        except:
            pass 

def run(self):
        """The thread's main activity.  Call start() instead."""
        self.socket = self.context.socket(zmq.DEALER)
        self.socket.setsockopt(zmq.IDENTITY, self.session.bsession)
        self.socket.connect(self.address)
        self.stream = zmqstream.ZMQStream(self.socket, self.ioloop)
        self.stream.on_recv(self._handle_recv)
        self._run_loop()
        try:
            self.socket.close()
        except:
            pass 

def run(self):
        player = self._build_player()
        context = zmq.Context()
        c2s_socket = context.socket(zmq.PUSH)
        c2s_socket.setsockopt(zmq.IDENTITY, self.identity)
        c2s_socket.set_hwm(2)
        c2s_socket.connect(self.c2s)

        s2c_socket = context.socket(zmq.DEALER)
        s2c_socket.setsockopt(zmq.IDENTITY, self.identity)
        #s2c_socket.set_hwm(5)
        s2c_socket.connect(self.s2c)

        state = player.current_state()
        reward, isOver = 0, False
        ts = 0

        while True:
            c2s_socket.send(dumps(
                (self.identity, state, reward, isOver, ts, True)),
                copy=False)
            #t.grel here we get the action
            (action, ts, isAlive) = loads(s2c_socket.recv(copy=False).bytes)
            if not isAlive:
                c2s_socket.send(dumps(
                    (self.identity, 0, 0, 0, 0, False)),
                    copy=False)
                print("closing thread : {}".format(self.identity))
                break

            reward, isOver = player.action(action)
            state = player.current_state()

# compatibility