Python six.b() Examples

def test_multipublish():
    @mock_server
    def handle(socket, address):
        assert socket.recv(4) == b'  V2'
        assert socket.recv(11) == b'MPUB topic\n'

        size = nsq.unpack_size(socket.recv(4))
        data = socket.recv(size)

        head, data = data[:4], data[4:]
        assert nsq.unpack_size(head) == 2

        for _ in range(2):
            head, data = data[:4], data[4:]
            assert nsq.unpack_size(head) == 3

            head, data = data[:3], data[3:]
            assert head == b'sup'

        assert data == b''

    with handle as server:
        conn = NsqdTCPClient('127.0.0.1', server.server_port)
        conn.connect()
        conn.multipublish('topic', [b'sup', b'sup']) 

def test_kv_recurse(self, consul_port):
        c = consul.Consul(port=consul_port)
        index, data = c.kv.get('foo/', recurse=True)
        assert data is None

        c.kv.put('foo/', None)
        index, data = c.kv.get('foo/', recurse=True)
        assert len(data) == 1

        c.kv.put('foo/bar1', '1')
        c.kv.put('foo/bar2', '2')
        c.kv.put('foo/bar3', '3')
        index, data = c.kv.get('foo/', recurse=True)
        assert [x['Key'] for x in data] == [
            'foo/', 'foo/bar1', 'foo/bar2', 'foo/bar3']
        assert [x['Value'] for x in data] == [
            None, six.b('1'), six.b('2'), six.b('3')] 

def gcc_llvm():
    """
    Detect if the g++ version used is the llvm one or not.

    It don't support all g++ parameters even if it support many of them.

    """
    if gcc_llvm.is_llvm is None:
        try:
            p_out = output_subprocess_Popen([theano.config.cxx, '--version'])
            output = p_out[0] + p_out[1]
        except OSError:
            # Typically means g++ cannot be found.
            # So it is not an llvm compiler.

            # Normally this should not happen as we should not try to
            # compile when g++ is not available. If this happen, it
            # will crash later so supposing it is not llvm is "safe".
            output = b('')
        gcc_llvm.is_llvm = b("llvm") in output
    return gcc_llvm.is_llvm 

def wrapper(self, environ, start_response):
        """Wrap the wsgi application to override some path:

        ``/__application__``: allow to ping the server.

        ``/__file__?__file__={path}``: serve the file found at ``path``
        """
        if '__file__' in environ['PATH_INFO']:
            req = webob.Request(environ)
            resp = webob.Response()
            resp.content_type = 'text/html; charset=UTF-8'
            filename = req.params.get('__file__')
            if os.path.isfile(filename):
                body = open(filename, 'rb').read()
                body = body.replace(six.b('http://localhost/'),
                                    six.b('http://%s/' % req.host))
                resp.body = body
            else:
                resp.status = '404 Not Found'
            return resp(environ, start_response)
        elif '__application__' in environ['PATH_INFO']:
            return webob.Response('server started')(environ, start_response)
        return self.test_app(environ, start_response) 

def squash_commands(commands):
    output = []
    buf = EMPTY

    for c in commands:
        buf = EMPTY.join((buf, SYM_STAR, b(str(len(c))), SYM_CRLF))

        for arg in map(encode, c):
            if len(buf) > 6000 or len(arg) > 6000:
                output.append(
                    EMPTY.join((buf, SYM_DOLLAR, b(str(len(arg))), SYM_CRLF)))
                output.append(arg)
                buf = SYM_CRLF
            else:
                buf = EMPTY.join((buf, SYM_DOLLAR, b(str(len(arg))), SYM_CRLF,
                                  arg, SYM_CRLF))
    output.append(buf)
    return output 

def test_kv_recurse(self, acl_consul):
        c = consul.Consul(port=acl_consul.port, token=acl_consul.token)
        index, data = c.kv.get('foo/', recurse=True)
        assert data is None

        c.kv.put('foo/', None)
        index, data = c.kv.get('foo/', recurse=True)
        assert len(data) == 1

        c.kv.put('foo/bar1', '1')
        c.kv.put('foo/bar2', '2')
        c.kv.put('foo/bar3', '3')
        index, data = c.kv.get('foo/', recurse=True)
        assert [x['Key'] for x in data] == [
            'foo/', 'foo/bar1', 'foo/bar2', 'foo/bar3']
        assert [x['Value'] for x in data] == [
            None, six.b('1'), six.b('2'), six.b('3')]
        c.kv.delete('foo')
        c.kv.delete('foo/')
        c.kv.delete('foo/bar1')
        c.kv.delete('foo/bar2')
        c.kv.delete('foo/bar3') 

def test_kv_missing(self, loop, consul_port):
        c = consul.tornado.Consul(port=consul_port)

        @gen.coroutine
        def main():
            yield c.kv.put('index', 'bump')
            index, data = yield c.kv.get('foo')
            assert data is None
            index, data = yield c.kv.get('foo', index=index)
            assert data['Value'] == six.b('bar')
            loop.stop()

        @gen.coroutine
        def put():
            yield c.kv.put('foo', 'bar')

        loop.add_timeout(time.time() + (2.0 / 100), put)
        loop.run_sync(main) 

def test_session_delete_ttl_renew(self, acl_consul):
        c = consul.Consul(port=acl_consul.port, token=acl_consul.token)

        s = c.session.create(behavior='delete', ttl=20)

        # attempt to renew an unknown session
        pytest.raises(consul.NotFound, c.session.renew, '1' * 36)

        session = c.session.renew(s)
        assert session['Behavior'] == 'delete'
        assert session['TTL'] == '20s'

        # trying out the behavior
        assert c.kv.put('foo', '1', acquire=s) is True
        index, data = c.kv.get('foo')
        assert data['Value'] == six.b('1')

        c.session.destroy(s)
        index, data = c.kv.get('foo')
        assert data is None 

def test_acl_translate(self, acl_consul):
        c = consul.Consul(port=acl_consul.port, token=acl_consul.token)

        payload = """
        agent "" {
            policy = "write"
        }
        """

        translate = c.acl.create_translate(
            payload=payload, token=acl_consul.token)
        assert translate == b'agent_prefix "" {\n  policy = "write"\n}'

        # fixme
        pytest.raises(consul.ConsulException,
                      c.acl.get_translate,
                      c.acl.self()['AccessorID'],
                      acl_consul.token) 

def test_kv_encoding(self, consul_port):
        c = consul.Consul(port=consul_port)

        # test binary
        c.kv.put('foo', struct.pack('i', 1000))
        index, data = c.kv.get('foo')
        assert struct.unpack('i', data['Value']) == (1000,)

        # test unicode
        c.kv.put('foo', u'bar')
        index, data = c.kv.get('foo')
        assert data['Value'] == six.b('bar')

        # test empty-string comes back as `None`
        c.kv.put('foo', '')
        index, data = c.kv.get('foo')
        assert data['Value'] is None

        # test None
        c.kv.put('foo', None)
        index, data = c.kv.get('foo')
        assert data['Value'] is None

        # check unencoded values raises assert * Python3 don't need
        # pytest.raises(AssertionError, c.kv.put, 'foo', {1: 2}) 

def pairwise(iterable):
    """
    from the itertools recipes
    s -> (s0,s1), (s1,s2), (s2, s3), (s3, None)...
    """
    a = next(iterable)
    b = None
    while True:
        b = None
        try:
            b = next(iterable)
        except StopIteration:
            break
        yield a, b
        a = b
    yield a, b 

def _try_flags(flag_list, preambule="", body="",
                   try_run=False, output=False, compiler=None):
        """
        Try to compile a dummy file with these flags.

        Returns True if compilation was successful, False if there
        were errors.

        """
        if not compiler:
            return False

        code = b("""
        %(preambule)s
        int main(int argc, char** argv)
        {
            %(body)s
            return 0;
        }
        """ % locals())
        return Compiler._try_compile_tmp(code, tmp_prefix='try_flags_',
                                         flags=flag_list, try_run=try_run,
                                         output=output, compiler=compiler) 

def test_kv_subscribe(self, loop, consul_port):
        c = consul.tornado.Consul(port=consul_port)

        @gen.coroutine
        def get():
            index, data = yield c.kv.get('foo')
            assert data is None
            index, data = yield c.kv.get('foo', index=index)
            assert data['Value'] == six.b('bar')
            loop.stop()

        @gen.coroutine
        def put():
            response = yield c.kv.put('foo', 'bar')
            assert response is True

        loop.add_timeout(time.time() + (1.0 / 100), put)
        loop.run_sync(get) 

def test_negotiation():
    @mock_server
    def handle(socket, address):
        assert socket.recv(4) == b'  V2'
        assert socket.recv(9) == b'IDENTIFY\n'

        size = nsq.unpack_size(socket.recv(4))
        data = json.loads(socket.recv(size).decode('utf-8'))

        assert 'gnsq' in data['user_agent']
        resp = six.b(json.dumps({'test': 42}))
        socket.sendall(mock_response(nsq.FRAME_TYPE_RESPONSE, resp))

    with handle as server:
        conn = NsqdTCPClient('127.0.0.1', server.server_port)
        conn.connect()

        assert conn.identify()['test'] == 42 

def test_identify():
    @mock_server
    def handle(socket, address):
        assert socket.recv(4) == b'  V2'
        assert socket.recv(9) == b'IDENTIFY\n'

        size = nsq.unpack_size(socket.recv(4))
        data = json.loads(socket.recv(size).decode('utf-8'))

        assert 'gnsq' in data['user_agent']
        socket.sendall(mock_response(nsq.FRAME_TYPE_RESPONSE, b'OK'))

    with handle as server:
        conn = NsqdTCPClient('127.0.0.1', server.server_port)
        conn.connect()

        assert conn.identify() is None 

def test_kv_subscribe(self, loop, consul_port):
        async def get():
            c = consul.aio.Consul(port=consul_port, loop=loop)
            fut = asyncio.ensure_future(put(), loop=loop)
            index, data = await c.kv.get('foo')
            assert data is None
            index, data = await c.kv.get('foo', index=index)
            assert data['Value'] == six.b('bar')
            await fut

        async def put():
            c = consul.aio.Consul(port=consul_port, loop=loop)
            await asyncio.sleep(1.0 / 100, loop=loop)
            response = await c.kv.put('foo', 'bar')
            assert response is True

        loop.run_until_complete(get()) 

def request(self, callback, method, url, **kwargs):
        if 'data' in kwargs and not isinstance(kwargs['data'], bytes):
            # python2/3 compatibility
            data = kwargs.pop('data')
            kwargs['data'] = data.encode(encoding='utf-8') \
                if hasattr(data, 'encode') else b(data)

        try:
            response = yield self.client.request(method, url, **kwargs)
            parsed = yield self._get_resp(response)
            returnValue(callback(self.response(*parsed)))
        except ConnectError as e:
            raise ConsulException(
                '{}: {}'.format(e.__class__.__name__, e.message))
        except ResponseNeverReceived:
            # this exception is raised if the connection to the server is lost
            # when yielding a response, this could be due to network issues or
            # server restarts
            raise ConsulException(
                'Server connection lost: {} {}'.format(method.upper(), url))
        except RequestTransmissionFailed:
            # this exception is expected if the reactor is stopped mid request
            raise ConsulException(
                'Request incomplete: {} {}'.format(method.upper(), url)) 

def test_disconnected():
    @mock_server
    def handle(socket, address):
        assert socket.recv(4) == b'  V2'
        assert socket.recv(1) == b''

    with handle as server:
        conn = NsqdTCPClient('127.0.0.1', server.server_port)
        conn.connect()
        conn.close_stream()
        assert conn.state == states.DISCONNECTED

        with pytest.raises(errors.NSQSocketError):
            conn.nop()

        with pytest.raises(errors.NSQSocketError):
            conn.read_response() 

def _test_instance_get_operation(self, operation, getter, key, utcnow):
        self.set_mock_db_items(fakes.DB_INSTANCE_2)
        os_instance_2 = fakes.OSInstance(fakes.OS_INSTANCE_2)
        self.nova.servers.get.return_value = os_instance_2
        getter.return_value = 'fake_data'
        utcnow.return_value = datetime.datetime(2015, 1, 19, 23, 34, 45, 123)
        resp = self.execute(operation,
                            {'InstanceId': fakes.ID_EC2_INSTANCE_2})
        expected_data = (base64.b64encode(six.b(getter.return_value))
                               .decode("utf-8"))
        self.assertEqual({'instanceId': fakes.ID_EC2_INSTANCE_2,
                          'timestamp': '2015-01-19T23:34:45.000Z',
                          key: expected_data},
                         resp)
        self.db_api.get_item_by_id.assert_called_once_with(
            mock.ANY, fakes.ID_EC2_INSTANCE_2)
        self.nova.servers.get.assert_called_once_with(fakes.ID_OS_INSTANCE_2)
        getter.assert_called_once_with(os_instance_2) 

def test_authentication_start_pack():
    username = 'user123'
    packed = TACACSAuthenticationStart(
        username,
        TAC_PLUS_AUTHEN_TYPE_ASCII
    ).packed
    assert packed == six.b(
        '\x01'           # tacacs_plus.flags.TAC_PLUS_AUTHEN_LOGIN \
        '\x00'           # tacacs_plus.flags.TAC_PLUS_PRIV_LVL_MIN \
        '\x01'           # tacacs_plus.flags.TAC_PLUS_AUTHEN_TYPE_ASCII \
        '\x01'           # tacacs_plus.flags.TAC_PLUS_AUTHEN_SVC_LOGIN \
        '\x07'           # username_len == 7 \
        '\x0b'           # port_len == 11 \
        '\x0d'           # rem_addr_len == 13 \
        '\x00'           # data_len == 0 \
        'user123'        # username \
        'python_tty0'    # port \
        'python_device'  # rem_addr
    ) 

def drain_stream(stream, decode='utf-8'):
    content = six.b('')

    while True:
        try:
            # read() needs to block
            # buf = os.read(buffer.fileno(), 4096)
            buf = stream.read(1024)
            if buf == six.b(''):
                break

            content += buf
        except Exception:
            break

    if decode:
        return content.decode(decode)

    return content 

def test_authenticate_pap(fake_socket, packets):
    """
    client -> AUTHSTART (user+pass)
              STATUS_PASS              <- server
    """
    client = TACACSClient('127.0.0.1', 49, None, session_id=12345)
    client._sock = fake_socket
    reply = client.authenticate('username', 'pass',
                                authen_type=TAC_PLUS_AUTHEN_TYPE_PAP)
    assert reply.valid

    fake_socket.buff.seek(0)
    first_header = TACACSHeader.unpacked(fake_socket.buff.read(12))
    assert (first_header.version_max, first_header.version_min) == (12, 1)
    first_body = fake_socket.buff.read(first_header.length)
    assert TACACSAuthenticationStart(
        'username',
        TAC_PLUS_AUTHEN_TYPE_PAP,
        data=six.b('pass')
    ).packed == first_body 

def unpacked(cls, raw):
        #  1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8
        # +----------------+----------------+----------------+----------------+
        # |         server_msg len          |            data_len             |
        # +----------------+----------------+----------------+----------------+
        # |     status     |         server_msg ...
        # +----------------+----------------+----------------+----------------+
        # |     data ...
        # +----------------+

        # B = unsigned char
        # !H = network-order (big-endian) unsigned short
        raw = six.BytesIO(raw)
        server_msg_len, data_len = struct.unpack('!HH', raw.read(4))
        status = struct.unpack('B', raw.read(1))[0]
        server_msg = raw.read(server_msg_len)
        data = raw.read(data_len) if data_len else b''
        return cls(status, server_msg, data) 

def test_authorize_chap(fake_socket, packets):
    client = TACACSClient('127.0.0.1', 49, None, session_id=12345)
    client._sock = fake_socket
    reply = client.authorize('username', arguments=[b"service=shell", b"cmd=show", b"cmdargs=version"],
                             authen_type=TAC_PLUS_AUTHEN_TYPE_CHAP)
    assert reply.valid

    fake_socket.buff.seek(0)
    first_header = TACACSHeader.unpacked(fake_socket.buff.read(12))
    assert (first_header.version_max, first_header.version_min) == (12, 0)
    first_body = fake_socket.buff.read(first_header.length)
    assert TACACSAuthorizationStart(
        'username', TAC_PLUS_AUTHEN_METH_TACACSPLUS, TAC_PLUS_PRIV_LVL_MIN,
        TAC_PLUS_AUTHEN_TYPE_CHAP, [b"service=shell", b"cmd=show", b"cmdargs=version"],
    ).packed == first_body


# test client.account 

def packed(self):
        # 1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8  1 2 3 4 5 6 7 8
        #
        # +----------------+----------------+----------------+----------------+
        # |          user_msg len           |            data len             |
        # +----------------+----------------+----------------+----------------+
        # |     flags      |  user_msg ...
        # +----------------+----------------+----------------+----------------+
        # |    data ...
        # +----------------+

        # B = unsigned char
        # !H = network-order (big-endian) unsigned short
        # s = char[]
        password = six.b(self.password)
        data = self.data
        return (
            struct.pack('!H', len(password)) +
            struct.pack('!H', len(data)) +
            struct.pack('B', self.flags) +
            struct.pack('%ds' % len(password), password) +
            struct.pack('%ds' % len(data), data)
        ) 

def __init__(self, password, data=six.b(''), flags=0):
        self.password = password
        self.data = data
        self.flags = flags 

def setup_git_ssh(config):
    import tempfile
    ssh_command = config['ssh']
    ssh_command += ' -p ' + str(config['ssh_port'])
    ssh_command += ' -o StrictHostKeyChecking=no'

    ssh_key = None
    if config['ssh_key_base64']:
        ssh_key = tempfile.NamedTemporaryFile(delete=False, prefix='ssh_key_')
        ssh_key.write(six.b(config['ssh_key_base64']))
        ssh_key.close()
        ssh_command += ' -i '+ ssh_key.name
        os.chmod(ssh_key.name, 0o600)
    # elif config['ssh_key']:
    #     ssh_command += ' -i '+ os.path.expanduser(config['ssh_key'])

    ssh_script = tempfile.NamedTemporaryFile(delete=False, prefix='git_ssh_')
    ssh_script.write(six.b(ssh_command + ' "$@"'))
    ssh_script.close()
    os.environ['GIT_SSH'] = ssh_script.name
    os.chmod(ssh_script.name, 0o700)

    def delete():
        if os.path.exists(ssh_script.name):
            os.unlink(ssh_script.name)

        if ssh_key and os.path.exists(ssh_key.name):
            os.unlink(ssh_key.name)

    return delete 

def test_zlib(serde):
    # This test is really about ensuring that binary data isn't corrupted
    data = six.b('compress me')
    compressed = zlib.compress(data)
    assert zlib.decompress(dumps_loads(serde, compressed)) == data 

def requeue(message_id, timeout=0):
    if not isinstance(timeout, int):
        raise TypeError('requeue timeout must be an integer')
    return _command(REQ, None, message_id, six.b('{}'.format(timeout))) 

def write(self, message):
        try:
            self.lock.acquire()

            if b'' == message:
                return

            if hasattr(message, 'decode'):
                # don't decode string again
                # necessary for Python3
                message = message.decode('utf-8', errors='replace')

            self.logger.write(message)
            self.logger.flush()

            self.last_messages += message
            if len(self.last_messages) > 20 * 1024:
                self.last_messages = self.last_messages[-20 * 1024:]

            if not self.buffer_disabled:
                for char in message:
                    if '\b' == char and self.buffer:
                        self.buffer = self.buffer[:-1]
                    else:
                        self.buffer += char

                if not self.last_timer:
                    self.last_timer = Timer(1.0, self.send_buffer)
                    self.last_timer.start()

        except (KeyboardInterrupt, SystemExit):
            raise
        except Exception:
            sys.__stderr__.write(traceback.format_exc() + '\n')
            sys.__stderr__.flush()
        finally:
            if self.lock.locked():
                self.lock.release()