Python struct.pack_into() Examples

def test_pack_into_fn(self):
        test_string = 'Reykjavik rocks, eow!'
        writable_buf = array.array('c', ' '*100)
        fmt = '21s'
        pack_into = lambda *args: struct.pack_into(fmt, *args)

        # Test without offset.
        pack_into(writable_buf, 0, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)]
        self.assertEqual(from_buf, test_string)

        # Test with offset.
        pack_into(writable_buf, 10, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)+10]
        self.assertEqual(from_buf, test_string[:10] + test_string)

        # Go beyond boundaries.
        small_buf = array.array('c', ' '*10)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0,
                          test_string)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2,
                          test_string) 

def test_pack_into_fn(self):
        test_string = b'Reykjavik rocks, eow!'
        writable_buf = array.array('b', b' '*100)
        fmt = '21s'
        pack_into = lambda *args: struct.pack_into(fmt, *args)

        # Test without offset.
        pack_into(writable_buf, 0, test_string)
        from_buf = writable_buf.tobytes()[:len(test_string)]
        self.assertEqual(from_buf, test_string)

        # Test with offset.
        pack_into(writable_buf, 10, test_string)
        from_buf = writable_buf.tobytes()[:len(test_string)+10]
        self.assertEqual(from_buf, test_string[:10] + test_string)

        # Go beyond boundaries.
        small_buf = array.array('b', b' '*10)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0,
                          test_string)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2,
                          test_string) 

def _write_header(self, buff, api_version=0, correlation_id=0):
        """Write the header for an outgoing message.

        :param buff: The buffer into which to write the header
        :type buff: buffer
        :param api_version: The "kafka api version id", used for feature flagging
        :type api_version: int
        :param correlation_id: This is a user-supplied integer. It will be
            passed back in the response by the server, unmodified. It is useful
            for matching request and response between the client and server.
        :type correlation_id: int
        """
        fmt = '!ihhih%ds' % len(self.CLIENT_ID)
        struct.pack_into(fmt, buff, 0,
                         len(buff) - 4,  # msglen excludes this int
                         self.API_KEY,
                         api_version,
                         correlation_id,
                         len(self.CLIENT_ID),
                         self.CLIENT_ID) 

def pack_into(self, buff, offset):
        """Serialize and write to ``buff`` starting at offset ``offset``.

        Intentionally follows the pattern of ``struct.pack_into``

        :param buff: The buffer to write into
        :param offset: The offset to start the write at
        """
        # NB a length of 0 means an empty string, whereas -1 means null
        len_key = -1 if self.partition_key is None else len(self.partition_key)
        len_value = -1 if self.value is None else len(self.value)
        fmt = '!BBi%dsi%ds' % (max(len_key, 0), max(len_value, 0))
        args = (self.MAGIC,
                self.compression_type,
                len_key,
                self.partition_key or b"",
                len_value,
                self.value or b"")
        struct.pack_into(fmt, buff, offset + 4, *args)
        fmt_size = struct.calcsize(fmt)
        data = buffer(buff[(offset + 4):(offset + 4 + fmt_size)])
        crc = crc32(data) & 0xffffffff
        struct.pack_into('!I', buff, offset, crc) 

def _get_compressed(self):
        """Get a compressed representation of all current messages.

        Returns a Message object with correct headers set and compressed
        data in the value field.
        """
        assert self.compression_type != CompressionType.NONE
        tmp_mset = MessageSet(messages=self._messages)
        uncompressed = bytearray(len(tmp_mset))
        tmp_mset.pack_into(uncompressed, 0)
        if self.compression_type == CompressionType.GZIP:
            compressed = compression.encode_gzip(buffer(uncompressed))
        elif self.compression_type == CompressionType.SNAPPY:
            compressed = compression.encode_snappy(buffer(uncompressed))
        else:
            raise TypeError("Unknown compression: %s" % self.compression_type)
        return Message(compressed, compression_type=self.compression_type) 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        struct.pack_into('!iiii', output, offset,
                         -1, self.timeout, self.min_bytes, len(self._reqs))
        offset += 16
        for topic_name, partitions in iteritems(self._reqs):
            fmt = '!h%dsi' % len(topic_name)
            struct.pack_into(
                fmt, output, offset, len(topic_name), topic_name,
                len(partitions)
            )
            offset += struct.calcsize(fmt)
            for partition_id, (fetch_offset, max_bytes) in iteritems(partitions):
                struct.pack_into('!iqi', output, offset,
                                 partition_id, fetch_offset, max_bytes)
                offset += 16
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        struct.pack_into('!ii', output, offset, -1, len(self._reqs))
        offset += 8
        for topic_name, partitions in iteritems(self._reqs):
            fmt = '!h%dsi' % len(topic_name)
            struct.pack_into(fmt, output, offset, len(topic_name),
                             topic_name, len(partitions))
            offset += struct.calcsize(fmt)
            for pnum, (offsets_before, max_offsets) in iteritems(partitions):
                struct.pack_into('!iqi', output, offset,
                                 pnum, offsets_before, max_offsets)
                offset += 16
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output, api_version=1)
        offset = self.HEADER_LEN
        fmt = '!h%dsi' % len(self.consumer_group)
        struct.pack_into(fmt, output, offset,
                         len(self.consumer_group), self.consumer_group,
                         len(self._reqs))
        offset += struct.calcsize(fmt)
        for topic_name, partitions in iteritems(self._reqs):
            fmt = '!h%dsi' % len(topic_name)
            struct.pack_into(fmt, output, offset, len(topic_name),
                             topic_name, len(partitions))
            offset += struct.calcsize(fmt)
            for pnum in partitions:
                fmt = '!i'
                struct.pack_into(fmt, output, offset, pnum)
                offset += struct.calcsize(fmt)
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        fmt = '!h%dsih%dsh%dsi' % (len(self.group_id), len(self.member_id),
                                   len(self.protocol_type))
        struct.pack_into(fmt, output, offset, len(self.group_id), self.group_id,
                         self.session_timeout, len(self.member_id), self.member_id,
                         len(self.protocol_type), self.protocol_type,
                         len(self.group_protocols))
        offset += struct.calcsize(fmt)
        for protocol_name, protocol_metadata in self.group_protocols:
            fmt = '!h%dsi%ds' % (len(protocol_name), len(protocol_metadata))
            struct.pack_into(fmt, output, offset, len(protocol_name), protocol_name,
                             len(protocol_metadata), protocol_metadata)
            offset += struct.calcsize(fmt)
        return output 

def get_bytes(self):
        output = bytearray(len(self))
        offset = 0
        fmt = '!hi'
        struct.pack_into(fmt, output, offset, self.version,
                         len(self.partition_assignment))
        offset += struct.calcsize(fmt)
        for topic_name, partitions in self.partition_assignment:
            fmt = '!h%dsi' % len(topic_name)
            struct.pack_into(fmt, output, offset, len(topic_name), topic_name,
                             len(partitions))
            offset += struct.calcsize(fmt)
            for partition_id in partitions:
                fmt = '!i'
                struct.pack_into(fmt, output, offset, partition_id)
                offset += struct.calcsize(fmt)
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        fmt = '!h%dsih%dsi' % (len(self.group_id), len(self.member_id))
        struct.pack_into(fmt, output, offset, len(self.group_id), self.group_id,
                         self.generation_id, len(self.member_id), self.member_id,
                         len(self.group_assignment))
        offset += struct.calcsize(fmt)
        for member_assignment in self.group_assignment:
            assignment_bytes = bytes(member_assignment.get_bytes())
            fmt = '!h%dsi%ds' % (len(member_assignment.member_id), len(assignment_bytes))
            struct.pack_into(fmt, output, offset, len(member_assignment.member_id),
                             member_assignment.member_id, len(assignment_bytes),
                             assignment_bytes)
            offset += struct.calcsize(fmt)
        return output 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        cur_offs = baseoffs = header_offs = offs
        cur_offs += _NDARRAY_HEADER_SIZE

        shape = obj.shape
        if len(shape) != 1:
            cur_offs = mapped_tuple.pack_into(shape, buf, cur_offs)
        dtype_offs = cur_offs - baseoffs

        dtype_params = cls._make_dtype_params(obj.dtype)
        if isinstance(dtype_params, basestring):
            dtype_params = _NDARRAY_STANDARD_DTYPES_TO_CODE.get(dtype_params, dtype_params)
        cur_offs = mapped_object.pack_into(dtype_params, buf, cur_offs)
        data_offs = cur_offs - baseoffs

        _NDARRAY_HEADER_PACKER.pack_into(buf, header_offs, dtype_offs, data_offs)
        return proxied_buffer.pack_into(buffer(obj), buf, cur_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        if idmap is not None:
            objid = shared_id(obj)
            idmap[objid] = offs + implicit_offs
            if isinstance(idmap, StrongIdMap):
                widmap = idmap
                widmap.link(objid, obj)

        if not isinstance(obj, (Decimal, cDecimal)):
            obj = cDecimal(obj)

        packer = cls.PACKER
        sign, digits, exponent = obj.as_tuple()
        packer.pack_into(buf, offs, (exponent << 1) | sign)
        offs += packer.size

        return mapped_tuple.pack_into(digits, buf, offs, None, implicit_offs) 

def pack(self, obj, idmap = None, packer = None, padding = None, implicit_offs = 0):
        """
        Pack ``obj`` into a byte array and return the corresponding slice.

        See :meth:`pack_into` for a description of the arguments.
        """
        buf = self._acquire_pack_buffer()
        try:
            for i in xrange(24):
                try:
                    endp = self.pack_into(obj, buf, 0, idmap, packer, padding, implicit_offs)
                    return buf[:endp]
                except (struct.error, IndexError):
                    # Buffer overflow, retry with a bigger buffer
                    # Idmap is probably corrupted beyond hope though :(
                    if len(buf) >= self.max_pack_buffer_size:
                        raise
                    buf.extend(buf)
                    if idmap is not None:
                        idmap.clear()
        finally:
            self._release_pack_buffer(buf) 

def test_pack_into_fn(self):
        test_string = 'Reykjavik rocks, eow!'
        writable_buf = array.array('c', ' '*100)
        fmt = '21s'
        pack_into = lambda *args: struct.pack_into(fmt, *args)

        # Test without offset.
        pack_into(writable_buf, 0, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)]
        self.assertEqual(from_buf, test_string)

        # Test with offset.
        pack_into(writable_buf, 10, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)+10]
        self.assertEqual(from_buf, test_string[:10] + test_string)

        # Go beyond boundaries.
        small_buf = array.array('c', ' '*10)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 0,
                          test_string)
        self.assertRaises((ValueError, struct.error), pack_into, small_buf, 2,
                          test_string) 

def initial_connection(self, data):
        state = self._connection
        state.ssrc = data.get('ssrc')
        state.voice_port = data.get('port')
        packet = bytearray(70)
        struct.pack_into('>I', packet, 0, state.ssrc)
        state.socket.sendto(packet, (state.endpoint_ip, state.voice_port))
        recv = yield from self.loop.sock_recv(state.socket, 70)
        log.debug('received packet in initial_connection: {}'.format(recv))

        # the ip is ascii starting at the 4th byte and ending at the first null
        ip_start = 4
        ip_end = recv.index(0, ip_start)
        state.ip = recv[ip_start:ip_end].decode('ascii')

        # the port is a little endian unsigned short in the last two bytes
        # yes, this is different endianness from everything else
        state.port = struct.unpack_from('<H', recv, len(recv) - 2)[0]

        log.debug('detected ip: {0.ip} port: {0.port}'.format(state))
        yield from self.select_protocol(state.ip, state.port)
        log.info('selected the voice protocol for use') 

def _get_voice_packet(self, data):
        header = bytearray(12)
        nonce = bytearray(24)
        box = nacl.secret.SecretBox(bytes(self.secret_key))

        # Formulate header
        header[0] = 0x80
        header[1] = 0x78
        struct.pack_into('>H', header, 2, self.sequence)
        struct.pack_into('>I', header, 4, self.timestamp)
        struct.pack_into('>I', header, 8, self.ssrc)

        # Copy header to nonce's first 12 bytes
        nonce[:12] = header

        # Encrypt and return the data
        return header + box.encrypt(bytes(data), bytes(nonce)).ciphertext 

def setUp_with_echo(self):
        self.echo_id = 13379
        self.echo_seq = 1
        self.echo_data = b'\x30\x0e\x09\x00\x00\x00\x00\x00' \
            + b'\x10\x11\x12\x13\x14\x15\x16\x17' \
            + b'\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f' \
            + b'\x20\x21\x22\x23\x24\x25\x26\x27' \
            + b'\x28\x29\x2a\x2b\x2c\x2d\x2e\x2f' \
            + b'\x30\x31\x32\x33\x34\x35\x36\x37'
        self.data = icmp.echo(
            id_=self.echo_id, seq=self.echo_seq, data=self.echo_data)

        self.type_ = icmp.ICMP_ECHO_REQUEST
        self.code = 0
        self.ic = icmp.icmp(self.type_, self.code, self.csum, self.data)

        self.buf = bytearray(struct.pack(
            icmp.icmp._PACK_STR, self.type_, self.code, self.csum))
        self.buf += self.data.serialize()
        self.csum_calc = packet_utils.checksum(self.buf)
        struct.pack_into('!H', self.buf, 2, self.csum_calc) 

def setUp_with_dest_unreach(self):
        self.unreach_mtu = 10
        self.unreach_data = b'abc'
        self.unreach_data_len = len(self.unreach_data)
        self.data = icmp.dest_unreach(
            data_len=self.unreach_data_len, mtu=self.unreach_mtu,
            data=self.unreach_data)

        self.type_ = icmp.ICMP_DEST_UNREACH
        self.code = icmp.ICMP_HOST_UNREACH_CODE
        self.ic = icmp.icmp(self.type_, self.code, self.csum, self.data)

        self.buf = bytearray(struct.pack(
            icmp.icmp._PACK_STR, self.type_, self.code, self.csum))
        self.buf += self.data.serialize()
        self.csum_calc = packet_utils.checksum(self.buf)
        struct.pack_into('!H', self.buf, 2, self.csum_calc) 

def serialize(self):
        buf = bytearray(struct.pack(self._PACK_STR, self.type_,
                        self.aux_len, self.num,
                        addrconv.ipv4.text_to_bin(self.address)))
        for src in self.srcs:
            buf.extend(struct.pack('4s', addrconv.ipv4.text_to_bin(src)))
        if 0 == self.num:
            self.num = len(self.srcs)
            struct.pack_into('!H', buf, 2, self.num)
        if self.aux is not None:
            mod = len(self.aux) % 4
            if mod:
                self.aux += bytearray(4 - mod)
                self.aux = six.binary_type(self.aux)
            buf.extend(self.aux)
            if 0 == self.aux_len:
                self.aux_len = len(self.aux) // 4
                struct.pack_into('!B', buf, 1, self.aux_len)
        return six.binary_type(buf) 

def serialize(self, payload=None, prev=None):
        tail = self.serialize_tail()
        self.length = self._HDR_LEN + len(tail)
        head = bytearray(
            struct.pack(self._HDR_PACK_STR, self.version,
                        self.type_, self.length,
                        addrconv.ipv4.text_to_bin(self.router_id),
                        addrconv.ipv4.text_to_bin(self.area_id), 0,
                        self.au_type, self.authentication))
        buf = head + tail
        csum = packet_utils.checksum(buf[:12] + buf[14:16] +
                                     buf[self._HDR_LEN:])
        self.checksum = csum
        struct.pack_into("!H", buf, 12, csum)
        return buf

# alias 

def test_pack_into(self):
        test_string = 'Reykjavik rocks, eow!'
        writable_buf = array.array('c', ' '*100)
        fmt = '21s'
        s = struct.Struct(fmt)

        # Test without offset
        s.pack_into(writable_buf, 0, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)]
        self.assertEqual(from_buf, test_string)

        # Test with offset.
        s.pack_into(writable_buf, 10, test_string)
        from_buf = writable_buf.tostring()[:len(test_string)+10]
        self.assertEqual(from_buf, test_string[:10] + test_string)

        # Go beyond boundaries.
        small_buf = array.array('c', ' '*10)
        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0,
                          test_string)
        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2,
                          test_string)

        # Test bogus offset (issue 3694)
        sb = small_buf
        self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb,
                          None) 

def numpy_array_from_structure(items, fmt, t):
    """Return numpy_array from the tuple returned by rand_structure()"""
    memlen, itemsize, ndim, shape, strides, offset = t
    buf = bytearray(memlen)
    for j, v in enumerate(items):
        struct.pack_into(fmt, buf, j*itemsize, v)
    return numpy_array(buffer=buf, shape=shape, strides=strides,
                       dtype=fmt, offset=offset)


# ======================================================================
#                          memoryview casts
# ====================================================================== 

def test_varargs9_kw(self):
        msg = r"^pack_into\(\) takes no keyword arguments$"
        self.assertRaisesRegex(TypeError, msg, struct.pack_into, x=2) 

def test_pack_into(self):
        test_string = b'Reykjavik rocks, eow!'
        writable_buf = array.array('b', b' '*100)
        fmt = '21s'
        s = struct.Struct(fmt)

        # Test without offset
        s.pack_into(writable_buf, 0, test_string)
        from_buf = writable_buf.tobytes()[:len(test_string)]
        self.assertEqual(from_buf, test_string)

        # Test with offset.
        s.pack_into(writable_buf, 10, test_string)
        from_buf = writable_buf.tobytes()[:len(test_string)+10]
        self.assertEqual(from_buf, test_string[:10] + test_string)

        # Go beyond boundaries.
        small_buf = array.array('b', b' '*10)
        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 0,
                          test_string)
        self.assertRaises((ValueError, struct.error), s.pack_into, small_buf, 2,
                          test_string)

        # Test bogus offset (issue 3694)
        sb = small_buf
        self.assertRaises((TypeError, struct.error), struct.pack_into, b'', sb,
                          None) 

def test_boundary_error_message(self):
        regex = (
            r'pack_into requires a buffer of at least 6 '
            r'bytes for packing 1 bytes at offset 5 '
            r'\(actual buffer size is 1\)'
        )
        with self.assertRaisesRegex(struct.error, regex):
            struct.pack_into('b', bytearray(1), 5, 1) 

def test_boundary_error_message_with_negative_offset(self):
        byte_list = bytearray(10)
        with self.assertRaisesRegex(
                struct.error,
                r'no space to pack 4 bytes at offset -2'):
            struct.pack_into('<I', byte_list, -2, 123)

        with self.assertRaisesRegex(
                struct.error,
                'offset -11 out of range for 10-byte buffer'):
            struct.pack_into('<B', byte_list, -11, 123) 

def test_boundary_error_message_with_large_offset(self):
        # Test overflows cause by large offset and value size (issue 30245)
        regex = (
            r'pack_into requires a buffer of at least ' + str(sys.maxsize + 4) +
            r' bytes for packing 4 bytes at offset ' + str(sys.maxsize) +
            r' \(actual buffer size is 10\)'
        )
        with self.assertRaisesRegex(struct.error, regex):
            struct.pack_into('<I', bytearray(10), sys.maxsize, 1) 

def initial_connection(self, data):
        state = self._connection
        state.ssrc = data['ssrc']
        state.voice_port = data['port']
        state.endpoint_ip = data['ip']

        packet = bytearray(70)
        struct.pack_into('>I', packet, 0, state.ssrc)
        state.socket.sendto(packet, (state.endpoint_ip, state.voice_port))
        recv = await self.loop.sock_recv(state.socket, 70)
        log.debug('received packet in initial_connection: %s', recv)

        # the ip is ascii starting at the 4th byte and ending at the first null
        ip_start = 4
        ip_end = recv.index(0, ip_start)
        state.ip = recv[ip_start:ip_end].decode('ascii')

        state.port = struct.unpack_from('>H', recv, len(recv) - 2)[0]
        log.debug('detected ip: %s port: %s', state.ip, state.port)

        # there *should* always be at least one supported mode (xsalsa20_poly1305)
        modes = [mode for mode in data['modes'] if mode in self._connection.supported_modes]
        log.debug('received supported encryption modes: %s', ", ".join(modes))

        mode = modes[0]
        await self.select_protocol(state.ip, state.port, mode)
        log.info('selected the voice protocol for use (%s)', mode)

        await self.client_connect() 

def _get_voice_packet(self, data):
        header = bytearray(12)

        # Formulate rtp header
        header[0] = 0x80
        header[1] = 0x78
        struct.pack_into('>H', header, 2, self.sequence)
        struct.pack_into('>I', header, 4, self.timestamp)
        struct.pack_into('>I', header, 8, self.ssrc)

        encrypt_packet = getattr(self, '_encrypt_' + self.mode)
        return encrypt_packet(header, data) 

def Connect(self):
    self.socket_mode = UDP_MODE
    self.mac = uuid.getnode()

    # Set up UDP receiver.
    self.udp_rx_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    self.udp_rx_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)

    # Pack multicast group structure correctly.
    mreq = struct.pack('=4sl', socket.inet_aton(MCAST_GRP),socket.INADDR_ANY) 
    
    # Request access to multicast group.
    self.udp_rx_sock.setsockopt(socket.IPPROTO_IP,
                                socket.IP_ADD_MEMBERSHIP, mreq)  
    
    # Bind to all intfs.
    self.udp_rx_sock.bind(('', MCAST_PORT))
    self.udp_rx_sock.settimeout(TIMEOUT)

    # Set up UDP transmitter.
    self.udp_tx_sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    self.udp_tx_sock.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, 255)
  
    # Get the MAC address of the local adapter.
    msg = bytearray(8)
    struct.pack_into('<Q', msg, 0, int(self.mac))
    self.local_mac = ''.join('{:02x}'.format(x) for x in msg[0:6])
    logging.debug('MAC Addr: %s', self.local_mac) 

def write_noop_rec(buf, rec):
    "tot_len", "xid", "len", "info", "rmid", "prev", "crc"
    struct.pack_into("IIIBBLI", buf, 0,
                rec.tot_len,
                rec.xid,
                rec.tot_len - RECORD_HEADER_LEN,
                I["XLOG_NOOP"],
                RM_XLOG_ID,
                rec.prev,
                0)
    struct.pack_into("III", buf, RECORD_HEADER_LEN, 0,0,0)
    crc = crc32.pgcrc32_arr(buf[0:24], init_zeroes=rec.tot_len - RECORD_HEADER_LEN)
    struct.pack_into("I", buf, 24, crc) 

def pack_into(self, buff, offset):
        """Serialize and write to ``buff`` starting at offset ``offset``.

        Intentionally follows the pattern of ``struct.pack_into``

        :param buff: The buffer to write into
        :param offset: The offset to start the write at
        """
        if self.compression_type == CompressionType.NONE:
            messages = self._messages
        else:
            if self._compressed is None:
                self._compressed = self._get_compressed()
            messages = [self._compressed]

        for message in messages:
            mlen = len(message)
            struct.pack_into('!qi', buff, offset, -1, mlen)
            offset += 12
            message.pack_into(buff, offset)
            offset += mlen


##
# Metadata API
## 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        struct.pack_into('!i', output, self.HEADER_LEN, len(self.topics))
        offset = self.HEADER_LEN + 4
        for t in self.topics:
            tlen = len(t)
            struct.pack_into('!h%ds' % tlen, output, offset, tlen, t)
            offset += 2 + tlen
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        cglen = len(self.consumer_group)
        struct.pack_into('!h%ds' % cglen, output, self.HEADER_LEN, cglen,
                         self.consumer_group)
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output, api_version=1)
        offset = self.HEADER_LEN
        fmt = '!h%dsih%dsi' % (len(self.consumer_group), len(self.consumer_id))
        struct.pack_into(fmt, output, offset,
                         len(self.consumer_group), self.consumer_group,
                         self.consumer_group_generation_id,
                         len(self.consumer_id), self.consumer_id,
                         len(self._reqs))

        offset += struct.calcsize(fmt)
        for topic_name, partitions in iteritems(self._reqs):
            fmt = '!h%dsi' % len(topic_name)
            struct.pack_into(fmt, output, offset, len(topic_name),
                             topic_name, len(partitions))
            offset += struct.calcsize(fmt)
            for pnum, (poffset, timestamp, metadata) in iteritems(partitions):
                fmt = '!iqq'
                struct.pack_into(fmt, output, offset,
                                 pnum, poffset, timestamp)
                offset += struct.calcsize(fmt)
                metalen = len(metadata) or -1
                fmt = '!h'
                pack_args = [fmt, output, offset, metalen]
                if metalen != -1:
                    fmt += '%ds' % metalen
                    pack_args = [fmt, output, offset, metalen, metadata]
                struct.pack_into(*pack_args)
                offset += struct.calcsize(fmt)
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        fmt = '!h%dsih%ds' % (len(self.group_id), len(self.member_id))
        struct.pack_into(fmt, output, offset, len(self.group_id), self.group_id,
                         self.generation_id, len(self.member_id), self.member_id)
        offset += struct.calcsize(fmt)
        return output 

def get_bytes(self):
        """Serialize the message

        :returns: Serialized message
        :rtype: :class:`bytearray`
        """
        output = bytearray(len(self))
        self._write_header(output)
        offset = self.HEADER_LEN
        fmt = '!h%dsh%ds' % (len(self.group_id), len(self.member_id))
        struct.pack_into(fmt, output, offset, len(self.group_id), self.group_id,
                         len(self.member_id), self.member_id)
        offset += struct.calcsize(fmt)
        return output 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        # Same format as tuple, only different base type
        return mapped_tuple.pack_into(obj, buf, offs, idmap, implicit_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        return proxied_dict.pack_into(obj, buf, offs, idmap, implicit_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        basepos = cur_offs = offs
        cur_offs += _DICT_HEADER_SIZE
        iobuf = BufferIO(buf, cur_offs)
        cur_offs += cython.cast(cython.Py_ssize_t, ObjectIdMapper.build(
            _enum_keys(obj), iobuf,
            return_mapper=False,
            idmap=idmap,
            implicit_offs=implicit_offs + cur_offs))
        _DICT_HEADER_PACKER.pack_into(buf, basepos, cur_offs - basepos)
        return proxied_list.pack_into([obj[k] for k in obj.iterkeys()], buf, cur_offs, idmap, implicit_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs buffer data from ``obj`` into ``buf``. Does not wrap it in :term:`RTTI`.

        See `mapped_object.pack_into` for argument details.
        """
        cur_offs = offs
        objlen = len(obj)
        _BUFFER_HEADER_PACKER.pack_into(buf, offs, objlen)
        cur_offs += _BUFFER_HEADER_SIZE

        end_offs = cur_offs + objlen
        buf[cur_offs:end_offs] = obj

        return end_offs 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        return mapped_frozenset.pack_into(obj, buf, offs, idmap, implicit_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs a unicode string into ``buf``. Does not wrap it in :term:`RTTI`.

        See `mapped_object.pack_into` for argument details.
        """
        if idmap is not None:
            objid = shared_id(obj)
            idmap[objid] = offs + implicit_offs
            if isinstance(idmap, StrongIdMap):
                widmap = idmap
                widmap.link(objid, obj)

        return mapped_bytes.pack_into(obj.encode("utf8"), buf, offs, None, implicit_offs) 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        if idmap is not None:
            objid = shared_id(obj)
            idmap[objid] = offs + implicit_offs
            if isinstance(idmap, StrongIdMap):
                widmap = idmap
                widmap.link(objid, obj)

        packer = cls.PACKER
        timestamp = int(time.mktime(obj.timetuple()))
        packer.pack_into(buf, offs, (timestamp << 20) + obj.microsecond)

        return offs + packer.size 

def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs an :term:`RTTI`-wrapped value into ``buf`` at offset ``offs``.

        :param obj: The object to be packed. Must be one of the supported types, or an instance
            of a type registered with :meth:`register_schema`.

        :param buf: A writeable buffer onto which the object should be packed. Like a bytearray.

        :param offs: The offset within ``buf`` where to place the object.

        :param idmap: *(optional)* A mapping (dict-like or an instance of :class:`StrongIdMap`) used to
            deduplicate references to recurring objects. If not given, a temporary :class:`StrongIdMap` will
            be constructed for the operation if necessary. See :term:`idmap`.

        :param implicit_offs: *(optional)* The implicit offset of ``buf`` within a larger data structure.
            If either ``buf`` is a slice of a larger buffer or if its contents will be copied onto a larger
            buffer, this should be the starting point of ``buf``, so new entries on the ``idmap`` are
            created with the proper absolute offset. Otherwise, :term:`idmap` mixups are likely to corrupt the
            resulting buffer.

        :return: The offset where writing finished. Further objects can be placed at this offset when
            packing multiple instances.
        """
        if not isinstance(obj, cls):
            obj = cls(obj)
        typecode = obj.typecode
        endp = offs
        if typecode in cls.PACKERS:
            packer, padding = cls.PACKERS[typecode]
            packer.pack_into(buf, offs, typecode, obj.value)
            endp += packer.size + padding
        elif typecode in cls.OBJ_PACKERS:
            cpacker, cpadding = cls.CODE_PACKER
            cpacker.pack_into(buf, offs, typecode)
            endp += cpacker.size + cpadding
            packer = cls.OBJ_PACKERS[typecode][0]
            endp = packer(obj.value, buf, endp, idmap, implicit_offs)
        else:
            raise TypeError("Unsupported type %r: %r" % (typecode, obj.value))
        return endp 

def _buffer_proxy_set(self, obj, code, elem):
        if obj is not None and not (obj.none_bitmap & self.mask):
            struct.pack_into(code, obj.buf, obj.offs + self.offs, elem) 

def _buffer_proxy_cas(self, obj, code, exp_val, new_val):
        # XXX: This is not atomic!
        if obj is not None and not (obj.none_bitmap & self.mask):
            tmp = struct.unpack_from(code, obj.buf, obj.offs + self.offs)[0]
            if tmp == exp_val:
                struct.pack_into(code, obj.buf, obj.offs + self.offs, new_val)
                return True
        return False 

def _buffer_proxy_add(self, obj, code, value):
        if obj is not None and not (obj.none_bitmap & self.mask):
            tmp = struct.unpack_from(code, obj.buf, obj.offs + self.offs)[0]
            struct.pack_into(code, obj.buf, obj.offs + self.offs, tmp + value) 

def pack_into(self, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs ``obj`` with :attr:`schema`.

        See also :meth:`mapped_object.pack_into`
        """
        return self._schema.pack_into(obj, buf, offs, idmap, None, None, implicit_offs)