Python struct.unpack() Examples

def getDouble(self):
        tmp = bytearray(8)
        self.get(tmp)
        # Swap bytes.
        tmp2 = tmp[0]
        tmp[0] = tmp[7]
        tmp[7] = tmp2
        tmp2 = tmp[1]
        tmp[1] = tmp[6]
        tmp[6] = tmp2
        tmp2 = tmp[2]
        tmp[2] = tmp[5]
        tmp[5] = tmp2
        tmp2 = tmp[3]
        tmp[3] = tmp[4]
        tmp[4] = tmp2
        return struct.unpack("d", tmp)[0] 

def _get_terminal_size(fd):
        columns = lines = 0

        try:
            handle = _handles[fd]
            csbi = create_string_buffer(22)
            res = windll.kernel32.GetConsoleScreenBufferInfo(handle, csbi)
            if res:
                res = struct.unpack("hhhhHhhhhhh", csbi.raw)
                left, top, right, bottom = res[5:9]
                columns = right - left + 1
                lines = bottom - top + 1
        except Exception:
            pass

        return terminal_size(columns, lines) 

def recv_data(self):
        data_remaining = struct.unpack("I", self.conn.recv(4))[0]
        if not data_remaining:
            log.debug("no data?!")
            return None
        log.debug("<- recving %d bytes", data_remaining)
        data = []
        while data_remaining:
            recv_bytes = data_remaining if data_remaining < self.SOCK_BUF else self.SOCK_BUF
            data.append(self.conn.recv(recv_bytes))
            data_len = len(data[-1])
            if data_len == 0:
                break
            data_remaining -= data_len
        data = pickle.loads("".join(data))
        if data["cmd"] != self.ACK:
            self.send_ack()
        return data 

def read_compress_mat(fd):
    """ 
        Reference to function Read in CompressMatrix
        Return a numpy ndarray object
    """
    cps_type = read_token(fd)
    print_info(f'\tFollowing matrix type: {cps_type}')
    head = struct.unpack('ffii', fd.read(16))
    print_info(f'\tCompress matrix header: {head}')
    # 8: sizeof PerColHeader
    # head: {min_value, range, num_rows, num_cols}
    num_rows, num_cols = head[2], head[3]
    if cps_type == 'CM':
        remain_size = num_cols * (8 + num_rows)
    elif cps_type == 'CM2':
        remain_size = 2 * num_rows * num_cols
    elif cps_type == 'CM3':
        remain_size = num_rows * num_cols
    else:
        throw_on_error(False, f'Unknown matrix compressing type: {cps_type}')
    # now uncompress it
    compress_data = fd.read(remain_size)
    mat = uncompress(compress_data, cps_type, head)
    return mat 

def run(self):
        print("VEDIO server starts...")
        self.sock.bind(self.ADDR)
        self.sock.listen(1)
        conn, addr = self.sock.accept()
        print("remote VEDIO client success connected...")
        data = "".encode("utf-8")
        payload_size = struct.calcsize("L")
        cv2.namedWindow('Remote', cv2.WINDOW_AUTOSIZE)
        while True:
            while len(data) < payload_size:
                data += conn.recv(81920)
            packed_size = data[:payload_size]
            data = data[payload_size:]
            msg_size = struct.unpack("L", packed_size)[0]
            while len(data) < msg_size:
                data += conn.recv(81920)
            zframe_data = data[:msg_size]
            data = data[msg_size:]
            frame_data = zlib.decompress(zframe_data)
            frame = pickle.loads(frame_data)
            cv2.imshow('Remote', frame)
            if cv2.waitKey(1) & 0xFF == 27:
                break 

def parse_data(path, dataset, flatten):
    if dataset != 'train' and dataset != 't10k':
        raise NameError('dataset must be train or t10k')

    label_file = os.path.join(path, dataset + '-labels-idx1-ubyte')
    with open(label_file, 'rb') as file:
        _, num = struct.unpack(">II", file.read(8))
        labels = np.fromfile(file, dtype=np.int8)  # int8
        new_labels = np.zeros((num, 10))
        new_labels[np.arange(num), labels] = 1

    img_file = os.path.join(path, dataset + '-images-idx3-ubyte')
    with open(img_file, 'rb') as file:
        _, num, rows, cols = struct.unpack(">IIII", file.read(16))
        imgs = np.fromfile(file, dtype=np.uint8).reshape(num, rows, cols)  # uint8
        imgs = imgs.astype(np.float32) / 255.0
        if flatten:
            imgs = imgs.reshape([num, -1])

    return imgs, new_labels 

def _get_data(self):
        if self._train:
            data, label = self._train_data, self._train_label
        else:
            data, label = self._test_data, self._test_label

        namespace = 'gluon/dataset/'+self._namespace
        data_file = download(_get_repo_file_url(namespace, data[0]),
                             path=self._root,
                             sha1_hash=data[1])
        label_file = download(_get_repo_file_url(namespace, label[0]),
                              path=self._root,
                              sha1_hash=label[1])

        with gzip.open(label_file, 'rb') as fin:
            struct.unpack(">II", fin.read(8))
            label = np.frombuffer(fin.read(), dtype=np.uint8).astype(np.int32)

        with gzip.open(data_file, 'rb') as fin:
            struct.unpack(">IIII", fin.read(16))
            data = np.frombuffer(fin.read(), dtype=np.uint8)
            data = data.reshape(len(label), 28, 28, 1)

        self._data = nd.array(data, dtype=data.dtype)
        self._label = label 

def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 

def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 

def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 

def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 

def _binary_to_text():
  reader = open(FLAGS.in_file, 'rb')
  writer = open(FLAGS.out_file, 'w')
  while True:
    len_bytes = reader.read(8)
    if not len_bytes:
      sys.stderr.write('Done reading\n')
      return
    str_len = struct.unpack('q', len_bytes)[0]
    tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
    tf_example = example_pb2.Example.FromString(tf_example_str)
    examples = []
    for key in tf_example.features.feature:
      examples.append('%s=%s' % (key, tf_example.features.feature[key].bytes_list.value[0]))
    writer.write('%s\n' % '\t'.join(examples))
  reader.close()
  writer.close() 

def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 

def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 

def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 

def recv(self):
		messages = []
		message = self.partial_message + self.comms_socket.recv(4096)
		if len(message) == len(self.partial_message):
			self._stop = True

		if len(message) < 2:
			return messages

		while True:
			length = struct.unpack(">H", message[0:2])[0]+2
			if len(message) >= length:
				messages.append(self.transform(self.encryption, message[2:length], 0))
				common.internal_print("{0} read: {1}".format(self.module_short, len(messages[len(messages)-1])), 0, self.verbosity, common.DEBUG)
				self.partial_message = ""
				message = message[length:]
			else:
				self.partial_message = message
				break

			if len(message) < 2:
				self.partial_message = message
				break

		return messages 

def cmh_autotune(self, module, message, additional_data, cm):
		message = message[len(self.CONTROL_AUTOTUNE)+2:]
		# get tune type, requested record type, length and encoding for crafting the answer
		(query_type, RRtype, length, encode_class) = struct.unpack("<BHHH", message[0:7])
		if self.DNS_proto.get_RR_type(RRtype)[0] == None:
			return True
		
		# extra parameters added to be able to response in the proper way
		additional_data = additional_data + (True, self.download_encoding_list[encode_class], self.DNS_proto.get_RR_type(RRtype)[0])		
		if (query_type == 0) or (query_type == 3):
			# record && downstream length discovery
			message = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
		if query_type == 1:
			# A record name length discovery
			message = struct.pack("<i", binascii.crc32(message[7:]))
		if query_type == 2:
			# checking download encoding, echoing back request payload
			message = message[7:]

		module.send(common.CONTROL_CHANNEL_BYTE, self.CONTROL_AUTOTUNE_CLIENT+message, additional_data)
		return True

	# tune control message handler
	# client sets the record type and encodings by calling this
	# server side 

def recv(self):
		message, addr = self.comms_socket.recvfrom(1508)

		identifier = struct.unpack("<H", message[24:26])[0]
		sequence = struct.unpack(">H", message[26:28])[0]

		if message[28:28+len(self.ICMP_prefix)] != self.ICMP_prefix:
			return ("", None, None, None, None)

		message = message[28+len(self.ICMP_prefix):]

		length = struct.unpack(">H", message[0:2])[0]
		if (length+2 != len(message)):
			common.internal_print("Error length mismatch {0} {1}".format(length, len(message)), -1)
			return ("", None, None, None, None)

		message = self.transform(self.get_client_encryption((addr, identifier, 0, 0)), message[2:length+2], 0)
		queue_length = struct.unpack(">B", message[0:1])[0]
		common.internal_print("ICMP read: {0} seq: {1} id: {2}".format(length, sequence, identifier), 0, self.verbosity, common.DEBUG)

		return message[1:], addr, identifier, sequence, queue_length 

def encode(self, bindata):
	    ''' Encode a bytearray to a Base91 string '''
	    b = 0
	    n = 0
	    out = ''
	    for count in range(len(bindata)):
	        byte = bindata[count:count+1]
	        b |= struct.unpack('B', byte)[0] << n
	        n += 8
	        if n>13:
	            v = b & 8191
	            if v > 88:
	                b >>= 13
	                n -= 13
	            else:
	                v = b & 16383
	                b >>= 14
	                n -= 14
	            out += self.base91_alphabet[v % 91] + self.base91_alphabet[v // 91]
	    if n:
	        out += self.base91_alphabet[b % 91]
	        if n>7 or b>90:
	            out += self.base91_alphabet[b // 91]
	    return out 

def get_data_length(self, header, masked, length_type):
		mask = 0
		if masked:
			mask = 4
		header = header[0:len(header)-mask]
		if length_type == 0:
			length = struct.unpack(">BB", header)[1] & 0x7F
			if length > 125:
				return -1
			else:
				return length

		if length_type == 1:
			length = struct.unpack(">HH", header)[1]
			return length

		if length_type == 2:
			length_tmp = struct.unpack(">HII", header)
			return (length_tmp[1] << 32 | length_tmp[2])

		return -1 

def hostnamebin_to_hostname(self, hostnamebin):
		hostname = ""
		i = 0
		length = 0

		while True:
			if len(hostnamebin) > i:
				l = struct.unpack("B",hostnamebin[i:i+1])[0]
				if l > 63:
					length += 2
					break
				if l == 0:
					length += 1
					break
				hostname += hostnamebin[i+1:i+1+l] + "."
				length += l + 1
				i = i + l + 1
			else:
				break

		return (length, hostname) 

def getFloat(self):
        tmp = bytearray(4)
        self.get(tmp)
        # Swap bytes.
        tmp2 = tmp[0]
        tmp[0] = tmp[3]
        tmp[3] = tmp2
        tmp2 = tmp[1]
        tmp[1] = tmp[2]
        tmp[2] = tmp2
        return struct.unpack("f", tmp)[0] 

def _get_terminal_size_windows(self):
        try:
            from ctypes import windll, create_string_buffer

            # stdin handle is -10
            # stdout handle is -11
            # stderr handle is -12
            h = windll.kernel32.GetStdHandle(-12)
            csbi = create_string_buffer(22)
            res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi)
            if res:
                (
                    bufx,
                    bufy,
                    curx,
                    cury,
                    wattr,
                    left,
                    top,
                    right,
                    bottom,
                    maxx,
                    maxy,
                ) = struct.unpack("hhhhHhhhhhh", csbi.raw)
                sizex = right - left + 1
                sizey = bottom - top + 1
                return sizex, sizey
        except:
            pass 

def _get_terminal_size_linux(self):
        def ioctl_GWINSZ(fd):
            try:
                import fcntl
                import termios

                cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234"))
                return cr
            except:
                pass

        cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2)
        if not cr:
            try:
                fd = os.open(os.ctermid(), os.O_RDONLY)
                cr = ioctl_GWINSZ(fd)
                os.close(fd)
            except:
                pass

        if not cr:
            try:
                cr = (os.environ["LINES"], os.environ["COLUMNS"])
            except:
                return None

        return int(cr[1]), int(cr[0]) 

def terminal_size():
            try:
                device = __import__('fcntl').ioctl(0, __import__('termios').TIOCGWINSZ, '\0\0\0\0\0\0\0\0')
            except IOError:
                return _DEFAULT_WIDTH, _DEFAULT_HEIGHT
            height, width = struct.unpack('hhhh', device)[:2]
            return width, height 

def read(self, file, path):
        """Read the content index from file.
        """
        pos, = struct.unpack('<Q', file.read(8))
        if pos == 0:
            raise VergeMLError("Invalid cache file: {}".format(path))
        file.seek(pos)
        self.index, self.meta, self.info = pickle.load(file) 

def read(self, index, n_samples):

        # get the entries as raw bytes from the superclass implementation
        entries = super().read(index, n_samples)

        res = []
        for i, entry in enumerate(entries):
            data, meta = entry
            type_ = self.cnt.info[index+i]

            if isinstance(type_, tuple):
                # If the type is a pair (x,y), deserialize independently
                buf = io.BytesIO(data)

                # First, get the position of the second item from the header
                pos, = struct.unpack('<Q', buf.read(8))

                # Read the first and second item
                data1 = buf.read(pos)
                data2 = buf.read()

                # Then deserialize the independently.
                data1 = self._deserialize(data1, type_[0])
                data2 = self._deserialize(data2, type_[1])

                res.append(((data1, data2), meta))
            else:
                data = self._deserialize(data, type_)
                res.append((data, meta))

        return res 

def _get_terminal_size(fd):
        try:
            res = fcntl.ioctl(fd, termios.TIOCGWINSZ, b"\x00" * 4)
            lines, columns = struct.unpack("hh", res)
        except Exception:
            columns = lines = 0

        return terminal_size(columns, lines) 

def fix_png(data):
    """
    Fix the signature and checksums on a fuzzed PNG image.
    """
    out = [b"\x89PNG\r\n\x1A\n"]
    data = bytes(data[8:])
    chunk = 0
    while len(data) >= 8:
        chunklen = data[:4]
        out.append(chunklen)
        chunklen = struct.unpack("!I", chunklen)[0]
        if chunk == 0:
            chunkname = b"IHDR" # make sure the first tag is correct
        else:
            chunkname = data[4:8]
            #chunkname = bytes(_coerce_ascii(c) for c in data[4:8])
        out.append(chunkname)
        data = data[8:]
        if len(data) < chunklen:
            break
        else:
            chunkdata = data[:chunklen]
            chunkcrc = zlib.crc32(chunkname) & 0xFFFFFFFF
            chunkcrc = zlib.crc32(chunkdata, chunkcrc) & 0xFFFFFFFF
            out.append(chunkdata)
            out.append(struct.pack("!I", chunkcrc))
            data = data[chunklen+4:] # skip the old crc
        chunk += 1
    out.append(data)
    return b"".join(out) 

def _decrypt(self, text, _id, exception=None):
        text = to_binary(text)
        plain_text = self.cipher.decrypt(base64.b64decode(text))
        padding = plain_text[-1]
        content = plain_text[16:-padding]
        xml_length = socket.ntohl(struct.unpack(b"I", content[:4])[0])
        xml_content = to_text(content[4 : xml_length + 4])
        from_id = to_text(content[xml_length + 4 :])
        if from_id != _id:
            exception = exception or Exception
            raise exception()
        return xml_content