Python struct.unpack() Examples

The following are 30 code examples of struct.unpack(). You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example. You may also want to check out all available functions/classes of the module struct , or try the search function .
Example #1
Source File: vachat.py    From The-chat-room with MIT License 10 votes vote down vote up
def run(self):
        print("VEDIO server starts...")
        self.sock.bind(self.ADDR)
        self.sock.listen(1)
        conn, addr = self.sock.accept()
        print("remote VEDIO client success connected...")
        data = "".encode("utf-8")
        payload_size = struct.calcsize("L")
        cv2.namedWindow('Remote', cv2.WINDOW_AUTOSIZE)
        while True:
            while len(data) < payload_size:
                data += conn.recv(81920)
            packed_size = data[:payload_size]
            data = data[payload_size:]
            msg_size = struct.unpack("L", packed_size)[0]
            while len(data) < msg_size:
                data += conn.recv(81920)
            zframe_data = data[:msg_size]
            data = data[msg_size:]
            frame_data = zlib.decompress(zframe_data)
            frame = pickle.loads(frame_data)
            cv2.imshow('Remote', frame)
            if cv2.waitKey(1) & 0xFF == 27:
                break 
Example #2
Source File: utils.py    From deep-learning-note with MIT License 9 votes vote down vote up
def parse_data(path, dataset, flatten):
    if dataset != 'train' and dataset != 't10k':
        raise NameError('dataset must be train or t10k')

    label_file = os.path.join(path, dataset + '-labels-idx1-ubyte')
    with open(label_file, 'rb') as file:
        _, num = struct.unpack(">II", file.read(8))
        labels = np.fromfile(file, dtype=np.int8)  # int8
        new_labels = np.zeros((num, 10))
        new_labels[np.arange(num), labels] = 1

    img_file = os.path.join(path, dataset + '-images-idx3-ubyte')
    with open(img_file, 'rb') as file:
        _, num, rows, cols = struct.unpack(">IIII", file.read(16))
        imgs = np.fromfile(file, dtype=np.uint8).reshape(num, rows, cols)  # uint8
        imgs = imgs.astype(np.float32) / 255.0
        if flatten:
            imgs = imgs.reshape([num, -1])

    return imgs, new_labels 
Example #3
Source File: DNS.py    From XFLTReaT with MIT License 8 votes vote down vote up
def cmh_autotune(self, module, message, additional_data, cm):
		message = message[len(self.CONTROL_AUTOTUNE)+2:]
		# get tune type, requested record type, length and encoding for crafting the answer
		(query_type, RRtype, length, encode_class) = struct.unpack("<BHHH", message[0:7])
		if self.DNS_proto.get_RR_type(RRtype)[0] == None:
			return True
		
		# extra parameters added to be able to response in the proper way
		additional_data = additional_data + (True, self.download_encoding_list[encode_class], self.DNS_proto.get_RR_type(RRtype)[0])		
		if (query_type == 0) or (query_type == 3):
			# record && downstream length discovery
			message = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
		if query_type == 1:
			# A record name length discovery
			message = struct.pack("<i", binascii.crc32(message[7:]))
		if query_type == 2:
			# checking download encoding, echoing back request payload
			message = message[7:]

		module.send(common.CONTROL_CHANNEL_BYTE, self.CONTROL_AUTOTUNE_CLIENT+message, additional_data)
		return True

	# tune control message handler
	# client sets the record type and encodings by calling this
	# server side 
Example #4
Source File: ja3.py    From ja3 with BSD 3-Clause "New" or "Revised" License 7 votes vote down vote up
def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 
Example #5
Source File: mnist_loader.py    From Handwritten-Digit-Recognition-using-Deep-Learning with MIT License 6 votes vote down vote up
def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 
Example #6
Source File: _io_kernel.py    From kaldi-python-io with Apache License 2.0 6 votes vote down vote up
def read_compress_mat(fd):
    """ 
        Reference to function Read in CompressMatrix
        Return a numpy ndarray object
    """
    cps_type = read_token(fd)
    print_info(f'\tFollowing matrix type: {cps_type}')
    head = struct.unpack('ffii', fd.read(16))
    print_info(f'\tCompress matrix header: {head}')
    # 8: sizeof PerColHeader
    # head: {min_value, range, num_rows, num_cols}
    num_rows, num_cols = head[2], head[3]
    if cps_type == 'CM':
        remain_size = num_cols * (8 + num_rows)
    elif cps_type == 'CM2':
        remain_size = 2 * num_rows * num_cols
    elif cps_type == 'CM3':
        remain_size = num_rows * num_cols
    else:
        throw_on_error(False, f'Unknown matrix compressing type: {cps_type}')
    # now uncompress it
    compress_data = fd.read(remain_size)
    mat = uncompress(compress_data, cps_type, head)
    return mat 
Example #7
Source File: ja3s.py    From ja3 with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 
Example #8
Source File: ja3.py    From ja3 with BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def parse_variable_array(buf, byte_len):
    """Unpack data from buffer of specific length.

    :param buf: Buffer to operate on
    :type buf: bytes
    :param byte_len: Length to process
    :type byte_len: int
    :returns: bytes, int
    """
    _SIZE_FORMATS = ['!B', '!H', '!I', '!I']
    assert byte_len <= 4
    size_format = _SIZE_FORMATS[byte_len - 1]
    padding = b'\x00' if byte_len == 3 else b''
    size = struct.unpack(size_format, padding + buf[:byte_len])[0]
    data = buf[byte_len:byte_len + size]

    return data, size + byte_len 
Example #9
Source File: data_convert_example.py    From DOTA_models with Apache License 2.0 6 votes vote down vote up
def _binary_to_text():
  reader = open(FLAGS.in_file, 'rb')
  writer = open(FLAGS.out_file, 'w')
  while True:
    len_bytes = reader.read(8)
    if not len_bytes:
      sys.stderr.write('Done reading\n')
      return
    str_len = struct.unpack('q', len_bytes)[0]
    tf_example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
    tf_example = example_pb2.Example.FromString(tf_example_str)
    examples = []
    for key in tf_example.features.feature:
      examples.append('%s=%s' % (key, tf_example.features.feature[key].bytes_list.value[0]))
    writer.write('%s\n' % '\t'.join(examples))
  reader.close()
  writer.close() 
Example #10
Source File: TCP_generic.py    From XFLTReaT with MIT License 6 votes vote down vote up
def recv(self):
		messages = []
		message = self.partial_message + self.comms_socket.recv(4096)
		if len(message) == len(self.partial_message):
			self._stop = True

		if len(message) < 2:
			return messages

		while True:
			length = struct.unpack(">H", message[0:2])[0]+2
			if len(message) >= length:
				messages.append(self.transform(self.encryption, message[2:length], 0))
				common.internal_print("{0} read: {1}".format(self.module_short, len(messages[len(messages)-1])), 0, self.verbosity, common.DEBUG)
				self.partial_message = ""
				message = message[length:]
			else:
				self.partial_message = message
				break

			if len(message) < 2:
				self.partial_message = message
				break

		return messages 
Example #11
Source File: mnist_loader.py    From Handwritten-Digit-Recognition-using-Deep-Learning with MIT License 6 votes vote down vote up
def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 
Example #12
Source File: ICMP.py    From XFLTReaT with MIT License 6 votes vote down vote up
def recv(self):
		message, addr = self.comms_socket.recvfrom(1508)

		identifier = struct.unpack("<H", message[24:26])[0]
		sequence = struct.unpack(">H", message[26:28])[0]

		if message[28:28+len(self.ICMP_prefix)] != self.ICMP_prefix:
			return ("", None, None, None, None)

		message = message[28+len(self.ICMP_prefix):]

		length = struct.unpack(">H", message[0:2])[0]
		if (length+2 != len(message)):
			common.internal_print("Error length mismatch {0} {1}".format(length, len(message)), -1)
			return ("", None, None, None, None)

		message = self.transform(self.get_client_encryption((addr, identifier, 0, 0)), message[2:length+2], 0)
		queue_length = struct.unpack(">B", message[0:1])[0]
		common.internal_print("ICMP read: {0} seq: {1} id: {2}".format(length, sequence, identifier), 0, self.verbosity, common.DEBUG)

		return message[1:], addr, identifier, sequence, queue_length 
Example #13
Source File: mnist_loader.py    From Handwritten-Digit-Recognition-using-Deep-Learning with MIT License 6 votes vote down vote up
def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 
Example #14
Source File: mnist_loader.py    From Handwritten-Digit-Recognition-using-Deep-Learning with MIT License 6 votes vote down vote up
def load(cls, path_img, path_lbl):
        with open(path_lbl, 'rb') as file:
            magic, size = struct.unpack(">II", file.read(8))
            if magic != 2049:
                raise ValueError('Magic number mismatch, expected 2049,'
                                 'got {}'.format(magic))

            labels = array("B", file.read())

        with open(path_img, 'rb') as file:
            magic, size, rows, cols = struct.unpack(">IIII", file.read(16))
            if magic != 2051:
                raise ValueError('Magic number mismatch, expected 2051,'
                                 'got {}'.format(magic))

            image_data = array("B", file.read())

        images = []
        for i in range(size):
            images.append([0] * rows * cols)

        for i in range(size):
            images[i][:] = image_data[i * rows * cols:(i + 1) * rows * cols]

        return images, labels 
Example #15
Source File: SockPuppet.py    From ALF with Apache License 2.0 6 votes vote down vote up
def recv_data(self):
        data_remaining = struct.unpack("I", self.conn.recv(4))[0]
        if not data_remaining:
            log.debug("no data?!")
            return None
        log.debug("<- recving %d bytes", data_remaining)
        data = []
        while data_remaining:
            recv_bytes = data_remaining if data_remaining < self.SOCK_BUF else self.SOCK_BUF
            data.append(self.conn.recv(recv_bytes))
            data_len = len(data[-1])
            if data_len == 0:
                break
            data_remaining -= data_len
        data = pickle.loads("".join(data))
        if data["cmd"] != self.ACK:
            self.send_ack()
        return data 
Example #16
Source File: encoding.py    From XFLTReaT with MIT License 6 votes vote down vote up
def encode(self, bindata):
	    ''' Encode a bytearray to a Base91 string '''
	    b = 0
	    n = 0
	    out = ''
	    for count in range(len(bindata)):
	        byte = bindata[count:count+1]
	        b |= struct.unpack('B', byte)[0] << n
	        n += 8
	        if n>13:
	            v = b & 8191
	            if v > 88:
	                b >>= 13
	                n -= 13
	            else:
	                v = b & 16383
	                b >>= 14
	                n -= 14
	            out += self.base91_alphabet[v % 91] + self.base91_alphabet[v // 91]
	    if n:
	        out += self.base91_alphabet[b % 91]
	        if n>7 or b>90:
	            out += self.base91_alphabet[b // 91]
	    return out 
Example #17
Source File: websocket_proto.py    From XFLTReaT with MIT License 6 votes vote down vote up
def get_data_length(self, header, masked, length_type):
		mask = 0
		if masked:
			mask = 4
		header = header[0:len(header)-mask]
		if length_type == 0:
			length = struct.unpack(">BB", header)[1] & 0x7F
			if length > 125:
				return -1
			else:
				return length

		if length_type == 1:
			length = struct.unpack(">HH", header)[1]
			return length

		if length_type == 2:
			length_tmp = struct.unpack(">HII", header)
			return (length_tmp[1] << 32 | length_tmp[2])

		return -1 
Example #18
Source File: datasets.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 6 votes vote down vote up
def _get_data(self):
        if self._train:
            data, label = self._train_data, self._train_label
        else:
            data, label = self._test_data, self._test_label

        namespace = 'gluon/dataset/'+self._namespace
        data_file = download(_get_repo_file_url(namespace, data[0]),
                             path=self._root,
                             sha1_hash=data[1])
        label_file = download(_get_repo_file_url(namespace, label[0]),
                              path=self._root,
                              sha1_hash=label[1])

        with gzip.open(label_file, 'rb') as fin:
            struct.unpack(">II", fin.read(8))
            label = np.frombuffer(fin.read(), dtype=np.uint8).astype(np.int32)

        with gzip.open(data_file, 'rb') as fin:
            struct.unpack(">IIII", fin.read(16))
            data = np.frombuffer(fin.read(), dtype=np.uint8)
            data = data.reshape(len(label), 28, 28, 1)

        self._data = nd.array(data, dtype=data.dtype)
        self._label = label 
Example #19
Source File: dns_proto.py    From XFLTReaT with MIT License 6 votes vote down vote up
def hostnamebin_to_hostname(self, hostnamebin):
		hostname = ""
		i = 0
		length = 0

		while True:
			if len(hostnamebin) > i:
				l = struct.unpack("B",hostnamebin[i:i+1])[0]
				if l > 63:
					length += 2
					break
				if l == 0:
					length += 1
					break
				hostname += hostnamebin[i+1:i+1+l] + "."
				length += l + 1
				i = i + l + 1
			else:
				break

		return (length, hostname) 
Example #20
Source File: GXByteBuffer.py    From Gurux.DLMS.Python with GNU General Public License v2.0 6 votes vote down vote up
def getDouble(self):
        tmp = bytearray(8)
        self.get(tmp)
        # Swap bytes.
        tmp2 = tmp[0]
        tmp[0] = tmp[7]
        tmp[7] = tmp2
        tmp2 = tmp[1]
        tmp[1] = tmp[6]
        tmp[6] = tmp2
        tmp2 = tmp[2]
        tmp[2] = tmp[5]
        tmp[5] = tmp2
        tmp2 = tmp[3]
        tmp[3] = tmp[4]
        tmp[4] = tmp2
        return struct.unpack("d", tmp)[0] 
Example #21
Source File: websocket_proto.py    From XFLTReaT with MIT License 5 votes vote down vote up
def get_length_type(self, header):
		if len(header) != 2:
			return -1

		length = struct.unpack(">BB", header)[1] & 0x7F
		if length < 126:
			return 0
		if length == 126:
			return 1
		if length == 127:
			return 2

		return -1 
Example #22
Source File: dns_proto.py    From XFLTReaT with MIT License 5 votes vote down vote up
def get_packet_number_from_header(self, header):
		header_num = struct.unpack(">H", header)[0]

		return (header_num >> 5) & 0x3FF 
Example #23
Source File: dns_proto.py    From XFLTReaT with MIT License 5 votes vote down vote up
def get_userid_from_header(self, header):
		header_num = struct.unpack(">H", header)[0]

		return (header_num >> 12) & 0x07 
Example #24
Source File: dns_proto.py    From XFLTReaT with MIT License 5 votes vote down vote up
def get_fragment_number_from_header(self, header):
		header_num = struct.unpack(">H", header)[0]

		return (header_num >> 1) & 0x0F 
Example #25
Source File: quadruped_playback.py    From soccer-matlab with BSD 2-Clause "Simplified" License 5 votes vote down vote up
def readLogFile(filename, verbose = True):
  f = open(filename, 'rb')
  
  print('Opened'),
  print(filename)

  keys = f.readline().decode('utf8').rstrip('\n').split(',')
  fmt = f.readline().decode('utf8').rstrip('\n')
  
  # The byte number of one record
  sz = struct.calcsize(fmt)
  # The type number of one record
  ncols = len(fmt)

  if verbose:
    print('Keys:'),
    print(keys)
    print('Format:'),
    print(fmt)
    print('Size:'),
    print(sz)
    print('Columns:'),
    print(ncols)

  # Read data
  wholeFile = f.read()
  # split by alignment word
  chunks = wholeFile.split(b'\xaa\xbb')
  print ("num chunks")
  print (len(chunks))
  
  log = list()
  for chunk in chunks:
    if len(chunk) == sz:
      values = struct.unpack(fmt, chunk)
      record = list()
      for i in range(ncols):
        record.append(values[i])
      log.append(record)

  return log 
Example #26
Source File: ja3.py    From ja3 with BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def ntoh(buf):
    """Convert to network order.

    :param buf: Bytes to convert
    :type buf: bytearray
    :returns: int
    """
    if len(buf) == 1:
        return buf[0]
    elif len(buf) == 2:
        return struct.unpack('!H', buf)[0]
    elif len(buf) == 4:
        return struct.unpack('!I', buf)[0]
    else:
        raise ValueError('Invalid input buffer size for NTOH') 
Example #27
Source File: data.py    From DOTA_models with Apache License 2.0 5 votes vote down vote up
def ExampleGen(data_path, num_epochs=None):
  """Generates tf.Examples from path of data files.

    Binary data format: <length><blob>. <length> represents the byte size
    of <blob>. <blob> is serialized tf.Example proto. The tf.Example contains
    the tokenized article text and summary.

  Args:
    data_path: path to tf.Example data files.
    num_epochs: Number of times to go through the data. None means infinite.

  Yields:
    Deserialized tf.Example.

  If there are multiple files specified, they accessed in a random order.
  """
  epoch = 0
  while True:
    if num_epochs is not None and epoch >= num_epochs:
      break
    filelist = glob.glob(data_path)
    assert filelist, 'Empty filelist.'
    random.shuffle(filelist)
    for f in filelist:
      reader = open(f, 'rb')
      while True:
        len_bytes = reader.read(8)
        if not len_bytes: break
        str_len = struct.unpack('q', len_bytes)[0]
        example_str = struct.unpack('%ds' % str_len, reader.read(str_len))[0]
        yield example_pb2.Example.FromString(example_str)

    epoch += 1 
Example #28
Source File: test_utils.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 5 votes vote down vote up
def get_mnist():
    """Download and load the MNIST dataset

    Returns
    -------
    dict
        A dict containing the data
    """
    def read_data(label_url, image_url):
        with gzip.open(mx.test_utils.download(label_url)) as flbl:
            struct.unpack(">II", flbl.read(8))
            label = np.frombuffer(flbl.read(), dtype=np.int8)
        with gzip.open(mx.test_utils.download(image_url), 'rb') as fimg:
            _, _, rows, cols = struct.unpack(">IIII", fimg.read(16))
            image = np.frombuffer(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)
            image = image.reshape(image.shape[0], 1, 28, 28).astype(np.float32)/255
        return (label, image)

    # changed to mxnet.io for more stable hosting
    # path = 'http://yann.lecun.com/exdb/mnist/'
    path = 'http://data.mxnet.io/data/mnist/'
    (train_lbl, train_img) = read_data(
        path+'train-labels-idx1-ubyte.gz', path+'train-images-idx3-ubyte.gz')
    (test_lbl, test_img) = read_data(
        path+'t10k-labels-idx1-ubyte.gz', path+'t10k-images-idx3-ubyte.gz')
    return {'train_data':train_img, 'train_label':train_lbl,
            'test_data':test_img, 'test_label':test_lbl} 
Example #29
Source File: recordio.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 5 votes vote down vote up
def unpack(s):
    """Unpack a MXImageRecord to string.

    Parameters
    ----------
    s : str
        String buffer from ``MXRecordIO.read``.

    Returns
    -------
    header : IRHeader
        Header of the image record.
    s : str
        Unpacked string.

    Examples
    --------
    >>> record = mx.recordio.MXRecordIO('test.rec', 'r')
    >>> item = record.read()
    >>> header, s = mx.recordio.unpack(item)
    >>> header
    HEADER(flag=0, label=14.0, id=20129312, id2=0)
    """
    header = IRHeader(*struct.unpack(_IR_FORMAT, s[:_IR_SIZE]))
    s = s[_IR_SIZE:]
    if header.flag > 0:
        header = header._replace(label=np.frombuffer(s, np.float32, header.flag))
        s = s[header.flag*4:]
    return header, s 
Example #30
Source File: train_mnist.py    From dynamic-training-with-apache-mxnet-on-aws with Apache License 2.0 5 votes vote down vote up
def read_data(label, image):
    """
    download and read data into numpy
    """
    base_url = 'http://yann.lecun.com/exdb/mnist/'
    with gzip.open(download_file(base_url+label, os.path.join('data',label))) as flbl:
        magic, num = struct.unpack(">II", flbl.read(8))
        label = np.fromstring(flbl.read(), dtype=np.int8)
    with gzip.open(download_file(base_url+image, os.path.join('data',image)), 'rb') as fimg:
        magic, num, rows, cols = struct.unpack(">IIII", fimg.read(16))
        image = np.fromstring(fimg.read(), dtype=np.uint8).reshape(len(label), rows, cols)
    return (label, image)