Python struct.html() Examples

The following are code examples for showing how to use struct.html(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: nxt-python   Author: castarco   File: digital.py    GNU General Public License v3.0 6 votes vote down vote up
def _i2c_query(self, address, format):
        """Reads an i2c value from given address, and returns a value unpacked
        according to the given format. Format is the same as in the struct
        module. See http://docs.python.org/library/struct.html#format-strings
        """
        n_bytes = struct.calcsize(format)
        msg = chr(self.I2C_DEV) + chr(address)
        now = time()
        if self.last_poll+self.poll_delay > now:
            diff = now - self.last_poll
            sleep(self.poll_delay - diff)
        self.last_poll = time()
        self.brick.ls_write(self.port, msg, n_bytes)
        try:
            self._ls_get_status(n_bytes)
        finally:
            #we should clear the buffer no matter what happens
            data = self.brick.ls_read(self.port)
        if len(data) < n_bytes:
            raise I2CError, 'Read failure: Not enough bytes'
        data = struct.unpack(format, data[-n_bytes:])
        return data 
Example 2
Project: ontology-python-sdk   Author: ontio   File: binary_reader.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def unpack(self, fmt, length=1):
        """
        Unpack the stream contents according to the specified format in `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            length (int): amount of bytes to read.

        Returns:
            variable: the result according to the specified format.
        """
        try:
            info = struct.unpack(fmt, self.stream.read(length))[0]
        except struct.error as e:
            raise SDKException(ErrorCode.unpack_error(e.args[0]))
        return info 
Example 3
Project: ontology-python-sdk   Author: ontio   File: binary_reader.py    GNU Lesser General Public License v3.0 6 votes vote down vote up
def read_var_int(self, max_size=sys.maxsize):
        """
        Read a variable length integer from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max_size (int): (Optional) maximum number of bytes to read.

        Returns:
            int:
        """
        fb = self.read_byte()
        if fb is 0:
            return fb
        if hex(fb) == '0xfd':
            value = self.read_uint16()
        elif hex(fb) == '0xfe':
            value = self.read_uint32()
        elif hex(fb) == '0xff':
            value = self.read_uint64()
        else:
            value = fb
        if value > max_size:
            raise SDKException(ErrorCode.param_err('Invalid format'))
        return int(value) 
Example 4
Project: nxt-python   Author: Eelviny   File: digital.py    GNU General Public License v3.0 6 votes vote down vote up
def _i2c_query(self, address, format):
        """Reads an i2c value from given address, and returns a value unpacked
        according to the given format. Format is the same as in the struct
        module. See http://docs.python.org/library/struct.html#format-strings
        """
        n_bytes = struct.calcsize(format)
        msg = bytes((self.I2C_DEV, address))
        now = time()
        if self.last_poll+self.poll_delay > now:
            diff = now - self.last_poll
            sleep(self.poll_delay - diff)
        self.last_poll = time()
        self.brick.ls_write(self.port, msg, n_bytes)
        try:
            self._ls_get_status(n_bytes)
        finally:
            #we should clear the buffer no matter what happens
            data = self.brick.ls_read(self.port)
        if len(data) < n_bytes:
            raise I2CError('Read failure: Not enough bytes')
        data = struct.unpack(format, data[-n_bytes:])
        return data 
Example 5
Project: IOT-GD   Author: sergiosanchezs   File: modport.py    Mozilla Public License 2.0 6 votes vote down vote up
def MD_request_data_pack (Module_Number, Function_Code, MD_Address, MD_lenght, debug=False):
	global serialport		# Getting global variable of serial port
	"""
	It takes a string output data from pack function
	fromat:
	>: means least significant byte should be first in the resulting string.
	B: Integer of 1 byte size
	H: Integer of 2 byte size
	https://docs.python.org/2/library/struct.html
	"""
	data = struct.pack('>BBHH',Module_Number, Function_Code, MD_Address, MD_lenght)

	crc = _calculateCrcString(data)
	tx_data = data + crc

	# if debug: print "0x" + " 0x".join("{:02x}".format(ord(c)) for c in tx_data)
	if debug: print "Transmit Data: ",
	if debug: print " ".join("{:02x}".format(ord(c)) for c in tx_data)

	serialport.write(tx_data)		# Send the request channels data to modport Module_Number 
Example 6
Project: LibID   Author: ucam-cl-dtg   File: lean_minhash.py    Apache License 2.0 6 votes vote down vote up
def bytesize(self, byteorder='@'):
        '''Compute the byte size after serialization.

        Args:
            byteorder (str, optional): This is byte order of the serialized data. Use one 
                of the `byte order characters 
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        Returns:
            int: Size in number of bytes after serialization.
        '''
        # Use 8 bytes to store the seed integer
        seed_size = struct.calcsize(byteorder+'q')
        # Use 4 bytes to store the number of hash values
        length_size = struct.calcsize(byteorder+'i')
        # Use 4 bytes to store each hash value as we are using the lower 32 bit
        hashvalue_size = struct.calcsize(byteorder+'I')
        return seed_size + length_size + len(self) * hashvalue_size 
Example 7
Project: LinkMechanismStewartGouph   Author: Bobeye   File: digital.py    GNU General Public License v3.0 6 votes vote down vote up
def _i2c_query(self, address, format):
        """Reads an i2c value from given address, and returns a value unpacked
        according to the given format. Format is the same as in the struct
        module. See http://docs.python.org/library/struct.html#format-strings
        """
        n_bytes = struct.calcsize(format)
        msg = chr(self.I2C_DEV) + chr(address)
        now = time()
        if self.last_poll+self.poll_delay > now:
            diff = now - self.last_poll
            sleep(self.poll_delay - diff)
        self.last_poll = time()
        self.brick.ls_write(self.port, msg, n_bytes)
        try:
            self._ls_get_status(n_bytes)
        finally:
            #we should clear the buffer no matter what happens
            data = self.brick.ls_read(self.port)
        if len(data) < n_bytes:
            raise I2CError, 'Read failure: Not enough bytes'
        data = struct.unpack(format, data[-n_bytes:])
        return data 
Example 8
Project: encryptit   Author: paulfurley   File: test_consume_header.py    GNU Affero General Public License v3.0 6 votes vote down vote up
def _make_old_format_header(packet_tag, body_length):
    tag_bits = (packet_tag & 0b00001111) << 2

    if 0 <= body_length <= 0xFF:
        length_type_bits = 0b00000000  # 0 means 1-octet length
        length_octets = bytearray([body_length])

    elif 0xFF < body_length <= 0xFFFF:
        length_type_bits = 0b00000001  # 1 means 2-octet length
        # https://docs.python.org/2/library/struct.html#format-characters
        length_octets = struct.pack('>H', body_length)

    elif 0xFFFF < body_length <= 0xFFFFFFFF:
        length_type_bits = 0b00000010  # 2 means 4-octet length
        length_octets = struct.pack('>I', body_length)

    else:
        raise ValueError(body_length)

    packet_first_octet = OLD_OCTET_0 | tag_bits | length_type_bits
    return bytearray([packet_first_octet]) + length_octets 
Example 9
Project: ontology-python-sdk   Author: ontio   File: binary_writer.py    GNU Lesser General Public License v3.0 5 votes vote down vote up
def pack(self, fmt, data):
        """
        Write bytes by packing them according to the provided format `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html
        """
        return self.write_bytes(struct.pack(fmt, data)) 
Example 10
Project: asws   Author: regiontog   File: reader.py    MIT License 5 votes vote down vote up
def feed(self, reader):
        data = await reader.readexactly(1)
        mask_flag = data[0] & WebSocketReader.MASK_BIT
        length = data[0] & ~WebSocketReader.MASK_BIT

        # "The form '!' is available for those poor souls who claim they can’t remember whether network byte order is
        # big-endian or little-endian."
        # - <https://docs.python.org/3/library/struct.html>
        if length == 126:
            data = await reader.readexactly(2)
            length, = struct.unpack('!H', data)
        elif length == 127:
            data = await reader.readexactly(8)
            length, = struct.unpack('!Q', data)

        left_to_read = length

        if not mask_flag:
            # TODO: Reject frame per <https://tools.ietf.org/html/rfc6455#section-5.1>
            logger.warning("Received message from client without mask.")
            await reader.readexactly(left_to_read)
            raise Exception("Received message from client without mask.")  # TODO: HANDLE

        mask = await reader.readexactly(4)

        first_read_size = min(WebSocketReader.BUFFER_SIZE, left_to_read)
        await self.que.put((await reader.readexactly(first_read_size), first_read_size, mask))
        left_to_read -= first_read_size

        while left_to_read > WebSocketReader.BUFFER_SIZE:
            await self.que.put(
                (await reader.readexactly(WebSocketReader.BUFFER_SIZE), WebSocketReader.BUFFER_SIZE, mask))
            left_to_read -= WebSocketReader.BUFFER_SIZE

        if left_to_read > 0:
            await self.que.put((await reader.readexactly(left_to_read), left_to_read, mask))

        return length 
Example 11
Project: gsensors   Author: enavarro222   File: i2c_devices.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def read_cast(self, cmd, nb_bytes=4, cast="f"):
        """
        to see all possible cast:
        http://docs.python.org/2/library/struct.html#format-characters
        """
        bytes = self.read_block(cmd, nb_bytes=nb_bytes)
        return struct.unpack(cast, "".join(map(chr, bytes)))[0] 
Example 12
Project: counsyl-pyads   Author: counsyl   File: adsdatatypes.py    MIT License 5 votes vote down vote up
def unpack(self, value):
        """Unpack a value using Python's struct.unpack()"""
        assert(self.pack_format is not None)
        # Note: "The result is a tuple even if it contains exactly one item."
        # (https://docs.python.org/2/library/struct.html#struct.unpack)
        # For single-valued data types, use AdsSingleValuedDatatype to get the
        # first (and only) entry of the tuple after unpacking.
        return struct.unpack(self.pack_format, value) 
Example 13
Project: MarkovProcess_Bedload   Author: smchartrand   File: evtk.py    MIT License 5 votes vote down vote up
def _get_byte_order_char():
# Check format in https://docs.python.org/3.5/library/struct.html
    if sys.byteorder == "little":
        return '<'
    else:
        return '>'
        
# ================================
#        Python interface
# ================================ 
Example 14
Project: MarkovProcess_Bedload   Author: smchartrand   File: evtk.py    MIT License 5 votes vote down vote up
def _get_byte_order_char():
# Check format in https://docs.python.org/3.5/library/struct.html
    if sys.byteorder == "little":
        return '<'
    else:
        return '>'
        
# ================================
#        Python interface
# ================================ 
Example 15
Project: LibID   Author: ucam-cl-dtg   File: lean_minhash.py    Apache License 2.0 5 votes vote down vote up
def deserialize(cls, buf, byteorder='@'):
        '''
        Deserialize a lean MinHash from a buffer.

        Args:
            buf (buffer): `buf` must implement the `buffer`_ interface.
                One such example is the built-in `bytearray`_ class.
            byteorder (str. optional): This is byte order of the serialized data. Use one 
                of the `byte order characters 
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.
       
        Return:
            datasketch.LeanMinHash: The deserialized lean MinHash

        Example:
            To deserialize a lean MinHash from a buffer.

            .. code-block:: python
                
                lean_minhash = LeanMinHash.deserialize(buf)
        '''
        fmt_seed_size = "%sqi" % byteorder
        fmt_hash = byteorder + "%dI"
        try:
            seed, num_perm = struct.unpack_from(fmt_seed_size, buf, 0)
        except TypeError:
            seed, num_perm = struct.unpack_from(fmt_seed_size, buffer(buf), 0)
        offset = struct.calcsize(fmt_seed_size)
        try:
            hashvalues = struct.unpack_from(fmt_hash % num_perm, buf, offset)
        except TypeError:
            hashvalues = struct.unpack_from(fmt_hash % num_perm, buffer(buf), offset)
        lmh = object.__new__(LeanMinHash)
        lmh._initialize_slots(seed, hashvalues)
        return lmh 
Example 16
Project: passbytcp   Author: mxdg   File: common_func.py    MIT License 5 votes vote down vote up
def _start(self):
        # memoryview act as an recv buffer
        # refer https://docs.python.org/3/library/stdtypes.html#memoryview
        buff = memoryview(bytearray(RECV_BUFFER_SIZE))
        while True:
            if not self.conn_rd:
                # sleep if there is no connections
                time.sleep(0.06)
                continue

            # blocks until there is socket(s) ready for .recv
            # notice: sockets which were closed by remote,
            #   are also regarded as read-ready by select()
            r, w, e = select.select(self.conn_rd, [], [], 0.5)

            for s in r:  # iter every read-ready or closed sockets
                try:
                    # here, we use .recv_into() instead of .recv()
                    #   recv data directly into the pre-allocated buffer
                    #   to avoid many unnecessary malloc()
                    # see https://docs.python.org/3/library/socket.html#socket.socket.recv_into
                    rec_len = s.recv_into(buff, RECV_BUFFER_SIZE)
                except:
                    # unable to read, in most cases, it's due to socket close
                    self._rd_shutdown(s)
                    continue

                if not rec_len:
                    # read zero size, closed or shutdowned socket
                    self._rd_shutdown(s)
                    continue

                try:
                    # send data, we use `buff[:rec_len]` slice because
                    #   only the front of buff is filled
                    self.map[s].send(buff[:rec_len])
                except:
                    # unable to send, close connection
                    self._rd_shutdown(s)
                    continue 
Example 17
Project: python-sinamics   Author: gilgab   File: sinamics.py    MIT License 5 votes vote down vote up
def __init__(self):
        # Defining this object as a S7 Client.
        # See: snap7.sourceforge.net/home.html

        self.converter = snap7.client.Client() 
Example 18
Project: encryptit   Author: paulfurley   File: test_consume_header.py    GNU Affero General Public License v3.0 5 votes vote down vote up
def _make_new_format_header(packet_tag, body_length):
    tag_bits = (packet_tag & 0b00111111)

    assert 0 < tag_bits & 0b00111111 <= 19

    if 0 <= body_length <= 191:  # one-octet length
        length_octets = bytearray([body_length])

    elif 192 <= body_length <= 8383:  # two-octet length
        # https://tools.ietf.org/html/rfc4880#section-4.2.2.2

        tmp = body_length - 192
        first_octet = (tmp >> 8) + 192
        second_octet = tmp & 0xFF

        assert 192 <= first_octet <= 223, first_octet
        assert 0 <= second_octet <= 255, second_octet

        length_octets = bytearray([first_octet, second_octet])

    elif 8383 < body_length <= 0xFFFFFFFF:  # five-octet length
        # https://tools.ietf.org/html/rfc4880#section-4.2.2.3
        length_octets = bytearray([255]) + struct.pack('>I', body_length)

    packet_first_octet = NEW_OCTET_0 | tag_bits
    return bytearray([packet_first_octet]) + length_octets 
Example 19
Project: mnist   Author: datapythonista   File: __init__.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def parse_idx(fd):
    """Parse an IDX file, and return it as a numpy array.

    Parameters
    ----------
    fd : file
        File descriptor of the IDX file to parse

    endian : str
        Byte order of the IDX file. See [1] for available options

    Returns
    -------
    data : numpy.ndarray
        Numpy array with the dimensions and the data in the IDX file

    1. https://docs.python.org/3/library/struct.html
        #byte-order-size-and-alignment
    """
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, '
                             'file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, '
                             'file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type '
                             '0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items,
                                                          len(data)))

    return numpy.array(data).reshape(dimension_sizes) 
Example 20
Project: Note-refinement-   Author: nkundiushuti   File: lazy_io.py    GNU General Public License v3.0 4 votes vote down vote up
def chunks(seq, size=None, dfmt="f", byte_order=None, padval=0.):
  """
  Chunk generator based on the struct module (Python standard library).

  Low-level data blockenizer for homogeneous data as a generator, to help
  writing an iterable into a file.
  The dfmt should be one char, chosen from the ones in link:

    `<http://docs.python.org/library/struct.html#format-characters>`_

  Useful examples (integer are signed, use upper case for unsigned ones):

  - "b" for 8 bits (1 byte) integer
  - "h" for 16 bits (2 bytes) integer
  - "i" for 32 bits (4 bytes) integer
  - "f" for 32 bits (4 bytes) float (default)
  - "d" for 64 bits (8 bytes) float (double)

  Byte order follows native system defaults. Other options are in the site:

    `<http://docs.python.org/library/struct.html#struct-alignment>`_

  They are:

  - "<" means little-endian
  - ">" means big-endian

  Note
  ----
  Default chunk size can be accessed (and changed) via chunks.size.

  """
  if size is None:
    size = chunks.size
  dfmt = str(size) + dfmt
  if byte_order is None:
    struct_string = dfmt
  else:
    struct_string = byte_order + dfmt
  s = struct.Struct(struct_string)
  for block in blocks(seq, size, padval=padval):
    yield s.pack(*block) 
Example 21
Project: DarwinexLabs   Author: darwinex   File: DWX_HISTORY_IO_v2_0_1_RC8.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def __init__(self):
        
        """
        .hst file format valid as of MT4 574 and later
        Refer to: https://www.mql5.com/en/forum/149178
        
        DATABASE HEADER (148 bytes):
            
        int         version;        // 4 bytes
        string	    copyright[64];	// 64 bytes
        string      symbol[12];     // 12 bytes
        int         period;         // 4 bytes
        int         digits;         // 4 bytes
        datetime	timesign;       // 4 bytes
        datetime	last_sync;      // 4 bytes
        int         unused[13];     // 52 bytes
        
        BAR STRUCTURE (60 bytes):
        
        datetime	ctm;            // 8 bytes
        double	    open;           // 8 bytes
        double      high;           // 8 bytes
        double      low;            // 8 bytes
        double      close;          // 8 bytes
        long        volume;         // 8 bytes
        int         spread;         // 4 bytes
        long        real_volume;	// 8 bytes
        
        Python STRUCT format characters:
        (Ref: https://docs.python.org/3/library/struct.html)
            
        < = (Byte Order: little-endian, Size: standard)
        Q = (C Type: unsigned long long, Python type: integer, Size: 8)
        d = (C Type: double, Python type: float, Size: 8)
        q = (C Type: long long, Python type: integer, Size: 8)
        i = (C Type: int, Python type: integer, Size: 4)    
        """
        
        self._HST_BYTE_FORMAT = '<Qddddqiq'
        self._HEADER_BYTES = 148
        self._BAR_BYTES = 60
    
    ########################################################################## 
Example 22
Project: LibID   Author: ucam-cl-dtg   File: lean_minhash.py    Apache License 2.0 4 votes vote down vote up
def serialize(self, buf, byteorder='@'):
        '''
        Serialize this lean MinHash and store the result in an allocated buffer.

        Args:
            buf (buffer): `buf` must implement the `buffer`_ interface.
                One such example is the built-in `bytearray`_ class.
            byteorder (str, optional): This is byte order of the serialized data. Use one 
                of the `byte order characters 
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        This is preferred over using `pickle`_ if the serialized lean MinHash needs
        to be used by another program in a different programming language.

        The serialization schema:
            1. The first 8 bytes is the seed integer
            2. The next 4 bytes is the number of hash values 
            3. The rest is the serialized hash values, each uses 4 bytes
        
        Example:
            To serialize a single lean MinHash into a `bytearray`_ buffer.

            .. code-block:: python
                
                buf = bytearray(lean_minhash.bytesize())
                lean_minhash.serialize(buf)

            To serialize multiple lean MinHash into a `bytearray`_ buffer.

            .. code-block:: python
                
                # assuming lean_minhashs is a list of LeanMinHash with the same size
                size = lean_minhashs[0].bytesize()
                buf = bytearray(size*len(lean_minhashs))
                for i, lean_minhash in enumerate(lean_minhashs):
                    lean_minhash.serialize(buf[i*size:])

        .. _`buffer`: https://docs.python.org/3/c-api/buffer.html
        .. _`bytearray`: https://docs.python.org/3.6/library/functions.html#bytearray
        .. _`byteorder`: https://docs.python.org/3/library/struct.html
        '''
        if len(buf) < self.bytesize():
            raise ValueError("The buffer does not have enough space\
                    for holding this MinHash.")
        fmt = "%sqi%dI" % (byteorder, len(self))
        struct.pack_into(fmt, buf, 0,
                self.seed, len(self), *self.hashvalues) 
Example 23
Project: passbytcp   Author: mxdg   File: common_func.py    MIT License 4 votes vote down vote up
def _start(self):
        # memoryview act as an recv buffer
        # refer https://docs.python.org/3/library/stdtypes.html#memoryview
        buff = memoryview(bytearray(RECV_BUFFER_SIZE))
        while True:
            if not self.conn_rd:
                # sleep if there is no connections
                time.sleep(0.06)
                continue

            # blocks until there is socket(s) ready for .recv
            # notice: sockets which were closed by remote,
            #   are also regarded as read-ready by select()
            r, w, e = select.select(self.conn_rd, [], [], 0.5)

            for s in r:  # iter every read-ready or closed sockets
                try:
                    # here, we use .recv_into() instead of .recv()
                    #   recv data directly into the pre-allocated buffer
                    #   to avoid many unnecessary malloc()
                    # see https://docs.python.org/3/library/socket.html#socket.socket.recv_into
                    rec_len = s.recv_into(buff, RECV_BUFFER_SIZE)

                    # agre = "http"
                    # url = agre + '://' + heads['Host']
                    # heads = httphead(buff.tobytes().decode('utf-8'))
                    # logging.info("recv head:{}".format(heads))
                except Exception as e:
                    # unable to read, in most cases, it's due to socket close
                    self._rd_shutdown(s)
                    continue

                if not rec_len:
                    # read zero size, closed or shutdowned socket
                    self._rd_shutdown(s)
                    continue

                try:
                    # send data, we use `buff[:rec_len]` slice because
                    #   only the front of buff is filled
                    self.map[s].send(buff[:rec_len])
                except Exception as e:
                    # unable to send, close connection
                    self._rd_shutdown(s)
                    continue 
Example 24
Project: FlyMambo   Author: kanishkaganguly   File: DroneSensorParser.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def get_data_format_and_size(data, data_type):
    """
    Internal function to convert data_type to the corresponding struct.pack format string
    as per https://docs.python.org/2/library/struct.html#format-characters

    Function contributed by awm102 on GitHub.  Amy moved this to DroneSensorParser to be
    more general, edited a bit to fit within the drone sensor parser as well.

    :param data: the data that will be packed. Not actually used here unless the data_type is string, then
                 it is used to calculate the data size.
    :param data_type: a string representing the data type
    :return: a tuple of a string representing the struct.pack format for the data type and an int representing
             the number of bytes 
    """

    if data_type == "u8" or data_type == "enum":
        format_char = "<B"
        data_size = 1
    elif data_type == "i8":
        format_char = "<b"
        data_size = 1
    elif data_type == "u16":
        format_char = "<H"
        data_size = 2
    elif data_type == "i16":
        format_char = "<h"
        data_size = 2
    elif data_type == "u32":
        format_char = "<I"
        data_size = 4
    elif data_type == "i32":
        format_char = "<i"
        data_size = 4
    elif data_type == "u64":
        format_char = "<Q"
        data_size = 8
    elif data_type == "i64":
        format_char = "<q"
        data_size = 8
    elif data_type == "float":
        format_char = "<f"
        data_size = 4
    elif data_type == "double":
        format_char = "<d"
        data_size = 8
    elif data_type == "string":
        format_char = "<s"
        data_size = len(data)
    else:
        format_char = ""
        data_size = 0

    return (format_char, data_size) 
Example 25
Project: tfbldr   Author: kastnerkyle   File: loaders.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def parse_idx(fd):
    """
    Parse an IDX file, and return it as a numpy array.
    From https://github.com/datapythonista/mnist

    Parameters
    ----------
    fd : file
        File descriptor of the IDX file to parse
    endian : str
        Byte order of the IDX file. See [1] for available options

    Returns
    -------
    data : numpy.ndarray
        Numpy array with the dimensions and the data in the IDX file
    1. https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment
    """
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type 0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items, len(data)))

    return np.array(data).reshape(dimension_sizes) 
Example 26
Project: NRU   Author: apsarath   File: load_mnist.py    Apache License 2.0 3 votes vote down vote up
def parse_idx(fd):
    """Parse an IDX file, and return it as a numpy array.
    Parameters
    ----------
    fd : file
        File descriptor of the IDX file to parse
    endian : str
        Byte order of the IDX file. See [1] for available options
    Returns
    -------
    data : numpy.ndarray
        Numpy array with the dimensions and the data in the IDX file
    1. https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment
    """
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type 0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items, len(data)))

    return np.array(data).reshape(dimension_sizes)