Python struct.html() Examples

The following are 39 code examples for showing how to use struct.html(). These examples are extracted from open source projects. You can vote up the ones you like or vote down the ones you don't like, and go to the original project or source file by following the links above each example.

You may want to check out the right sidebar which shows the related API usage.

You may also want to check out all available functions/classes of the module struct , or try the search function .

Example 1
def unpack(self, fmt, length=1):
        """
        Unpack the stream contents according to the specified format in `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            length (int): amount of bytes to read.

        Returns:
            variable: the result according to the specified format.
        """
        try:
            info = struct.unpack(fmt, self.stream.read(length))[0]
        except struct.error as e:
            raise SDKException(ErrorCode.unpack_error(e.args[0]))
        return info 
Example 2
def read_var_int(self, max_size=sys.maxsize):
        """
        Read a variable length integer from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max_size (int): (Optional) maximum number of bytes to read.

        Returns:
            int:
        """
        fb = self.read_byte()
        if fb is 0:
            return fb
        if hex(fb) == '0xfd':
            value = self.read_uint16()
        elif hex(fb) == '0xfe':
            value = self.read_uint32()
        elif hex(fb) == '0xff':
            value = self.read_uint64()
        else:
            value = fb
        if value > max_size:
            raise SDKException(ErrorCode.param_err('Invalid format'))
        return int(value) 
Example 3
Project: nxt-python   Author: Eelviny   File: digital.py    License: GNU General Public License v3.0 6 votes vote down vote up
def _i2c_query(self, address, format):
        """Reads an i2c value from given address, and returns a value unpacked
        according to the given format. Format is the same as in the struct
        module. See http://docs.python.org/library/struct.html#format-strings
        """
        n_bytes = struct.calcsize(format)
        msg = bytes((self.I2C_DEV, address))
        now = time()
        if self.last_poll+self.poll_delay > now:
            diff = now - self.last_poll
            sleep(self.poll_delay - diff)
        self.last_poll = time()
        self.brick.ls_write(self.port, msg, n_bytes)
        try:
            self._ls_get_status(n_bytes)
        finally:
            #we should clear the buffer no matter what happens
            data = self.brick.ls_read(self.port)
        if len(data) < n_bytes:
            raise I2CError('Read failure: Not enough bytes')
        data = struct.unpack(format, data[-n_bytes:])
        return data 
Example 4
Project: neo-python   Author: CityOfZion   File: BinaryReader.py    License: MIT License 6 votes vote down vote up
def ReadVarBytes(self, max=sys.maxsize):
        """
        Read a variable length of bytes from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max (int): (Optional) maximum number of bytes to read.

        Raises:
            ValueError: if the amount of bytes indicated by the variable int cannot be read

        Returns:
            bytes:
        """
        length = self.ReadVarInt(max)
        return self.SafeReadBytes(length) 
Example 5
Project: neo-python   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 6 votes vote down vote up
def WriteVarString(self, value, encoding="utf-8"):
        """
        Write a string value to the stream.
        Read more about variable size encoding here: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            value (string): value to write to the stream.
            encoding (str): string encoding format.
        """
        if type(value) is str:
            value = value.encode(encoding)

        length = len(value)
        ba = bytearray(value)
        byts = binascii.hexlify(ba)
        string = byts.decode(encoding)
        self.WriteVarInt(length)
        self.WriteBytes(string) 
Example 6
Project: neo-python   Author: CityOfZion   File: binary_reader.py    License: MIT License 6 votes vote down vote up
def _unpack(self, fmt, length=1) -> Any:
        """
        Unpack the stream contents according to the specified format in `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            length (int): amount of bytes to read.

        Returns:
            variable: the result according to the specified format.
        """
        try:
            values = struct.unpack(fmt, self._stream.read(length))
            return values[0]
        except struct.error as e:
            raise ValueError(e) 
Example 7
Project: datasketch   Author: ekzhu   File: lean_minhash.py    License: MIT License 6 votes vote down vote up
def bytesize(self, byteorder='@'):
        '''Compute the byte size after serialization.

        Args:
            byteorder (str, optional): This is byte order of the serialized data. Use one
                of the `byte order characters
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        Returns:
            int: Size in number of bytes after serialization.
        '''
        # Use 8 bytes to store the seed integer
        seed_size = struct.calcsize(byteorder+'q')
        # Use 4 bytes to store the number of hash values
        length_size = struct.calcsize(byteorder+'i')
        # Use 4 bytes to store each hash value as we are using the lower 32 bit
        hashvalue_size = struct.calcsize(byteorder+'I')
        return seed_size + length_size + len(self) * hashvalue_size 
Example 8
Project: pedlar   Author: nuric   File: agent.py    License: Apache License 2.0 6 votes vote down vote up
def remote_run(self):
    """Start main loop and receive updates."""
    # Check connection
    if not self._session:
      self.connect()
    # We'll trade forever until interrupted
    logger.info("Starting main trading loop...")
    try:
      while True:
        socks = self._poller.poll(self.polltimeout)
        if not socks:
          continue
        raw = socks[0][0].recv()
        # unpack bytes https://docs.python.org/3/library/struct.html
        if len(raw) == 17:
          # We have tick data
          bid, ask = struct.unpack_from('dd', raw, 1) # offset topic
          self.on_tick(bid, ask, datetime.now())
        elif len(raw) == 33:
          # We have bar data
          bo, bh, bl, bc = struct.unpack_from('dddd', raw, 1) # offset topic
          self.on_bar(bo, bh, bl, bc, datetime.now())
    finally:
      logger.info("Stopping agent...")
      self.disconnect() 
Example 9
Project: pedlar   Author: nuric   File: flask_ticker.py    License: Apache License 2.0 6 votes vote down vote up
def run(self):
    """Connect to ticker server and emit updates."""
    socket = context.socket(zmq.SUB)
    # Set topic filter, this is a binary prefix
    # to check for each incoming message
    # set from server as uchar topic = X
    # We'll subsribe to only tick updates for now
    socket.setsockopt(zmq.SUBSCRIBE, bytes.fromhex('00'))
    with self.app.app_context():
      current_app.logger.debug("Connecting to ticker: %s", current_app.config['TICKER_URL'])
      socket.connect(current_app.config['TICKER_URL'])
      while True:
        raw = socket.recv()
        # unpack bytes https://docs.python.org/3/library/struct.html
        bid, ask = struct.unpack_from('dd', raw, 1) # offset topic
        self.socketio.emit('tick', {'bid': round(bid, 5), 'ask': round(ask, 5)})
    # socket will be cleaned up at garbarge collection 
Example 10
Project: pedlar   Author: nuric   File: lbroker.py    License: Apache License 2.0 6 votes vote down vote up
def handle_tick():
  """Listen to incoming tick updates."""
  socket = context.socket(zmq.SUB)
  # Set topic filter, this is a binary prefix
  # to check for each incoming message
  # set from server as uchar topic = X
  # We'll subsribe to only tick updates for now
  socket.setsockopt(zmq.SUBSCRIBE, bytes.fromhex('00'))
  logger.info("Connecting to ticker: %s", ARGS.ticker)
  socket.connect(ARGS.ticker)
  while True:
    raw = socket.recv()
    # unpack bytes https://docs.python.org/3/library/struct.html
    bid, ask = struct.unpack_from('dd', raw, 1) # offset topic
    logger.debug("Tick: %f %f", bid, ask)
    # We'll use global to pass tick data between green threads
    # since only 1 actually run at a time
    global BID, ASK # pylint: disable=global-statement
    BID, ASK = bid, ask
  # socket will be cleaned up at garbarge collection 
Example 11
Project: neo-python-core   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 6 votes vote down vote up
def WriteVarString(self, value, encoding="utf-8"):
        """
        Write a string value to the stream.
        Read more about variable size encoding here: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            value (string): value to write to the stream.
            encoding (str): string encoding format.
        """
        if type(value) is str:
            value = value.encode(encoding)

        length = len(value)
        ba = bytearray(value)
        byts = binascii.hexlify(ba)
        string = byts.decode(encoding)
        self.WriteVarInt(length)
        self.WriteBytes(string) 
Example 12
def pack(self, fmt, data):
        """
        Write bytes by packing them according to the provided format `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html
        """
        return self.write_bytes(struct.pack(fmt, data)) 
Example 13
Project: naz   Author: komuw   File: state.py    License: MIT License 5 votes vote down vote up
def _find_data_coding(encoding):
        # NB:
        # We cant use all python standard encodings[1]
        # We can only use the ones defined in SMPP spec[2];
        #
        # 1. https://docs.python.org/3/library/codecs.html#standard-encodings
        # 2. section 5.2.19 of smpp ver 3.4 spec document.
        try:
            return SmppDataCoding.__dict__[encoding]
        except Exception as e:
            raise ValueError(
                "That encoding: `{0}` is not a recognised SMPP encoding.".format(encoding)
            ) from e 
Example 14
Project: basil   Author: SiLab-Bonn   File: BitLogic.py    License: BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def from_value(cls, value, size=None, fmt='Q', **kwargs):
        '''
        Factory method

        For format characters see: https://docs.python.org/2/library/struct.html
        '''
        bl = cls(**kwargs)  # size is 0 by default
        bl.fromvalue(value=value, size=size, fmt=fmt)
        return bl 
Example 15
Project: passbytcp   Author: mxdg   File: common_func.py    License: MIT License 5 votes vote down vote up
def _start(self):
        # memoryview act as an recv buffer
        # refer https://docs.python.org/3/library/stdtypes.html#memoryview
        buff = memoryview(bytearray(RECV_BUFFER_SIZE))
        while True:
            if not self.conn_rd:
                # sleep if there is no connections
                time.sleep(0.06)
                continue

            # blocks until there is socket(s) ready for .recv
            # notice: sockets which were closed by remote,
            #   are also regarded as read-ready by select()
            r, w, e = select.select(self.conn_rd, [], [], 0.5)

            for s in r:  # iter every read-ready or closed sockets
                try:
                    # here, we use .recv_into() instead of .recv()
                    #   recv data directly into the pre-allocated buffer
                    #   to avoid many unnecessary malloc()
                    # see https://docs.python.org/3/library/socket.html#socket.socket.recv_into
                    rec_len = s.recv_into(buff, RECV_BUFFER_SIZE)
                except:
                    # unable to read, in most cases, it's due to socket close
                    self._rd_shutdown(s)
                    continue

                if not rec_len:
                    # read zero size, closed or shutdowned socket
                    self._rd_shutdown(s)
                    continue

                try:
                    # send data, we use `buff[:rec_len]` slice because
                    #   only the front of buff is filled
                    self.map[s].send(buff[:rec_len])
                except:
                    # unable to send, close connection
                    self._rd_shutdown(s)
                    continue 
Example 16
Project: wradlib   Author: wradlib   File: iris.py    License: MIT License 5 votes vote down vote up
def decode_string(data):
    """Decode string and strip NULL-bytes from end."""
    return data.decode("utf-8").rstrip("\0")


# IRIS Data Types and corresponding python struct format characters
# 4.2 Scalar Definitions, Page 23
# https://docs.python.org/3/library/struct.html#format-characters 
Example 17
Project: wav2vec   Author: cristoper   File: WavDecoder.py    License: Do What The F*ck You Want To Public License 5 votes vote down vote up
def struct_fmt_char(self):
        """
        Calculates the character to use with `struct.unpack()` to decode sample
        bytes compatible with the data file's sample width.

        Supported PCM file formats:
            - 8-bit unsigned WAV
            - 8-bit signed AIFF
            - 16-bit signed WAV (little endian)and AIFF (big endian)
            - 32-bit signed WAV (little endian)and AIFF (big endian)

        Raises ValueError if `filename` is not a supported file type.

        see: https://docs.python.org/library/struct.html
        """
        sampwidth = self.params.sampwidth
        if sampwidth == 1 and not self.signed:
            logger.info("unsigned 8-bit ('B')")
            return 'B'
        elif sampwidth == 1:
            logger.info("signed 8-bit ('b')")
            return 'b'
        elif sampwidth == 2:
            logger.info("signed 16-bit ('h')")
            return 'h'
        elif sampwidth == 4:
            logger.info("signed 32-bit ('h')")
            return 'i'
        else:
            raise ValueError("Unsupported file type.") 
Example 18
def __generate_struct_format_string(self):
        format_string = "<"  # Use little endian format with standard sizes for python,
        # see https://docs.python.org/2/library/struct.html

        # If you got error message AttributeError: 'int' object has no attribute 'value'.
        # Uncomment line 102 and comment out line 103 and run 'python setup.py install'
        #for key in sorted(self.init_packet_fields.keys(), key=lambda x: x):
        for key in sorted(list(self.init_packet_fields.keys()), key=lambda x: x.value):
            if key in [PacketField.DEVICE_TYPE,
                       PacketField.DEVICE_REVISION,
                       ]:
                format_string += Packet.UNSIGNED_SHORT

            elif key in [PacketField.APP_VERSION]:
                format_string += Packet.UNSIGNED_INT
            elif key in [PacketField.REQUIRED_SOFTDEVICES_ARRAY]:
                array_elements = self.init_packet_fields[key]
                format_string += Packet.UNSIGNED_SHORT  # Add length field to format packet

                for _ in range(len(array_elements)):
                    format_string += Packet.UNSIGNED_SHORT
            elif key in [PacketField.OPT_DATA]:
                format_string += Packet.UNSIGNED_SHORT  # Add length field to optional data
                format_string += "{0}{1}".format(len(self.init_packet_fields[key]), Packet.CHAR_ARRAY)
            elif key in [PacketField.NORDIC_PROPRIETARY_OPT_DATA_EXT_PACKET_ID]:
                format_string += Packet.UNSIGNED_INT  # Add the extended packet id field
            elif key == PacketField.NORDIC_PROPRIETARY_OPT_DATA_FIRMWARE_LENGTH:
                format_string += Packet.UNSIGNED_INT  # Add the firmware length field
            elif key == PacketField.NORDIC_PROPRIETARY_OPT_DATA_FIRMWARE_HASH:
                format_string += "32{0}".format(Packet.CHAR_ARRAY)  # SHA-256 requires 32 bytes
            elif key == PacketField.NORDIC_PROPRIETARY_OPT_DATA_FIRMWARE_CRC16:
                format_string += Packet.UNSIGNED_SHORT
            elif key == PacketField.NORDIC_PROPRIETARY_OPT_DATA_INIT_PACKET_ECDS:
                format_string += "64{0}".format(Packet.CHAR_ARRAY)  # ECDS based on P-256 using SHA-256 requires 64 bytes

        return format_string 
Example 19
Project: neo-python   Author: CityOfZion   File: BinaryReader.py    License: MIT License 5 votes vote down vote up
def unpack(self, fmt, length=1):
        """
        Unpack the stream contents according to the specified format in `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            length (int): amount of bytes to read.

        Returns:
            variable: the result according to the specified format.
        """
        return struct.unpack(fmt, self.stream.read(length))[0] 
Example 20
Project: neo-python   Author: CityOfZion   File: BinaryReader.py    License: MIT License 5 votes vote down vote up
def ReadVarInt(self, max=sys.maxsize):
        """
        Read a variable length integer from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max (int): (Optional) maximum number of bytes to read.

        Returns:
            int:

        Raises:
            ValueError: if the specified `max` number of bytes is exceeded
        """
        try:
            fb = self.ReadByte()
        except ValueError:
            return 0
        value = 0
        if fb == b'\xfd':
            value = self.ReadUInt16()
        elif fb == b'\xfe':
            value = self.ReadUInt32()
        elif fb == b'\xff':
            value = self.ReadUInt64()
        else:
            value = int.from_bytes(fb, "little")

        if value > max:
            raise ValueError(f"Maximum number of bytes ({max}) exceeded.")

        return int(value) 
Example 21
Project: neo-python   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 5 votes vote down vote up
def pack(self, fmt, data):
        """
        Write bytes by packing them according to the provided format `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            data (object): the data to write to the raw stream.

        Returns:
            int: the number of bytes written.
        """
        return self.WriteBytes(struct.pack(fmt, data), unhex=False) 
Example 22
Project: neo-python   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 5 votes vote down vote up
def WriteVarInt(self, value, endian="<"):
        """
        Write an integer value in a space saving way to the stream.
        Read more about variable size encoding here: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            value (int):
            endian (str): specify the endianness. (Default) Little endian ('<'). Use '>' for big endian.

        Returns:
            int: the number of bytes written.

        Raises:
            TypeError: if `value` is not of type int.
            ValueError: if `value` is < 0.
        """
        if not isinstance(value, int):
            raise TypeError(f'{value} not int type.')

        if value < 0:
            raise ValueError(f'{value} too small.')

        elif value < 0xfd:
            return self.WriteByte(value)

        elif value <= 0xffff:
            self.WriteByte(0xfd)
            return self.WriteUInt16(value, endian)

        elif value <= 0xFFFFFFFF:
            self.WriteByte(0xfe)
            return self.WriteUInt32(value, endian)

        else:
            self.WriteByte(0xff)
            return self.WriteUInt64(value, endian) 
Example 23
Project: neo-python   Author: CityOfZion   File: binary_reader.py    License: MIT License 5 votes vote down vote up
def read_var_int(self, max=sys.maxsize) -> int:
        """
        Read a variable length integer from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max: (Optional) maximum number of bytes to read.

        Returns:
            int:
        """
        fb = int.from_bytes(self.read_byte(), 'little')
        if fb is 0:
            return fb

        if fb == 0xfd:
            value = self.read_uint16()
        elif fb == 0xfe:
            value = self.read_uint32()
        elif fb == 0xff:
            value = self.read_uint64()
        else:
            value = fb

        if value > max:
            raise ValueError("Invalid format")

        return value 
Example 24
Project: neo-python   Author: CityOfZion   File: binary_reader.py    License: MIT License 5 votes vote down vote up
def read_var_bytes(self, max=sys.maxsize):
        """
        Read a variable length of bytes from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max (int): (Optional) maximum number of bytes to read.

        Returns:
            bytes:
        """
        length = self.read_var_int(max)
        return self.read_bytes(length) 
Example 25
Project: neo-python   Author: CityOfZion   File: binary_writer.py    License: MIT License 5 votes vote down vote up
def _pack(self, fmt, data) -> int:
        """
        Write bytes by packing them according to the provided format `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html
        Args:
            fmt (str): format string.
            data (object): the data to write to the raw stream.
        Returns:
            int: the number of bytes written.
        """
        return self.write_bytes(struct.pack(fmt, data), unhex=False) 
Example 26
Project: neo-python   Author: CityOfZion   File: binary_writer.py    License: MIT License 5 votes vote down vote up
def write_var_int(self, value: int, endian: str = "<") -> int:
        """
        Write an integer value in a space saving way to the stream.
        Read more about variable size encoding here: http://docs.neo.org/en-us/node/network-protocol.html#convention
        Args:
            value:
            endian: specify the endianness. (Default) Little endian ('<'). Use '>' for big endian.
        Raises:
            {TypeError}: if ``value`` is not of type int.
            ValueError: if `value` is < 0.
        Returns:
            int: the number of bytes written.
        """
        if not isinstance(value, int):
            raise TypeError('%s not int type.' % value)

        if value < 0:
            raise ValueError('%d too small.' % value)

        elif value < 0xfd:
            return self.write_bytes(bytes([value]))

        elif value <= 0xffff:
            self.write_bytes(bytes([0xfd]))
            return self.write_uint16(value, endian)

        elif value <= 0xFFFFFFFF:
            self.write_bytes(bytes([0xfe]))
            return self.write_uint32(value, endian)

        else:
            self.write_bytes(bytes([0xff]))
            return self.write_uint64(value, endian) 
Example 27
Project: datasketch   Author: ekzhu   File: lean_minhash.py    License: MIT License 5 votes vote down vote up
def deserialize(cls, buf, byteorder='@'):
        '''
        Deserialize a lean MinHash from a buffer.

        Args:
            buf (buffer): `buf` must implement the `buffer`_ interface.
                One such example is the built-in `bytearray`_ class.
            byteorder (str. optional): This is byte order of the serialized data. Use one
                of the `byte order characters
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        Return:
            datasketch.LeanMinHash: The deserialized lean MinHash

        Example:
            To deserialize a lean MinHash from a buffer.

            .. code-block:: python

                lean_minhash = LeanMinHash.deserialize(buf)
        '''
        fmt_seed_size = "%sqi" % byteorder
        fmt_hash = byteorder + "%dI"
        try:
            seed, num_perm = struct.unpack_from(fmt_seed_size, buf, 0)
        except TypeError:
            seed, num_perm = struct.unpack_from(fmt_seed_size, buffer(buf), 0)
        offset = struct.calcsize(fmt_seed_size)
        try:
            hashvalues = struct.unpack_from(fmt_hash % num_perm, buf, offset)
        except TypeError:
            hashvalues = struct.unpack_from(fmt_hash % num_perm, buffer(buf), offset)
        lmh = object.__new__(LeanMinHash)
        lmh._initialize_slots(seed, hashvalues)
        return lmh 
Example 28
Project: terkin-datalogger   Author: hiveeyes   File: epsolar.py    License: GNU Affero General Public License v3.0 5 votes vote down vote up
def read_serial(self):
        string = b'\xAA\x55\xEB\x90\xEB\x90\xEB\x90\x16\xA0\x00\xB1\xA7\x7F'
        self.config.get('ser').write(string)
        payload = self.config.get('ser').readline()
        # 1. decode payload
        try:
            data_length = payload[8]
            pos_data_start = 9
            pos_data_stop = pos_data_start + data_length

            data = payload[pos_data_start:pos_data_stop]

        except Exception as exc:
            # print traceback.format_exc()
            print(exc)
            # self.output(exc)
            serial_read = False
            return False, serial_read

        # 2. decode data
        # http://docs.python.org/2/library/struct.html#format-characters
        try:
            serial_data = list(struct.unpack('hhhhhh???c????bh', data))
            serial_read = True
            # 15 Battery temp., there is 30 difference in value with the real value (spec)
            serial_data[14] -= 35
            # 35 seems to be the right value
            self.serial_data = serial_data
            return serial_data, serial_read

        except Exception as exc:
            serial_read = False
            return False, serial_read 
Example 29
Project: neo-python-core   Author: CityOfZion   File: BinaryReader.py    License: MIT License 5 votes vote down vote up
def unpack(self, fmt, length=1):
        """
        Unpack the stream contents according to the specified format in `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            length (int): amount of bytes to read.

        Returns:
            variable: the result according to the specified format.
        """
        return struct.unpack(fmt, self.stream.read(length))[0] 
Example 30
Project: neo-python-core   Author: CityOfZion   File: BinaryReader.py    License: MIT License 5 votes vote down vote up
def ReadVarInt(self, max=sys.maxsize):
        """
        Read a variable length integer from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max (int): (Optional) maximum number of bytes to read.

        Returns:
            int:
        """
        fb = self.ReadByte()
        if fb is 0:
            return fb
        value = 0
        if hex(fb) == '0xfd':
            value = self.ReadUInt16()
        elif hex(fb) == '0xfe':
            value = self.ReadUInt32()
        elif hex(fb) == '0xff':
            value = self.ReadUInt64()
        else:
            value = fb

        if value > max:
            raise Exception("Invalid format")

        return int(value) 
Example 31
Project: neo-python-core   Author: CityOfZion   File: BinaryReader.py    License: MIT License 5 votes vote down vote up
def ReadVarBytes(self, max=sys.maxsize):
        """
        Read a variable length of bytes from the stream.
        The NEO network protocol supports encoded storage for space saving. See: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            max (int): (Optional) maximum number of bytes to read.

        Returns:
            bytes:
        """
        length = self.ReadVarInt(max)
        return self.ReadBytes(length) 
Example 32
Project: neo-python-core   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 5 votes vote down vote up
def pack(self, fmt, data):
        """
        Write bytes by packing them according to the provided format `fmt`.
        For more information about the `fmt` format see: https://docs.python.org/3/library/struct.html

        Args:
            fmt (str): format string.
            data (object): the data to write to the raw stream.

        Returns:
            int: the number of bytes written.
        """
        return self.WriteBytes(struct.pack(fmt, data), unhex=False) 
Example 33
Project: neo-python-core   Author: CityOfZion   File: BinaryWriter.py    License: MIT License 5 votes vote down vote up
def WriteVarInt(self, value, endian="<"):
        """
        Write an integer value in a space saving way to the stream.
        Read more about variable size encoding here: http://docs.neo.org/en-us/node/network-protocol.html#convention

        Args:
            value (int):
            endian (str): specify the endianness. (Default) Little endian ('<'). Use '>' for big endian.

        Raises:
            TypeError: if `value` is not of type int.
            Exception: if `value` is < 0.

        Returns:
            int: the number of bytes written.
        """
        if not isinstance(value, int):
            raise TypeError('%s not int type.' % value)

        if value < 0:
            raise Exception('%d too small.' % value)

        elif value < 0xfd:
            return self.WriteByte(value)

        elif value <= 0xffff:
            self.WriteByte(0xfd)
            return self.WriteUInt16(value, endian)

        elif value <= 0xFFFFFFFF:
            self.WriteByte(0xfe)
            return self.WriteUInt32(value, endian)

        else:
            self.WriteByte(0xff)
            return self.WriteUInt64(value, endian) 
Example 34
Project: mnist   Author: datapythonista   File: __init__.py    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def parse_idx(fd):
    """Parse an IDX file, and return it as a numpy array.

    Parameters
    ----------
    fd : file
        File descriptor of the IDX file to parse

    endian : str
        Byte order of the IDX file. See [1] for available options

    Returns
    -------
    data : numpy.ndarray
        Numpy array with the dimensions and the data in the IDX file

    1. https://docs.python.org/3/library/struct.html
        #byte-order-size-and-alignment
    """
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, '
                             'file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, '
                             'file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type '
                             '0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items,
                                                          len(data)))

    return numpy.array(data).reshape(dimension_sizes) 
Example 35
def __init__(self):
        
        """
        .hst file format valid as of MT4 574 and later
        Refer to: https://www.mql5.com/en/forum/149178
        
        DATABASE HEADER (148 bytes):
            
        int         version;        // 4 bytes
        string	    copyright[64];	// 64 bytes
        string      symbol[12];     // 12 bytes
        int         period;         // 4 bytes
        int         digits;         // 4 bytes
        datetime	timesign;       // 4 bytes
        datetime	last_sync;      // 4 bytes
        int         unused[13];     // 52 bytes
        
        BAR STRUCTURE (60 bytes):
        
        datetime	ctm;            // 8 bytes
        double	    open;           // 8 bytes
        double      high;           // 8 bytes
        double      low;            // 8 bytes
        double      close;          // 8 bytes
        long        volume;         // 8 bytes
        int         spread;         // 4 bytes
        long        real_volume;	// 8 bytes
        
        Python STRUCT format characters:
        (Ref: https://docs.python.org/3/library/struct.html)
            
        < = (Byte Order: little-endian, Size: standard)
        Q = (C Type: unsigned long long, Python type: integer, Size: 8)
        d = (C Type: double, Python type: float, Size: 8)
        q = (C Type: long long, Python type: integer, Size: 8)
        i = (C Type: int, Python type: integer, Size: 4)    
        """
        
        self._HST_BYTE_FORMAT = '<Qddddqiq'
        self._HEADER_BYTES = 148
        self._BAR_BYTES = 60
    
    ########################################################################## 
Example 36
Project: passbytcp   Author: mxdg   File: common_func.py    License: MIT License 4 votes vote down vote up
def _start(self):
        # memoryview act as an recv buffer
        # refer https://docs.python.org/3/library/stdtypes.html#memoryview
        buff = memoryview(bytearray(RECV_BUFFER_SIZE))
        while True:
            if not self.conn_rd:
                # sleep if there is no connections
                time.sleep(0.06)
                continue

            # blocks until there is socket(s) ready for .recv
            # notice: sockets which were closed by remote,
            #   are also regarded as read-ready by select()
            r, w, e = select.select(self.conn_rd, [], [], 0.5)

            for s in r:  # iter every read-ready or closed sockets
                try:
                    # here, we use .recv_into() instead of .recv()
                    #   recv data directly into the pre-allocated buffer
                    #   to avoid many unnecessary malloc()
                    # see https://docs.python.org/3/library/socket.html#socket.socket.recv_into
                    rec_len = s.recv_into(buff, RECV_BUFFER_SIZE)

                    # agre = "http"
                    # url = agre + '://' + heads['Host']
                    # heads = httphead(buff.tobytes().decode('utf-8'))
                    # logging.info("recv head:{}".format(heads))
                except Exception as e:
                    # unable to read, in most cases, it's due to socket close
                    self._rd_shutdown(s)
                    continue

                if not rec_len:
                    # read zero size, closed or shutdowned socket
                    self._rd_shutdown(s)
                    continue

                try:
                    # send data, we use `buff[:rec_len]` slice because
                    #   only the front of buff is filled
                    self.map[s].send(buff[:rec_len])
                except Exception as e:
                    # unable to send, close connection
                    self._rd_shutdown(s)
                    continue 
Example 37
Project: pyparrot   Author: amymcgovern   File: DroneSensorParser.py    License: MIT License 4 votes vote down vote up
def get_data_format_and_size(data, data_type):
    """
    Internal function to convert data_type to the corresponding struct.pack format string
    as per https://docs.python.org/2/library/struct.html#format-characters

    Function contributed by awm102 on GitHub.  Amy moved this to DroneSensorParser to be
    more general, edited a bit to fit within the drone sensor parser as well.

    :param data: the data that will be packed. Not actually used here unless the data_type is string, then
                 it is used to calculate the data size.
    :param data_type: a string representing the data type
    :return: a tuple of a string representing the struct.pack format for the data type and an int representing
             the number of bytes
    """

    if data_type == "u8" or data_type == "enum":
        format_char = "<B"
        data_size = 1
    elif data_type == "i8":
        format_char = "<b"
        data_size = 1
    elif data_type == "u16":
        format_char = "<H"
        data_size = 2
    elif data_type == "i16":
        format_char = "<h"
        data_size = 2
    elif data_type == "u32":
        format_char = "<I"
        data_size = 4
    elif data_type == "i32":
        format_char = "<i"
        data_size = 4
    elif data_type == "u64":
        format_char = "<Q"
        data_size = 8
    elif data_type == "i64":
        format_char = "<q"
        data_size = 8
    elif data_type == "float":
        format_char = "<f"
        data_size = 4
    elif data_type == "double":
        format_char = "<d"
        data_size = 8
    elif data_type == "string":
        format_char = "<s"
        data_size = len(data)
    else:
        format_char = ""
        data_size = 0

    return (format_char, data_size) 
Example 38
Project: datasketch   Author: ekzhu   File: lean_minhash.py    License: MIT License 4 votes vote down vote up
def serialize(self, buf, byteorder='@'):
        '''
        Serialize this lean MinHash and store the result in an allocated buffer.

        Args:
            buf (buffer): `buf` must implement the `buffer`_ interface.
                One such example is the built-in `bytearray`_ class.
            byteorder (str, optional): This is byte order of the serialized data. Use one
                of the `byte order characters
                <https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment>`_:
                ``@``, ``=``, ``<``, ``>``, and ``!``.
                Default is ``@`` -- the native order.

        This is preferred over using `pickle`_ if the serialized lean MinHash needs
        to be used by another program in a different programming language.

        The serialization schema:
            1. The first 8 bytes is the seed integer
            2. The next 4 bytes is the number of hash values
            3. The rest is the serialized hash values, each uses 4 bytes

        Example:
            To serialize a single lean MinHash into a `bytearray`_ buffer.

            .. code-block:: python

                buf = bytearray(lean_minhash.bytesize())
                lean_minhash.serialize(buf)

            To serialize multiple lean MinHash into a `bytearray`_ buffer.

            .. code-block:: python

                # assuming lean_minhashs is a list of LeanMinHash with the same size
                size = lean_minhashs[0].bytesize()
                buf = bytearray(size*len(lean_minhashs))
                for i, lean_minhash in enumerate(lean_minhashs):
                    lean_minhash.serialize(buf[i*size:])

        .. _`buffer`: https://docs.python.org/3/c-api/buffer.html
        .. _`bytearray`: https://docs.python.org/3.6/library/functions.html#bytearray
        .. _`byteorder`: https://docs.python.org/3/library/struct.html
        '''
        if len(buf) < self.bytesize():
            raise ValueError("The buffer does not have enough space\
                    for holding this MinHash.")
        fmt = "%sqi%dI" % (byteorder, len(self))
        struct.pack_into(fmt, buf, 0,
                self.seed, len(self), *self.hashvalues) 
Example 39
Project: representation_mixing   Author: kastnerkyle   File: loaders.py    License: BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def parse_idx(fd):
    """
    Parse an IDX file, and return it as a numpy array.
    From https://github.com/datapythonista/mnist

    Parameters
    ----------
    fd : file
        File descriptor of the IDX file to parse
    endian : str
        Byte order of the IDX file. See [1] for available options

    Returns
    -------
    data : numpy.ndarray
        Numpy array with the dimensions and the data in the IDX file
    1. https://docs.python.org/3/library/struct.html#byte-order-size-and-alignment
    """
    DATA_TYPES = {0x08: 'B',  # unsigned byte
                  0x09: 'b',  # signed byte
                  0x0b: 'h',  # short (2 bytes)
                  0x0c: 'i',  # int (4 bytes)
                  0x0d: 'f',  # float (4 bytes)
                  0x0e: 'd'}  # double (8 bytes)

    header = fd.read(4)
    if len(header) != 4:
        raise IdxDecodeError('Invalid IDX file, file empty or does not contain a full header.')

    zeros, data_type, num_dimensions = struct.unpack('>HBB', header)

    if zeros != 0:
        raise IdxDecodeError('Invalid IDX file, file must start with two zero bytes. '
                             'Found 0x%02x' % zeros)

    try:
        data_type = DATA_TYPES[data_type]
    except KeyError:
        raise IdxDecodeError('Unknown data type 0x%02x in IDX file' % data_type)

    dimension_sizes = struct.unpack('>' + 'I' * num_dimensions,
                                    fd.read(4 * num_dimensions))

    data = array.array(data_type, fd.read())
    data.byteswap()  # looks like array.array reads data as little endian

    expected_items = functools.reduce(operator.mul, dimension_sizes)
    if len(data) != expected_items:
        raise IdxDecodeError('IDX file has wrong number of items. '
                             'Expected: %d. Found: %d' % (expected_items, len(data)))

    return np.array(data).reshape(dimension_sizes)