Python struct.iter_unpack() Examples

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def load( self, mem_image ):

    # Iterate over the sections

    sections = mem_image.get_sections()
    for section in sections:

      # For .mngr2proc sections, copy section into mngr2proc src

      if section.name == ".mngr2proc":
        self.src.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For .proc2mngr sections, copy section into proc2mngr_ref src

      elif section.name == ".proc2mngr":
        self.sink.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For all other sections, simply copy them into the memory

      else:
        self.mem.write_mem( section.addr, section.data )

  #-----------------------------------------------------------------------
  # done
  #----------------------------------------------------------------------- 

def _read_file(self, mode='r'):
        """Read in the directory information for the pak file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadPakFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.directory_size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = PakInfo(entry.filename, entry.file_offset, entry.file_size)
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 

def _read_file(self, mode='r'):
        """Read in the directory information for the wad file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadWadFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.lump_count * Entry.size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = WadInfo(entry.filename, entry.file_offset, entry.file_size)
            info.compression = entry.compression
            info.disk_size = entry.disk_size
            info.type = entry.type

            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 

def _read_file(self, mode='r'):
        """Read in the directory information for the grp file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.signature != IDENTITY:
            raise BadGrpFile(f'Bad magic number: {header.signature}')

        size_of_directory = header.number_of_entries * Entry.size
        data = self.fp.read(size_of_directory)

        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]
        offset = Header.size + size_of_directory

        for entry in entries:
            info = GrpInfo(entry.filename, offset, entry.file_size)
            offset += info.file_size
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info

        if mode == 'a':
            self.data_buffer = io.BytesIO(self.fp.read()) 

def _read_file(self, mode='r'):
        """Read in the directory information for the pak file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadPakFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.directory_size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = PakInfo(entry.filename, entry.file_offset, entry.file_size)
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 

def receive_rel(sock: socket, num_cols: int):
    num_elements_bytes = sock.recv(INT_SIZE)
    num_elements = struct.unpack('i', num_elements_bytes)[0]
    total_size = num_elements * INT_SIZE
    byte_buf = bytearray(total_size)
    view = memoryview(byte_buf)
    while total_size:
        received = sock.recv_into(view, total_size)
        view = view[received:]
        total_size -= received
    element_it = struct.iter_unpack("i", byte_buf)

    rel = []
    col_idx = 0
    row = []
    for el in element_it:
        row.append(el[0])
        col_idx += 1
        if col_idx >= num_cols:
            rel.append(row)
            col_idx = 0
            row = []
    byte_buf = None
    return rel 

def receive_set(sock: socket):
    num_elements_bytes = sock.recv(INT_SIZE)
    num_elements = struct.unpack('i', num_elements_bytes)[0]
    total_size = num_elements * INT_SIZE
    byte_buf = bytearray(total_size)
    view = memoryview(byte_buf)
    while total_size:
        received = sock.recv_into(view, total_size)
        view = view[received:]
        total_size -= received
    element_it = struct.iter_unpack("i", byte_buf)

    rel = set()
    for el in element_it:
        rel.add(el[0])
    byte_buf = None
    return rel 

def __iter__(self):
        if self.stream is None or self.audio_controller is None:
            return
        else:
            for i in range(self.frames_per_buffer):
                data = self.stream.read(self.frames_per_buffer, exception_on_overflow=False)
                fmt = "<H"
                data = np.array(list(struct.iter_unpack(fmt, data)))
                yield self.feature_extractor(data) 

def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return chunks(raw) 

def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return struct.iter_unpack(EVENT_FORMAT, raw) 

def _do_iter(self):
        read_size = self._get_total_read_size()
        data = self._get_data(read_size)
        if not data:
            return None
        evdev_objects = iter_unpack(data)
        events = [self._make_event(*event) for event in evdev_objects]
        return events

    # pylint: disable=too-many-arguments 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 

def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def read_file_list(f, entry_num):
    def decode(data, index):
        for (n, item) in enumerate(data):
            item = item[0]
            item -= 39
            item ^= 0xA5
            item -= (27 + index + n)
            yield item % 256
    def to_str(data):
        res_string = ''
        for item in data:
            if item >= 0x21 and item <= 0x7E:
                res_string += '{:c}'.format(item)
        return res_string
    def to_int(data):
        result = 0
        data = list(data)[::-1]
        for item in data:
            result <<= 8
            result += item
        return result
    result = []
    last_offs, offs = 0, 0
    Item = namedtuple('Item', 'name offset size')
    for index in range(0, entry_num * 17, 17):
        last_offs = offs
        raw = list(decode(iter_unpack('B', f.read(17)), index))
        name = to_str(raw[:13])
        offs = to_int(raw[13:])
        if index > 1:
            pos = index // 17 - 1
            result[pos] = Item(result[pos][0], result[pos][1], offs - last_offs)
        result.append(Item(name, offs, 0))
    result.pop()
    return result 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 

def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 

def parse(cls, stream):
        num, protocol, spi_size, n_transforms = struct.unpack('>BBBB', stream.read(4))
        spi = stream.read(spi_size)
        transforms = []
        more = True
        while more:
            more, length, type, id = struct.unpack('>BxHBxH', stream.read(8))
            keylen = None
            for attr_type, value in struct.iter_unpack('>HH', stream.read(length-8)):
                if attr_type & 0x7FFF == 0xe:
                    keylen = value
            transforms.append(Transform(enums.Transform(type), enums.TransformTable[type](id), keylen))
        return Proposal(num, protocol, spi, transforms) 

def iter_i24_as_i32(data):
    for l, h in struct.iter_unpack('<BH', data):
        yield twos_complement(h << 8 | l, 24) << 8 

def iter_i16_as_i32(data):
    for x, in struct.iter_unpack('<h', data):
        yield x << 16 

def __init__(
            self,
            path,
            #  create_if_missing=False,
            #  error_if_exists=False,
            #  paranoid_checks=None,
            #  write_buffer_size=None,
            #  max_open_files=None,
            #  lru_cache_size=None,
            #  block_size=None,
            #  block_restart_interval=None,
            #  compression='snappy',
            #  bloom_filter_bits=0,
            #  comparator=None,
            #  comparator_name=None
    ):
        super(Emstore, self).__init__()
        _, _, _, kwargs = getargvalues(currentframe())
        name = kwargs['path']
        self.path = name
        kwargs = {
            k: v
            for k, v in kwargs.items()
            if k in [
                'create_if_missing', 'error_if_exists', 'paranoid_checks',
                'write_buffer_size', 'max_open_files', 'lru_cache_size',
                'block_size', 'block_restart_interval', 'compression',
                'bloom_filter_bits', 'comparator', 'comparator_name'
            ]
        }
        db = plyvel.DB(name, **kwargs)
        vector = struct.iter_unpack(STRUCT_FORMAT, next(db.__iter__())[1])
        vector_size = len(list(vector))
        self.vector_size = vector_size
        db.close()
        self.db = plyvel.DB(name, **kwargs)
        self.unpack = struct.Struct(str(vector_size) + STRUCT_FORMAT).unpack 

def convert_palette(palette, max_colors):
    if not palette:
        raise ValueError('Image must have a palette')

    data_type, pdata = palette.getdata()

    if data_type != 'RGB':
        raise ValueError('Image palette is invalid')

    n_colors = len(pdata) / 3
    if n_colors > max_colors:
        raise ValueError('Image palette has too many colors')

    snes_pal_data = bytearray()

    for c in struct.iter_unpack('BBB', pdata):
        r, g, b = c

        b = (b >> 3) & 31;
        g = (g >> 3) & 31;
        r = (r >> 3) & 31;

        u16 = (b << 10) | (g << 5) | r;

        snes_pal_data.append(u16 & 0xff);
        snes_pal_data.append(u16 >> 8);

    assert len(snes_pal_data) == n_colors * 2;

    return snes_pal_data 

def decode_chunk_into(chunk, buf, block_size):
    num_channels = chunk.shape[0]
    # Grid size (number of blocks in the chunk)
    gx = ceil_div(chunk.shape[3], block_size[0])
    gy = ceil_div(chunk.shape[2], block_size[1])
    gz = ceil_div(chunk.shape[1], block_size[2])

    if len(buf) < num_channels * (4 + 8 * gx * gy * gz):
        raise InvalidFormatError("compressed_segmentation file too short")

    if sys.version_info < (3,):
        channel_offsets = struct.unpack("<I", buf[:4*num_channels])
        channel_offsets = [ 4 * ret for ret in channel_offsets ]
    else:
        channel_offsets = [
            4 * ret[0] for ret in struct.iter_unpack("<I", buf[:4*num_channels])
        ]

    for channel, (offset, next_offset) in \
        enumerate(zip_longest(channel_offsets, channel_offsets[1:])):

        # next_offset will be None for the last channel
        if offset + 8 * gx * gy * gz > len(buf):
            raise InvalidFormatError("compressed_segmentation channel offset "
                                     "is too large (truncated file?)")
        _decode_channel_into(
            chunk, channel, buf[offset:next_offset], block_size
        )

    return chunk 

def read_array_data(io, data_struct):
	struct_size = struct.calcsize(data_struct)
	data_struct = "<" + data_struct # force little endianness

	count = struct.unpack("<I", io.read(4))[0]
	data = io.read(count * struct_size)
	unpacked_data = list(struct.iter_unpack(data_struct, data))
	return [tup[0] if len(tup) == 1 else tup for tup in unpacked_data ] 

def get_mijia360_gyro(img_path):
    im = Image.open(img_path)
    exif_data = im._getexif()
    USERCOMMENT = None
    for intval, strval in PIL.ExifTags.TAGS.items():
        if strval == 'UserComment':
            USERCOMMENT = intval
    raw_data = exif_data[USERCOMMENT]
    return [e[0] for e in struct.iter_unpack('f', raw_data)] 

def db_neighbors(db, keyfmt='I', valfmt='f'):
    fmt = keyfmt + valfmt
    def get_neighbors(i):
        return list((d, idx)
                    for idx, d in iter_unpack(fmt, db[pack(keyfmt, i)]))
    return get_neighbors 

def hnsw_distances(data, d, m=5, ef=200, m0=None, level_mult=None,
                   heuristic=True, balanced=True, fmt='fI', print_every=None):

    distances = [bytearray() for _ in range(len(data))]
    def decorated_d(i, j):
        dist = d(data[i], data[j])
        distances[i].append(pack(fmt, dist, j))
        distances[j].extend(pack(fmt, dist, i))
        return dist
    
    hnsw = HNSW(decorated_d, m, ef, m0, level_mult, heuristic)
    add = hnsw.balanced_add if balanced else hnsw.add
    for i in range(len(data)):
        if print_every and i % print_every == 0 and i != 0:
            print(i)
        add(i)
    hnsw.distance = d

    size = calcsize(fmt)
    fmt0 = fmt[0]
    def sort_elem(dd):
        # assert len(dd) % size == 0
        def key(j):
            return unpack_from(fmt0, dd, size * j)
        ddsorted = bytearray()
        for j in sorted(range(len(dd) // size), key=key):
            start = size * j
            ddsorted.extend(dd[start:start+size])
        #assert ([d for d, _ in iter_unpack(fmt, ddsorted)]
        #        == sorted(d for d, _ in iter_unpack(fmt, ddsorted)))
        return ddsorted

    return hnsw, [sort_elem(dd) for dd in distances] 

def get_get_neighbors(distances, fmt='fI'):
    def get_neighbors(i):
        return iter_unpack(fmt, distances[i])
    return get_neighbors 

def compute_optics(data, d, minpts, m=5, ef=200, m0=None, level_mult=None,
                   heuristic=True, balanced=True, fmt='fI'):
    distances = hnsw_distances(data, d, m, ef, m0, level_mult, heuristic,
                               balanced, fmt)[1]
    print(sum(map(len, distances)) // 2 // calcsize(fmt))
    for i, dd in enumerate(distances):
        print(i, list(iter_unpack(fmt, dd))[:5])
    return optics.optics(len(data), minpts, get_get_neighbors(distances, fmt)) 

def decode_ipv4(buffer: bytes) -> Tuple[str, int]:
    try:
        ip_str, port = struct.unpack(">4sH", buffer)
        ip = ".".join([str(n) for n, in struct.iter_unpack(">B", ip_str)])
        return ip, port
    except struct.error:
        pass
    raise ValueError("Invalid (ip, port)") 

def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]
        flag_spar = 0

        if self.config["SurfaceParVoltageAdded"]:
            flag_spar = 1

        the_bytes = b"".join(self.raw_bytes)

        #specify how each line of raw data should be broken down
        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "2s" * flag_spar + "4s" * flag_spar + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6 - flag_spar * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        measurements = measurements[:,0:-1] #remove after applying _parse_scans_meta in here
        #print(measurements.shape)
        return measurements 

def _breakdown(self, line, ordering):
        '''
        Convert hex metadata to science useable data.

        Steps:
        1. Throw away away first part of the line
        2. Start processing the rest of the line
        3. Return a sequence (tuple? list?)

        Input:
        line - a sequence specified by format in _parse_scans_meta
        ordering - the order the string comes in

        Output:
        output - a string of converted data according to format

        Format:
        Lat, Lon, pressure temp, bottle fire status, NMEA time, Scan time
        '''
        output = ''
        #print(ordering, line)
        for x, y in zip(ordering, line):
            if x == 'nmea_pos':
                tokens = []
                #print(y)
                for t in struct.iter_unpack("2s2s2s2s2s2s2s", y):
                    for tt in t:
                        tokens.append(tt)
                output = output + str(self._location_fix(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6])) + ','
            elif x == 'nmea_time':
                output = output + str(self._sbe_time(self._reverse_bytes(y), 'nmea')) + ','
            elif x == 'scan_time':
                output = output + str(self._sbe_time(self._reverse_bytes(y), 'scan'))
            elif x == 'flag_ctd_status':
                output = output + str(self._pump_status(y)) + ',' + str(self._bottle_fire(y)) + ',' #need to make final comma dynamic later so less errors down the line
            elif x == 'pressure_temp':
                output = output + str(int(y, 16)) + ','

        return output 

def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]

        the_bytes = b"".join(self.raw_bytes)

        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        return measurements 

def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]

        the_bytes = b"".join(self.raw_bytes)

        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        return measurements 

def read_analog_samples(self, n, sample_width):
        if self._transactions:
            raise TypeError("Command transaction in progress")
        if sample_width == 2:
            data = await self._reader.readexactly(2*n)
            return array.array('f', ((value+32768)/65536 for (value,) in struct.iter_unpack('>h', data)))
        elif sample_width == 1:
            data = await self._reader.readexactly(n)
            return array.array('f', (value/256 for value in data))
        else:
            raise ValueError(f"Bad sample width: {sample_width}") 

def read(cls, file):
        data = file.read(cls.size)
        lumps_start = struct.calcsize('<i')

        version_data = data[:lumps_start]
        version_struct = struct.unpack('<i', version_data)
        version = version_struct[0]

        lumps_data = data[lumps_start:]
        lumps = [Lump(*l) for l in struct.iter_unpack(Lump.format, lumps_data)]

        return Header(version, lumps) 

def _read_file(cls, file, mode):
        def _read_iter_lump(lump, class_):
            """Read iteratively from file unpacking according to format
            class attribute on the class_ object"""
            offset, size = lump.offset, lump.length
            file.seek(offset)

            return [class_(*s) for s in struct.iter_unpack(class_.format, file.read(size))]

        def _read_lump(lump, reader):
            """Read from file using the provided reader object"""
            file.seek(lump.offset)
            return reader.read(file, lump.length)

        bsp = cls()
        bsp.mode = mode
        bsp.fp = file

        factory = bsp.factory

        # Header
        header = factory.Header.read(file)
        bsp.version = header.version

        bsp.entities = _read_lump(header.entities, _Entities)
        bsp.planes = _read_iter_lump(header.planes, factory.Plane)
        bsp.miptextures = _read_lump(header.miptextures, _Miptextures)
        bsp.vertexes = _read_iter_lump(header.vertexes, factory.Vertex)
        bsp.visibilities = _read_lump(header.visibilities, _Visibilities)
        bsp.nodes = _read_iter_lump(header.nodes, factory.Node)
        bsp.texture_infos = _read_iter_lump(header.texture_infos, factory.TextureInfo)
        bsp.faces = _read_iter_lump(header.faces, factory.Face)
        bsp.lighting = _read_lump(header.lighting, _Lighting)
        bsp.clip_nodes = _read_iter_lump(header.clip_nodes, factory.ClipNode)
        bsp.leafs = _read_iter_lump(header.leafs, factory.Leaf)
        bsp.mark_surfaces = _read_lump(header.mark_surfaces, _MarkSurfaces)
        bsp.edges = _read_iter_lump(header.edges, factory.Edge)
        bsp.surf_edges = _read_lump(header.surf_edges, _SurfEdges)
        bsp.models = _read_iter_lump(header.models, factory.Model)

        return bsp 

def read(cls, file):
        return [cls.Class(*st) for st in struct.iter_unpack(cls.Class.format, file.read())] 

def read(cls, file):
        return [cls.Class(*c) for c in struct.iter_unpack(cls.Class.format, file.read())] 

def read(file, size=-1):
        return [lf[0] for lf in struct.iter_unpack('<H', file.read(size))] 

def read(file, size=-1):
        return [lb[0] for lb in struct.iter_unpack('<H', file.read(size))] 

def read(file, size=-1):
        return [se[0] for se in struct.iter_unpack('<H', file.read(size))] 

def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it)