Python struct.iter_unpack() Examples

The following are code examples for showing how to use struct.iter_unpack(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: NiujiaoDebugger   Author: MrSrc   File: test_struct.py    GNU General Public License v3.0 7 votes vote down vote up
def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 
Example 2
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_struct.py    GNU General Public License v2.0 6 votes vote down vote up
def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 
Example 3
Project: pymtl3   Author: cornell-brg   File: harness.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def load( self, mem_image ):

    # Iterate over the sections

    sections = mem_image.get_sections()
    for section in sections:

      # For .mngr2proc sections, copy section into mngr2proc src

      if section.name == ".mngr2proc":
        self.src.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For .proc2mngr sections, copy section into proc2mngr_ref src

      elif section.name == ".proc2mngr":
        self.sink.msgs.extend(Bits32(bits[0]) for bits in struct.iter_unpack("<I", section.data))

      # For all other sections, simply copy them into the memory

      else:
        self.mem.write_mem( section.addr, section.data )

  #-----------------------------------------------------------------------
  # done
  #----------------------------------------------------------------------- 
Example 4
Project: game-tools   Author: joshuaskelly   File: pak.py    MIT License 6 votes vote down vote up
def _read_file(self, mode='r'):
        """Read in the directory information for the pak file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadPakFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.directory_size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = PakInfo(entry.filename, entry.file_offset, entry.file_size)
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 
Example 5
Project: game-tools   Author: joshuaskelly   File: wad.py    MIT License 6 votes vote down vote up
def _read_file(self, mode='r'):
        """Read in the directory information for the wad file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadWadFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.lump_count * Entry.size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = WadInfo(entry.filename, entry.file_offset, entry.file_size)
            info.compression = entry.compression
            info.disk_size = entry.disk_size
            info.type = entry.type

            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 
Example 6
Project: game-tools   Author: joshuaskelly   File: grp.py    MIT License 6 votes vote down vote up
def _read_file(self, mode='r'):
        """Read in the directory information for the grp file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.signature != IDENTITY:
            raise BadGrpFile(f'Bad magic number: {header.signature}')

        size_of_directory = header.number_of_entries * Entry.size
        data = self.fp.read(size_of_directory)

        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]
        offset = Header.size + size_of_directory

        for entry in entries:
            info = GrpInfo(entry.filename, offset, entry.file_size)
            offset += info.file_size
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info

        if mode == 'a':
            self.data_buffer = io.BytesIO(self.fp.read()) 
Example 7
Project: game-tools   Author: joshuaskelly   File: pak.py    MIT License 6 votes vote down vote up
def _read_file(self, mode='r'):
        """Read in the directory information for the pak file."""
        self.fp.seek(0)
        header = Header.read(self.fp)

        if header.identity != IDENTITY:
            raise BadPakFile(f'Bad magic number: {header.identity}')

        self.end_of_data = header.directory_offset
        size_of_directory = header.directory_size

        self.fp.seek(self.end_of_data)
        data = self.fp.read(size_of_directory)
        entries = [Entry(*e) for e in struct.iter_unpack(Entry.format, data)]

        for entry in entries:
            info = PakInfo(entry.filename, entry.file_offset, entry.file_size)
            self.file_list.append(info)
            self.NameToInfo[info.filename] = info 
Example 8
Project: Project-New-Reign---Nemesis-Main   Author: ShikyoKira   File: test_struct.py    GNU General Public License v3.0 6 votes vote down vote up
def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 
Example 9
Project: cells   Author: AlesTsurko   File: test_struct.py    MIT License 6 votes vote down vote up
def test_construct(self):
        def _check_iterator(it):
            self.assertIsInstance(it, abc.Iterator)
            self.assertIsInstance(it, abc.Iterable)
        s = struct.Struct('>ibcp')
        it = s.iter_unpack(b"")
        _check_iterator(it)
        it = s.iter_unpack(b"1234567")
        _check_iterator(it)
        # Wrong bytes length
        with self.assertRaises(struct.error):
            s.iter_unpack(b"123456")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12345678")
        # Zero-length struct
        s = struct.Struct('>')
        with self.assertRaises(struct.error):
            s.iter_unpack(b"")
        with self.assertRaises(struct.error):
            s.iter_unpack(b"12") 
Example 10
Project: conclave   Author: multiparty   File: python.py    MIT License 6 votes vote down vote up
def receive_rel(sock: socket, num_cols: int):
    num_elements_bytes = sock.recv(INT_SIZE)
    num_elements = struct.unpack('i', num_elements_bytes)[0]
    total_size = num_elements * INT_SIZE
    byte_buf = bytearray(total_size)
    view = memoryview(byte_buf)
    while total_size:
        received = sock.recv_into(view, total_size)
        view = view[received:]
        total_size -= received
    element_it = struct.iter_unpack("i", byte_buf)

    rel = []
    col_idx = 0
    row = []
    for el in element_it:
        row.append(el[0])
        col_idx += 1
        if col_idx >= num_cols:
            rel.append(row)
            col_idx = 0
            row = []
    byte_buf = None
    return rel 
Example 11
Project: conclave   Author: multiparty   File: python.py    MIT License 6 votes vote down vote up
def receive_set(sock: socket):
    num_elements_bytes = sock.recv(INT_SIZE)
    num_elements = struct.unpack('i', num_elements_bytes)[0]
    total_size = num_elements * INT_SIZE
    byte_buf = bytearray(total_size)
    view = memoryview(byte_buf)
    while total_size:
        received = sock.recv_into(view, total_size)
        view = view[received:]
        total_size -= received
    element_it = struct.iter_unpack("i", byte_buf)

    rel = set()
    for el in element_it:
        rel.add(el[0])
    byte_buf = None
    return rel 
Example 12
Project: SyNEThesia   Author: RunOrVeith   File: live_viewer.py    MIT License 5 votes vote down vote up
def __iter__(self):
        if self.stream is None or self.audio_controller is None:
            return
        else:
            for i in range(self.frames_per_buffer):
                data = self.stream.read(self.frames_per_buffer, exception_on_overflow=False)
                fmt = "<H"
                data = np.array(list(struct.iter_unpack(fmt, data)))
                yield self.feature_extractor(data) 
Example 13
Project: inputs   Author: zeth   File: inputs.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return chunks(raw) 
Example 14
Project: inputs   Author: zeth   File: inputs.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def iter_unpack(raw):
        """Yield successive EVENT_SIZE chunks from message."""
        return struct.iter_unpack(EVENT_FORMAT, raw) 
Example 15
Project: inputs   Author: zeth   File: inputs.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _do_iter(self):
        read_size = self._get_total_read_size()
        data = self._get_data(read_size)
        if not data:
            return None
        evdev_objects = iter_unpack(data)
        events = [self._make_event(*event) for event in evdev_objects]
        return events

    # pylint: disable=too-many-arguments 
Example 16
Project: NiujiaoDebugger   Author: MrSrc   File: test_struct.py    GNU General Public License v3.0 5 votes vote down vote up
def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 17
Project: NiujiaoDebugger   Author: MrSrc   File: test_struct.py    GNU General Public License v3.0 5 votes vote down vote up
def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 18
Project: NiujiaoDebugger   Author: MrSrc   File: test_struct.py    GNU General Public License v3.0 5 votes vote down vote up
def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 
Example 19
Project: NiujiaoDebugger   Author: MrSrc   File: test_struct.py    GNU General Public License v3.0 5 votes vote down vote up
def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 20
Project: dockerizeme   Author: dockerizeme   File: snippet.py    Apache License 2.0 5 votes vote down vote up
def read_file_list(f, entry_num):
    def decode(data, index):
        for (n, item) in enumerate(data):
            item = item[0]
            item -= 39
            item ^= 0xA5
            item -= (27 + index + n)
            yield item % 256
    def to_str(data):
        res_string = ''
        for item in data:
            if item >= 0x21 and item <= 0x7E:
                res_string += '{:c}'.format(item)
        return res_string
    def to_int(data):
        result = 0
        data = list(data)[::-1]
        for item in data:
            result <<= 8
            result += item
        return result
    result = []
    last_offs, offs = 0, 0
    Item = namedtuple('Item', 'name offset size')
    for index in range(0, entry_num * 17, 17):
        last_offs = offs
        raw = list(decode(iter_unpack('B', f.read(17)), index))
        name = to_str(raw[:13])
        offs = to_int(raw[13:])
        if index > 1:
            pos = index // 17 - 1
            result[pos] = Item(result[pos][0], result[pos][1], offs - last_offs)
        result.append(Item(name, offs, 0))
    result.pop()
    return result 
Example 21
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_struct.py    GNU General Public License v2.0 5 votes vote down vote up
def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 22
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_struct.py    GNU General Public License v2.0 5 votes vote down vote up
def test_arbitrary_buffer(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 11))
        it = s.iter_unpack(memoryview(b))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 23
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_struct.py    GNU General Public License v2.0 5 votes vote down vote up
def test_length_hint(self):
        lh = operator.length_hint
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(lh(it), 3)
        next(it)
        self.assertEqual(lh(it), 2)
        next(it)
        self.assertEqual(lh(it), 1)
        next(it)
        self.assertEqual(lh(it), 0)
        self.assertRaises(StopIteration, next, it)
        self.assertEqual(lh(it), 0) 
Example 24
Project: cqp-sdk-for-py37-native   Author: crud-boy   File: test_struct.py    GNU General Public License v2.0 5 votes vote down vote up
def test_module_func(self):
        # Sanity check for the global struct.iter_unpack()
        it = struct.iter_unpack('>IB', bytes(range(1, 11)))
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it) 
Example 25
Project: python-vpn   Author: winbotscript   File: message.py    MIT License 5 votes vote down vote up
def parse(cls, stream):
        num, protocol, spi_size, n_transforms = struct.unpack('>BBBB', stream.read(4))
        spi = stream.read(spi_size)
        transforms = []
        more = True
        while more:
            more, length, type, id = struct.unpack('>BxHBxH', stream.read(8))
            keylen = None
            for attr_type, value in struct.iter_unpack('>HH', stream.read(length-8)):
                if attr_type & 0x7FFF == 0xe:
                    keylen = value
            transforms.append(Transform(enums.Transform(type), enums.TransformTable[type](id), keylen))
        return Proposal(num, protocol, spi, transforms) 
Example 26
Project: mqaid   Author: redsudo   File: is_mqa.py    MIT License 5 votes vote down vote up
def iter_i24_as_i32(data):
    for l, h in struct.iter_unpack('<BH', data):
        yield twos_complement(h << 8 | l, 24) << 8 
Example 27
Project: mqaid   Author: redsudo   File: is_mqa.py    MIT License 5 votes vote down vote up
def iter_i16_as_i32(data):
    for x, in struct.iter_unpack('<h', data):
        yield x << 16 
Example 28
Project: emstore   Author: MichaMucha   File: emstore.py    MIT License 5 votes vote down vote up
def __init__(
            self,
            path,
            #  create_if_missing=False,
            #  error_if_exists=False,
            #  paranoid_checks=None,
            #  write_buffer_size=None,
            #  max_open_files=None,
            #  lru_cache_size=None,
            #  block_size=None,
            #  block_restart_interval=None,
            #  compression='snappy',
            #  bloom_filter_bits=0,
            #  comparator=None,
            #  comparator_name=None
    ):
        super(Emstore, self).__init__()
        _, _, _, kwargs = getargvalues(currentframe())
        name = kwargs['path']
        self.path = name
        kwargs = {
            k: v
            for k, v in kwargs.items()
            if k in [
                'create_if_missing', 'error_if_exists', 'paranoid_checks',
                'write_buffer_size', 'max_open_files', 'lru_cache_size',
                'block_size', 'block_restart_interval', 'compression',
                'bloom_filter_bits', 'comparator', 'comparator_name'
            ]
        }
        db = plyvel.DB(name, **kwargs)
        vector = struct.iter_unpack(STRUCT_FORMAT, next(db.__iter__())[1])
        vector_size = len(list(vector))
        self.vector_size = vector_size
        db.close()
        self.db = plyvel.DB(name, **kwargs)
        self.unpack = struct.Struct(str(vector_size) + STRUCT_FORMAT).unpack 
Example 29
Project: maze-runner   Author: undisbeliever   File: png2snes.py    MIT License 5 votes vote down vote up
def convert_palette(palette, max_colors):
    if not palette:
        raise ValueError('Image must have a palette')

    data_type, pdata = palette.getdata()

    if data_type != 'RGB':
        raise ValueError('Image palette is invalid')

    n_colors = len(pdata) / 3
    if n_colors > max_colors:
        raise ValueError('Image palette has too many colors')

    snes_pal_data = bytearray()

    for c in struct.iter_unpack('BBB', pdata):
        r, g, b = c

        b = (b >> 3) & 31;
        g = (g >> 3) & 31;
        r = (r >> 3) & 31;

        u16 = (b << 10) | (g << 5) | r;

        snes_pal_data.append(u16 & 0xff);
        snes_pal_data.append(u16 >> 8);

    assert len(snes_pal_data) == n_colors * 2;

    return snes_pal_data 
Example 30
Project: cloud-volume   Author: seung-lab   File: py_compressed_segmentation.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def decode_chunk_into(chunk, buf, block_size):
    num_channels = chunk.shape[0]
    # Grid size (number of blocks in the chunk)
    gx = ceil_div(chunk.shape[3], block_size[0])
    gy = ceil_div(chunk.shape[2], block_size[1])
    gz = ceil_div(chunk.shape[1], block_size[2])

    if len(buf) < num_channels * (4 + 8 * gx * gy * gz):
        raise InvalidFormatError("compressed_segmentation file too short")

    if sys.version_info < (3,):
        channel_offsets = struct.unpack("<I", buf[:4*num_channels])
        channel_offsets = [ 4 * ret for ret in channel_offsets ]
    else:
        channel_offsets = [
            4 * ret[0] for ret in struct.iter_unpack("<I", buf[:4*num_channels])
        ]

    for channel, (offset, next_offset) in \
        enumerate(zip_longest(channel_offsets, channel_offsets[1:])):

        # next_offset will be None for the last channel
        if offset + 8 * gx * gy * gz > len(buf):
            raise InvalidFormatError("compressed_segmentation channel offset "
                                     "is too large (truncated file?)")
        _decode_channel_into(
            chunk, channel, buf[offset:next_offset], block_size
        )

    return chunk 
Example 31
Project: blender-datasmith-export   Author: 0xafbf   File: data_types.py    GNU General Public License v3.0 5 votes vote down vote up
def read_array_data(io, data_struct):
	struct_size = struct.calcsize(data_struct)
	data_struct = "<" + data_struct # force little endianness

	count = struct.unpack("<I", io.read(4))[0]
	data = io.read(count * struct_size)
	unpacked_data = list(struct.iter_unpack(data_struct, data))
	return [tup[0] if len(tup) == 1 else tup for tup in unpacked_data ] 
Example 32
Project: FishPyPano   Author: rnbguy   File: mijia.py    MIT License 5 votes vote down vote up
def get_mijia360_gyro(img_path):
    im = Image.open(img_path)
    exif_data = im._getexif()
    USERCOMMENT = None
    for intval, strval in PIL.ExifTags.TAGS.items():
        if strval == 'UserComment':
            USERCOMMENT = intval
    raw_data = exif_data[USERCOMMENT]
    return [e[0] for e in struct.iter_unpack('f', raw_data)] 
Example 33
Project: flexible-clustering   Author: matteodellamico   File: hnsw_optics.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def db_neighbors(db, keyfmt='I', valfmt='f'):
    fmt = keyfmt + valfmt
    def get_neighbors(i):
        return list((d, idx)
                    for idx, d in iter_unpack(fmt, db[pack(keyfmt, i)]))
    return get_neighbors 
Example 34
Project: flexible-clustering   Author: matteodellamico   File: hnsw_optics.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def hnsw_distances(data, d, m=5, ef=200, m0=None, level_mult=None,
                   heuristic=True, balanced=True, fmt='fI', print_every=None):

    distances = [bytearray() for _ in range(len(data))]
    def decorated_d(i, j):
        dist = d(data[i], data[j])
        distances[i].append(pack(fmt, dist, j))
        distances[j].extend(pack(fmt, dist, i))
        return dist
    
    hnsw = HNSW(decorated_d, m, ef, m0, level_mult, heuristic)
    add = hnsw.balanced_add if balanced else hnsw.add
    for i in range(len(data)):
        if print_every and i % print_every == 0 and i != 0:
            print(i)
        add(i)
    hnsw.distance = d

    size = calcsize(fmt)
    fmt0 = fmt[0]
    def sort_elem(dd):
        # assert len(dd) % size == 0
        def key(j):
            return unpack_from(fmt0, dd, size * j)
        ddsorted = bytearray()
        for j in sorted(range(len(dd) // size), key=key):
            start = size * j
            ddsorted.extend(dd[start:start+size])
        #assert ([d for d, _ in iter_unpack(fmt, ddsorted)]
        #        == sorted(d for d, _ in iter_unpack(fmt, ddsorted)))
        return ddsorted

    return hnsw, [sort_elem(dd) for dd in distances] 
Example 35
Project: flexible-clustering   Author: matteodellamico   File: hnsw_optics.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def get_get_neighbors(distances, fmt='fI'):
    def get_neighbors(i):
        return iter_unpack(fmt, distances[i])
    return get_neighbors 
Example 36
Project: flexible-clustering   Author: matteodellamico   File: hnsw_optics.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def compute_optics(data, d, minpts, m=5, ef=200, m0=None, level_mult=None,
                   heuristic=True, balanced=True, fmt='fI'):
    distances = hnsw_distances(data, d, m, ef, m0, level_mult, heuristic,
                               balanced, fmt)[1]
    print(sum(map(len, distances)) // 2 // calcsize(fmt))
    for i, dd in enumerate(distances):
        print(i, list(iter_unpack(fmt, dd))[:5])
    return optics.optics(len(data), minpts, get_get_neighbors(distances, fmt)) 
Example 37
Project: PyTo   Author: nbedos   File: utilities.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def decode_ipv4(buffer: bytes) -> Tuple[str, int]:
    try:
        ip_str, port = struct.unpack(">4sH", buffer)
        ip = ".".join([str(n) for n, in struct.iter_unpack(">B", ip_str)])
        return ip, port
    except struct.error:
        pass
    raise ValueError("Invalid (ip, port)") 
Example 38
Project: odf-ctd-proc   Author: somts   File: sbe_reader.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]
        flag_spar = 0

        if self.config["SurfaceParVoltageAdded"]:
            flag_spar = 1

        the_bytes = b"".join(self.raw_bytes)

        #specify how each line of raw data should be broken down
        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "2s" * flag_spar + "4s" * flag_spar + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6 - flag_spar * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        measurements = measurements[:,0:-1] #remove after applying _parse_scans_meta in here
        #print(measurements.shape)
        return measurements 
Example 39
Project: odf-ctd-proc   Author: somts   File: sbe_reader.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _breakdown(self, line, ordering):
        '''
        Convert hex metadata to science useable data.

        Steps:
        1. Throw away away first part of the line
        2. Start processing the rest of the line
        3. Return a sequence (tuple? list?)

        Input:
        line - a sequence specified by format in _parse_scans_meta
        ordering - the order the string comes in

        Output:
        output - a string of converted data according to format

        Format:
        Lat, Lon, pressure temp, bottle fire status, NMEA time, Scan time
        '''
        output = ''
        #print(ordering, line)
        for x, y in zip(ordering, line):
            if x == 'nmea_pos':
                tokens = []
                #print(y)
                for t in struct.iter_unpack("2s2s2s2s2s2s2s", y):
                    for tt in t:
                        tokens.append(tt)
                output = output + str(self._location_fix(tokens[0], tokens[1], tokens[2], tokens[3], tokens[4], tokens[5], tokens[6])) + ','
            elif x == 'nmea_time':
                output = output + str(self._sbe_time(self._reverse_bytes(y), 'nmea')) + ','
            elif x == 'scan_time':
                output = output + str(self._sbe_time(self._reverse_bytes(y), 'scan'))
            elif x == 'flag_ctd_status':
                output = output + str(self._pump_status(y)) + ',' + str(self._bottle_fire(y)) + ',' #need to make final comma dynamic later so less errors down the line
            elif x == 'pressure_temp':
                output = output + str(int(y, 16)) + ','

        return output 
Example 40
Project: odf-ctd-proc   Author: somts   File: eng_to_sci_convert.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]

        the_bytes = b"".join(self.raw_bytes)

        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        return measurements 
Example 41
Project: odf-ctd-proc   Author: somts   File: convert_format.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _parse_scans(self):
        '''The order according to the SBE docs are:
        1) Data from the instrument
          a) Frequency (3 bytes each)
          b) Voltage (12 bits each)
        2) Surface Par (3 bytes)
        3) NMEA lat/lon (7 bytes)
        4) NMEA depth (3 bytes)
        5) NMEA time (4 bytes) (low byte first)
        6) Additional Data from the instrument
          a) Pressure temp (12 bits)
          b) pump status (4 bits)
          c) modulo byte (1 byte)
        7) System time (4 bytes) (low byte first)
        If any of the above are omitted, the length of the hex will be smaller.
        '''

        # parse the frequencies
        #scan_list = list(scan)
        num_frequencies = 5 - self.config["FrequencyChannelsSuppressed"]
        num_voltages = 8 - self.config["VoltageWordsSuppressed"]

        the_bytes = b"".join(self.raw_bytes)

        unpack_str = "6s" * num_frequencies + "3s" * num_voltages + "{}s".format(self.scan_length - num_voltages * 3 - num_frequencies * 6)
        measurements = np.array([[int(x, 16) for x in line] for line in struct.iter_unpack(unpack_str, the_bytes)])

        #this uses numpy magic to just "apply" the needed operations to the correct part of the array all in one go
        measurements[:,:num_frequencies] = measurements[:,:num_frequencies] / 256
        measurements[:,num_frequencies:num_frequencies+num_voltages] = (5 * (1 - (measurements[:,num_frequencies:num_frequencies+num_voltages] / 4095)))
        return measurements 
Example 42
Project: scopething   Author: jonathanhogg   File: vm.py    BSD 2-Clause "Simplified" License 5 votes vote down vote up
def read_analog_samples(self, n, sample_width):
        if self._transactions:
            raise TypeError("Command transaction in progress")
        if sample_width == 2:
            data = await self._reader.readexactly(2*n)
            return array.array('f', ((value+32768)/65536 for (value,) in struct.iter_unpack('>h', data)))
        elif sample_width == 1:
            data = await self._reader.readexactly(n)
            return array.array('f', (value/256 for value in data))
        else:
            raise ValueError(f"Bad sample width: {sample_width}") 
Example 43
Project: game-tools   Author: joshuaskelly   File: bsp29.py    MIT License 5 votes vote down vote up
def read(cls, file):
        data = file.read(cls.size)
        lumps_start = struct.calcsize('<i')

        version_data = data[:lumps_start]
        version_struct = struct.unpack('<i', version_data)
        version = version_struct[0]

        lumps_data = data[lumps_start:]
        lumps = [Lump(*l) for l in struct.iter_unpack(Lump.format, lumps_data)]

        return Header(version, lumps) 
Example 44
Project: game-tools   Author: joshuaskelly   File: bsp29.py    MIT License 5 votes vote down vote up
def _read_file(cls, file, mode):
        def _read_iter_lump(lump, class_):
            """Read iteratively from file unpacking according to format
            class attribute on the class_ object"""
            offset, size = lump.offset, lump.length
            file.seek(offset)

            return [class_(*s) for s in struct.iter_unpack(class_.format, file.read(size))]

        def _read_lump(lump, reader):
            """Read from file using the provided reader object"""
            file.seek(lump.offset)
            return reader.read(file, lump.length)

        bsp = cls()
        bsp.mode = mode
        bsp.fp = file

        factory = bsp.factory

        # Header
        header = factory.Header.read(file)
        bsp.version = header.version

        bsp.entities = _read_lump(header.entities, _Entities)
        bsp.planes = _read_iter_lump(header.planes, factory.Plane)
        bsp.miptextures = _read_lump(header.miptextures, _Miptextures)
        bsp.vertexes = _read_iter_lump(header.vertexes, factory.Vertex)
        bsp.visibilities = _read_lump(header.visibilities, _Visibilities)
        bsp.nodes = _read_iter_lump(header.nodes, factory.Node)
        bsp.texture_infos = _read_iter_lump(header.texture_infos, factory.TextureInfo)
        bsp.faces = _read_iter_lump(header.faces, factory.Face)
        bsp.lighting = _read_lump(header.lighting, _Lighting)
        bsp.clip_nodes = _read_iter_lump(header.clip_nodes, factory.ClipNode)
        bsp.leafs = _read_iter_lump(header.leafs, factory.Leaf)
        bsp.mark_surfaces = _read_lump(header.mark_surfaces, _MarkSurfaces)
        bsp.edges = _read_iter_lump(header.edges, factory.Edge)
        bsp.surf_edges = _read_lump(header.surf_edges, _SurfEdges)
        bsp.models = _read_iter_lump(header.models, factory.Model)

        return bsp 
Example 45
Project: game-tools   Author: joshuaskelly   File: md2.py    MIT License 5 votes vote down vote up
def read(cls, file):
        return [cls.Class(*st) for st in struct.iter_unpack(cls.Class.format, file.read())] 
Example 46
Project: game-tools   Author: joshuaskelly   File: md2.py    MIT License 5 votes vote down vote up
def read(cls, file):
        return [cls.Class(*c) for c in struct.iter_unpack(cls.Class.format, file.read())] 
Example 47
Project: game-tools   Author: joshuaskelly   File: bsp.py    MIT License 5 votes vote down vote up
def read(file, size=-1):
        return [lf[0] for lf in struct.iter_unpack('<H', file.read(size))] 
Example 48
Project: game-tools   Author: joshuaskelly   File: bsp.py    MIT License 5 votes vote down vote up
def read(file, size=-1):
        return [lb[0] for lb in struct.iter_unpack('<H', file.read(size))] 
Example 49
Project: game-tools   Author: joshuaskelly   File: bsp.py    MIT License 5 votes vote down vote up
def read(file, size=-1):
        return [se[0] for se in struct.iter_unpack('<H', file.read(size))] 
Example 50
Project: Project-New-Reign---Nemesis-Main   Author: ShikyoKira   File: test_struct.py    GNU General Public License v3.0 5 votes vote down vote up
def test_iterate(self):
        s = struct.Struct('>IB')
        b = bytes(range(1, 16))
        it = s.iter_unpack(b)
        self.assertEqual(next(it), (0x01020304, 5))
        self.assertEqual(next(it), (0x06070809, 10))
        self.assertEqual(next(it), (0x0b0c0d0e, 15))
        self.assertRaises(StopIteration, next, it)
        self.assertRaises(StopIteration, next, it)