Python msgpack.Unpacker() Examples

def msgpack_appendable_pack(o, path):
    open(path, 'a+').close()  # touch
    with open(path, mode='r+b') as f:
        packer = msgpack.Packer()
        unpacker = msgpack.Unpacker(f)

        if type(o) == list:
            try:
                previous_len = unpacker.read_array_header()
            except msgpack.OutOfData:
                previous_len = 0

            # calculate and replace header
            header = packer.pack_array_header(previous_len + len(o))
            f.seek(0)
            f.write(header)
            f.write(bytes(1) * (MAX_MSGPACK_ARRAY_HEADER_LEN - len(header)))

            # append new elements
            f.seek(0, 2)
            for element in o:
                f.write(packer.pack(element))
        else:
            f.write(packer.pack(o)) 

def unpack_rawr_zip_payload(table_sources, payload):
    """unpack a zipfile and turn it into a callable "tables" object."""
    # the io we get from S3 is streaming, so we can't seek on it, but zipfile
    # seems to require that. so we buffer it all in memory. RAWR tiles are
    # generally up to around 100MB in size, which should be safe to store in
    # RAM.
    from tilequeue.query.common import Table
    from io import BytesIO

    zfh = zipfile.ZipFile(BytesIO(payload), 'r')

    def get_table(table_name):
        # need to extract the whole compressed file from zip reader, as it
        # doesn't support .tell() on the filelike, which gzip requires.
        data = zfh.open(table_name, 'r').read()
        unpacker = Unpacker(file_like=BytesIO(data))
        source = table_sources[table_name]
        return Table(source, unpacker)

    return get_table 

def handle_stream(self, stream, address):
        unpacker = msgpack.Unpacker(raw=False)
        while True:
            try:
                wire_bytes = yield stream.read_bytes(1024, partial=True)
                if not wire_bytes:
                    break
                try:
                    unpacker.feed(wire_bytes)
                except msgpack.exceptions.BufferFull:
                    # Start over loosing some data?!
                    unpacker = msgpack.Unpacker(raw=False)
                    unpacker.feed(wire_bytes)
                for record_dict in unpacker:
                    record = logging.makeLogRecord(record_dict)
                    logger = logging.getLogger(record.name)
                    logger.handle(record)
            except (EOFError, KeyboardInterrupt, SystemExit, StreamClosedError):
                break
            except Exception as exc:  # pylint: disable=broad-except
                log.exception(exc) 

def _messages_generator(self):
            unpacker = msgpack.Unpacker()
            while True:
                # f.read(n) on sys.stdin blocks until n bytes are read, causing
                # serializer to hang.
                # os.read(fileno, n) will block if there is nothing to read, but will
                # return as soon as it is able to read at most n bytes.
                with self._reader_lock:
                    try:
                        line = os.read(self.input_stream.fileno(), self.CHUNK_SIZE)
                    except io.UnsupportedOperation:
                        line = self.input_stream.read(self.CHUNK_SIZE)
                if not line:
                    # Handle EOF, which usually means Storm went away
                    raise StormWentAwayError()
                # As python-msgpack docs suggest, we feed data to the unpacker
                # internal buffer in order to let the unpacker deal with message
                # boundaries recognition and uncomplete messages. In case input ends
                # with a partial message, unpacker raises a StopIteration and will be
                # able to continue after being feeded with the rest of the message.
                unpacker.feed(line)
                for i in unpacker:
                    yield i 

def bulk_import_error_records(self, name, params=None):
        """List the records that have errors under the specified bulk import name.

        Args:
            name (str): Bulk import name.
            params (dict, optional): Extra parameters.
        Yields:
            Row of the data
        """
        params = {} if params is None else params
        with self.get(
            create_url("/v3/bulk_import/error_records/{name}", name=name), params
        ) as res:
            code = res.status
            if code != 200:
                body = res.read()
                self.raise_error("Failed to get bulk import error records", res, body)

            body = io.BytesIO(res.read())
            decompressor = gzip.GzipFile(fileobj=body)

            unpacker = msgpack.Unpacker(decompressor, raw=False)
            for row in unpacker:
                yield row 

def job_result_format_each(self, job_id, format):
        """Yield a row of the job result with specified format.

        Args:
            job_id (int): job ID
            format (str): Output format of the job result information. "json" or "msgpack"

        Yields:
             The query result of the specified job in.
        """
        with self.get(
            create_url("/v3/job/result/{job_id}", job_id=job_id), {"format": format}
        ) as res:
            code = res.status
            if code != 200:
                self.raise_error("Get job result failed", res, "")
            if format == "msgpack":
                unpacker = msgpack.Unpacker(res, raw=False)
                for row in unpacker:
                    yield row
            elif format == "json":
                for row in codecs.getreader("utf-8")(res):
                    yield json.loads(row)
            else:
                yield res.read() 

def make_post_request(self, msg, expected_status=200):
        request = self.encode(msg)
        response = self.fetch('/v1/source', method='POST', body=request,
                              headers={'Content-type':
                                       'application/vnd.msgpack'})
        self.assertEqual(response.code, expected_status)

        responses = []
        if expected_status < 400:
            unpacker = msgpack.Unpacker(**unpack_kwargs)
            unpacker.feed(response.body)

            for msg in unpacker:
                responses.append(msg)
        else:
            responses = [{'error': str(response.error)}]

        return responses 

def test_msgpack_large_data(self):

        big_string = s_const.mebibyte * 129 * 'V'
        struct = ('test', {'key': big_string})

        buf = s_msgpack.en(struct)

        unpacked_struct = s_msgpack.un(buf)
        self.eq(struct, unpacked_struct)

        # Ensure our use of msgpack.Unpacker can also handle this data
        with self.getTestDir() as dirn:
            with s_common.genfile(dirn, 'test.mpk') as fd:
                fd.write(buf)
            with s_common.genfile(dirn, 'test.mpk') as fd:
                genr = s_msgpack.iterfd(fd)
                objs = list(genr)
                self.len(1, objs)
                self.eq(objs[0], struct)

        # Ensure that our streaming Unpk object can also handle this data
        unpk = s_msgpack.Unpk()
        objs = unpk.feed(buf)
        self.len(1, objs)
        self.eq(objs[0], (135266320, struct)) 

def iterfd(fd):
    '''
    Generator which unpacks a file object of msgpacked content.

    Args:
        fd: File object to consume data from.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Yields:
        Objects from a msgpack stream.
    '''
    unpk = msgpack.Unpacker(fd, **unpacker_kwargs)
    for mesg in unpk:
        yield mesg 

def iterfile(path, since=-1):
    '''
    Generator which yields msgpack objects from a file path.

    Args:
        path: File path to open and consume data from.

    Notes:
        String objects are decoded using utf8 encoding.  In order to handle
        potentially malformed input, ``unicode_errors='surrogatepass'`` is set
        to allow decoding bad input strings.

    Yields:
        Objects from a msgpack stream.
    '''
    with io.open(path, 'rb') as fd:

        unpk = msgpack.Unpacker(fd, **unpacker_kwargs)

        for i, mesg in enumerate(unpk):
            if i <= since:
                continue

            yield mesg 

def msgunpackb(bytes):
    """bytes -> list"""
    unpacker = msgpack.Unpacker(io.BytesIO(bytes), raw=False)
    return list(unpacker) 

def __init__(self):
        self.size = 0
        self.unpk = msgpack.Unpacker(**unpacker_kwargs) 

def _read_msgpack_file(self, file_like, **kwargs):
        # current impl doesn't tolerate any unpack error
        unpacker = msgpack.Unpacker(file_like, raw=False)
        for record in unpacker:
            validate_record(record)
            yield record 

def __init__(self, event_loop):
        """Wrap `event_loop` on a msgpack-aware interface."""
        self.loop = event_loop
        self._packer = Packer(unicode_errors=unicode_errors_default)
        self._unpacker = Unpacker(unicode_errors=unicode_errors_default)
        self._message_cb = None 

def tail(self, db, table, count, to=None, _from=None, block=None):
        """Get the contents of the table in reverse order based on the registered time
        (last data first).

        Args:
            db (str): Target database name.
            table (str): Target table name.
            count (int): Number for record to show up from the end.
            to: Deprecated parameter.
            _from: Deprecated parameter.
            block: Deprecated parameter.

        Returns:
            [dict]: Contents of the table.
        """
        params = {"count": count, "format": "msgpack"}
        with self.get(
            create_url("/v3/table/tail/{db}/{table}", db=db, table=table), params
        ) as res:
            code = res.status
            if code != 200:
                self.raise_error("Tail table failed", res, "")

            unpacker = msgpack.Unpacker(res, raw=False)
            result = []
            for row in unpacker:
                result.append(row)

            return result 

def _open_connection(self):
        _logger.debug("connect to %s:%s...", *self.getpeername())
        if self._host:
            reader, writer = await asyncio.open_connection(self._host, self._port, loop=self._loop)
        else:
            reader, writer = await asyncio.open_unix_connection(self._path, loop=self._loop)
        self._conn = Connection(reader, writer,
                                msgpack.Unpacker(raw=False,
                                                 **self._unpack_params))
        _logger.debug("Connection to %s:%s established", *self.getpeername()) 

def test_streaming():

    # Define a custom class, encoder and decoder
    class CustomClass2:
        def __eq__(self, other):
            return isinstance(other, CustomClass2)

    def enc_CustomClass(obj):
        if isinstance(obj, CustomClass2):
            return msgpack.ExtType(12, b'')
        raise TypeError("Unknown type: %r" % (obj,))

    def dec_CustomClass(code, data):
        if code == 12:
            return CustomClass2()

        return msgpack.ExtType(code, data)

    # Define a structure
    G = EcGroup()
    custom_obj = CustomClass2()
    test_data = [G, G.generator(), G.order(), custom_obj]
    packed1 = encode(test_data, enc_CustomClass)
    packed2 = encode(test_data, enc_CustomClass)

    data = packed1 + packed2

    decoder = make_decoder(dec_CustomClass)
    Up = msgpack.Unpacker(ext_hook=decoder)
    Up.feed(data)
    for o in Up:
        assert o == test_data 

def __init__(self, reader, writer):
        """ Pass ina  stream reader to unmarshall msgpack objects from. """
        self.reader = reader
        self.writer = writer
        self.decoder = make_decoder()
        self.unpacker = Unpacker(ext_hook=self.decoder, encoding="utf8")
        self.obj_buf = [] 

def test_drop_nan(self):
        records = [{"x": "a", "y": np.nan}, {"x": np.nan, "y": 1.0}]
        data = self.uploader._pack(pd.DataFrame(records))
        unpacker = msgpack.Unpacker(io.BytesIO(data), encoding="utf-8")
        assert unpacker.unpack() == {"x": "a"}
        assert unpacker.unpack() == {"y": 1.0} 

def test_pack_int_array(self):
        records = [{"time": 0, "x": 0, "y": 0}, {"time": 1, "x": 1, "y": 1}]
        data = self.uploader._pack(pd.DataFrame(records))
        for unpacked in msgpack.Unpacker(io.BytesIO(data), encoding="utf-8"):
            assert unpacked == records[0]
            records = records[1:]
        assert records == [] 

def run(self):
        with open(self.params_file, "rb") as pfile:
            params = msgpack.loads(pfile.read(), raw=False, strict_map_key=False)
        self.columns = params["columns"]
        prep_hit = partial(prepare_tab_hit,
                           columns=self.columns, qmult=self.qmult, tmult=self.tmult,
                           matrix_name=self.matrix_name)
        if self.logging_queue is not None:
            create_queue_logger(self)
            sql_logger = logging.getLogger("sqlalchemy.engine")
            sql_logger.setLevel(self.sql_level)
            sql_logger.addHandler(self.logger.handlers[0])
        else:
            sql_logger = None
        self.engine = db_connect(self.conf, logger=sql_logger)
        self.logger.debug("Started %s", self.identifier)
        session = Session(bind=self.engine)
        self.session = session
        hits, hsps = [], []
        with open(self.index_file, "rb") as index_handle:
            for key, rows in msgpack.Unpacker(index_handle, raw=False, strict_map_key=False):
                curr_hit, curr_hsps = prep_hit(key, rows)
                hits.append(curr_hit)
                hsps += curr_hsps
                hits, hsps = load_into_db(self, hits, hsps, force=False, raw=True)
        _, _ = load_into_db(self, hits, hsps, force=True, raw=True)
        self.logger.debug("Finished %s", self.identifier)
        os.remove(self.index_file)  # Clean it up
        return 

def test_pack(self):
        records = [{"x": "a", "y": 1}, {"x": "b", "y": 2}]
        data = self.uploader._pack(pd.DataFrame(records))
        for unpacked in msgpack.Unpacker(io.BytesIO(data), encoding="utf-8"):
            assert unpacked == records[0]
            records = records[1:]
        assert records == [] 

def __iter__(self):
        for record in msgpack.Unpacker(self, encoding="utf-8"):
            yield record 

def wait_for_at_least_one_message(self, channel):
        """
        Reads until we receive at least one message we can unpack. Return all found messages.
        """

        unpacker = msgpack.Unpacker(encoding='utf-8')

        while True:
            try:
                start = time.time()
                chunk = self.ssh_channel[channel].recv(1024)
                end = time.time()
                self.read_speeds.append( len(chunk) / (end-start) )
                if len(self.read_speeds) > 20:
                    self.read_speeds = self.read_speeds[10:]

                if chunk == b'':
                    # happens only when connection broke. If nothing is to be received, it hangs instead.
                    self.connection_error(channel, 'Connection broken w')
                    return False
            except Exception as error:
                self.connection_error(channel, error)
                raise

            unpacker.feed(chunk)

            messages = [m for m in unpacker]
            if messages:
                return messages 

def redis_get_master_list_from_syncstring(self, syncstring):
        try:
            if syncstring:
                local_list = msgpack.Unpacker(six.BytesIO(syncstring)).unpack()
                if local_list:
                    self.master_list = local_list
                    LOG.info("get new master from syncstring master=%s",
                             self.master_list)
                    return True

            return False

        except Exception:
            LOG.exception("exception happened "
                          "when get new master from syncstring") 

def stream_unpacker(fp, object_hook=None):
    return msgpack.Unpacker(fp, object_hook=object_hook, encoding='utf8') 

def get_lines(self, stream):
        return msgpack.Unpacker(stream, encoding='utf-8', object_pairs_hook=decode_to_sorted) 

def msgpack_appendable_unpack(path):
    # if not list?
    # return msgpack.unpackb(f.read())
    with open(path, 'rb') as f:
        packer = msgpack.Packer()
        unpacker = msgpack.Unpacker(f, encoding='utf-8')
        length = unpacker.read_array_header()

        header_lenght = len(packer.pack_array_header(length))
        unpacker.read_bytes(MAX_MSGPACK_ARRAY_HEADER_LEN - header_lenght)
        f.seek(MAX_MSGPACK_ARRAY_HEADER_LEN)

        return [unpacker.unpack() for _ in range(length)] 

def load_levels(col, path):
    with open(path, 'rb') as fp:
        unpacker = msgpack.Unpacker(fp, raw=False)
        for i, level in enumerate(unpacker, start=1):
            col.objects(id=level['id']).modify(
                upsert=True,
                set__name=level['label'],
                set__parents=[level_ref(p) for p in level['parents']],
                set__admin_level=level.get('admin_level')
            )
    return i 

def load_zones(col, path):
    with open(path, 'rb') as fp:
        unpacker = msgpack.Unpacker(fp, raw=False)
        next(unpacker)  # Skip headers.
        for i, geozone in enumerate(unpacker):
            params = {
                'slug': slugify.slugify(geozone['name'], separator='-'),
                'level': geozone['level'],
                'code': geozone['code'],
                'name': geozone['name'],
                'keys': geozone.get('keys'),
                'parents': geozone.get('parents', []),
                'ancestors': geozone.get('ancestors', []),
                'successors': geozone.get('successors', []),
                'validity': geozone.get('validity'),
                'population': geozone.get('population'),
                'dbpedia': geozone.get('dbpedia'),
                'flag': geozone.get('flag'),
                'blazon': geozone.get('blazon'),
                'wikidata': geozone.get('wikidata'),
                'wikipedia': geozone.get('wikipedia'),
                'area': geozone.get('area'),
            }
            if geozone.get('geom') and (
                geozone['geom']['type'] != 'GeometryCollection' or
                    geozone['geom']['geometries']):
                params['geom'] = geozone['geom']
            try:
                col.objects(id=geozone['_id']).modify(upsert=True, **{
                    'set__{0}'.format(k): v for k, v in params.items()
                })
            except errors.ValidationError as e:
                log.warning('Validation error (%s) for %s with %s',
                            e, geozone['_id'], params)
                continue
    return i