Python types.BufferType() Examples

The following are code examples for showing how to use types.BufferType(). They are from open source Python projects. You can vote up the examples you like or vote down the ones you don't like.

Example 1
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def link(self, key, obj):
        if isinstance(obj, (proxied_list, proxied_dict, BufferProxyObject)):
            # There's no need to link lifecycles, the object
            # is identified uniquely by its buffer mapping and that is a stable id
            return True
        if is_equality_key(key):
            # These are equality keys, they are hard references already
            return True
        if self.stable_set is not None:
            if key in self.stable_set:
                return True
            elif is_wrapped_key(key) and get_wrapped_key(key) in self.stable_set:
                return True

        try:
            self.objmap[key] = obj
            return True
        except TypeError:
            # Not weakly referenceable, try to hold a strong reference then to stop
            # its id from being reused while we hold its idmap entry
            self.strong_refs[key] = obj
            return False 
Example 2
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        cur_offs = baseoffs = header_offs = offs
        cur_offs += _NDARRAY_HEADER_SIZE

        shape = obj.shape
        if len(shape) != 1:
            cur_offs = mapped_tuple.pack_into(shape, buf, cur_offs)
        dtype_offs = cur_offs - baseoffs

        dtype_params = cls._make_dtype_params(obj.dtype)
        if isinstance(dtype_params, basestring):
            dtype_params = _NDARRAY_STANDARD_DTYPES_TO_CODE.get(dtype_params, dtype_params)
        cur_offs = mapped_object.pack_into(dtype_params, buf, cur_offs)
        data_offs = cur_offs - baseoffs

        _NDARRAY_HEADER_PACKER.pack_into(buf, header_offs, dtype_offs, data_offs)
        return proxied_buffer.pack_into(buffer(obj), buf, cur_offs) 
Example 3
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def compatible(self, other):
        """
        Checks whether this schema is compatible with ``other``.

        Compatible schemas are those that have binary-compatible in-buffer data representations.
        """
        if not isinstance(other, Schema):
            return False

        other_schema = other
        if self.slot_keys != other_schema.slot_keys or self.alignment != other_schema.alignment:
            return False

        for k in self.slot_keys:
            if self.slot_types[k] is not other_schema.slot_types[k]:
                return False

        return True 
Example 4
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def from_typed_slots(cls, struct_class_or_slot_types, *p, **kw):
        """
        Constructs a :class:`Schema` out of a description of attribute (slot) types.

        :param struct_class_or_slot_types: Either a class with a ``__slot_types__`` attribute,
            or the mapping of attribute names to attribute types itself.

        :keyword int alignment: *(default 8)* Enforce alignment on object sizes. Having objects
            aligned to a native word size helps with performance.

        :keyword int pack_buffer_size: *(default 64k)* Initial :meth:`pack` buffer size. Will auto-expand when
            necessary, but having it sized correctly from the start can help avoid the performance impact of
            such resizing.

        :keyword int max_pack_buffer_size: *(optional)* Maximum :meth:`pack` buffer size. Will not auto-expand
            beyond this.

        :rtype: Schema
        """
        if hasattr(struct_class_or_slot_types, '__slot_types__'):
            return cls(struct_class_or_slot_types.__slot_types__, *p, **kw)
        elif isinstance(struct_class_or_slot_types, dict):
            return cls(struct_class_or_slot_types, *p, **kw)
        else:
            raise ValueError("Cant build a schema out of %r" % (type(struct_class_or_slot_types),)) 
Example 5
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def map_file(cls, fileobj, offset = 0, size = None, read_only = True):
        """
        Returns a buffer mapping the file object's requested
        range, and the underlying mmap object as a tuple.
        """
        if isinstance(fileobj, zipfile.ZipExtFile):
            return cls.map_zipfile(fileobj, offset, size, read_only = read_only)

        if size is None:
            fileobj.seek(0, os.SEEK_END)
            size = fileobj.tell() - offset
        fileobj.seek(offset)
        map_start = offset - offset % mmap.ALLOCATIONGRANULARITY
        if read_only:
            access = mmap.ACCESS_READ
        else:
            access = mmap.ACCESS_WRITE
        buf = mmap.mmap(fileobj.fileno(), size + offset - map_start,
            access = access, offset = map_start)
        return buffer(buf, offset - map_start, size), buf 
Example 6
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _numeric_id_get_gen(self, elem, startpos, hkey, nitems):
    #lint:disable
    buf = self._likebuf
    PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_SIMPLE)
    try:
        PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
        try:
            if (indexbuf.strides == cython.NULL
                   or indexbuf.ndim < 2
                   or indexbuf.len < nitems * indexbuf.strides[0]):
                raise ValueError("Invalid buffer state")
            stride0 = indexbuf.strides[0]
            stride1 = indexbuf.strides[1]
            pindex = cython.cast(cython.p_char, indexbuf.buf) + startpos * stride0
            pindexend = cython.cast(cython.p_char, indexbuf.buf) + indexbuf.len - stride0 + 1
            if pindex < pindexend and cython.cast('numeric_A *', pindex)[0] == hkey:
                return cython.cast('numeric_A *', pindex + stride1)[0]
        finally:
            PyBuffer_Release(cython.address(indexbuf))
    finally:
        PyBuffer_Release(cython.address(pybuf))
    #lint:enable 
Example 7
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _obj_id_get_gen(self, elem, startpos, hkey, key, default):
    #lint:disable
    nitems = self.index_elements
    PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
    try:
        if (indexbuf.strides == cython.NULL
               or indexbuf.ndim < 2
               or indexbuf.len < nitems * indexbuf.strides[0]):
            raise ValueError("Invalid buffer state")
        stride0 = indexbuf.strides[0]
        stride1 = indexbuf.strides[1]
        pindex = cython.cast(cython.p_char, indexbuf.buf) + startpos * stride0
        pindexend = cython.cast(cython.p_char, indexbuf.buf) + indexbuf.len - stride0 + 1
        while pindex < pindexend and cython.cast('numeric_A *', pindex)[0] == hkey:
            if self._compare_keys(self._buf,
                  cython.cast('numeric_A *', pindex + stride1)[0], key):
                return cython.cast('numeric_A *', pindex + 2*stride1)[0]
            pindex += stride0
        return default
    finally:
        PyBuffer_Release(cython.address(indexbuf))
    #lint:enable 
Example 8
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def __init__(self, buf, offset = 0):
        # Accelerate class attributes
        self._encode = self.encode
        self._dtype = self.dtype
        self._xxh = self.xxh

        # Initialize buffer
        if offset:
            self._buf = self._likebuf = buffer(buf, offset)
        else:
            self._buf = buf
            self._likebuf = _likebuffer(buf)

        # Parse header and map index
        self.index_elements, self.index_offset = self._Header.unpack_from(self._buf, 0)

        self.index = numpy.ndarray(buffer = self._buf,
            offset = self.index_offset,
            dtype = self.dtype,
            shape = (self.index_elements, 3)) 
Example 9
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _numeric_id_multi_get_gen(self, elem, rv, hkey, startpos, default):
    #lint:disable
    nitems = self.index_elements
    PyObject_GetBuffer(self._likebuf, cython.address(pybuf), PyBUF_SIMPLE)
    try:
        try:
            PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
            if (indexbuf.strides == cython.NULL
                   or indexbuf.ndim < 2
                   or indexbuf.len < nitems * indexbuf.strides[0]):
                raise ValueError("Invalid buffer state")
            stride0 = indexbuf.strides[0]
            stride1 = indexbuf.strides[1]
            pindex = cython.cast(cython.p_char, indexbuf.buf) + startpos * stride0
            pindexend = cython.cast(cython.p_char, indexbuf.buf) + indexbuf.len - stride0 + 1
            while pindex < pindexend and cython.cast('numeric_A *', pindex)[0] == hkey:
                rv.append(cython.cast('numeric_A *', pindex + stride1)[0])
                pindex += stride0

            return rv if rv else default
        finally:
            PyBuffer_Release(cython.address(indexbuf))
    finally:
       PyBuffer_Release(cython.address(pybuf))
    #lint:enable 
Example 10
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _numeric_id_multi_has_gen(self, elem, startpos, hkey):
    #lint:disable
    nitems = self.index_elements
    PyObject_GetBuffer(self._likebuf, cython.address(pybuf), PyBUF_SIMPLE)
    try:
        PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
        try:
            if (indexbuf.strides == cython.NULL
                   or indexbuf.ndim < 2
                   or indexbuf.len < nitems * indexbuf.strides[0]):
                raise ValueError("Invalid buffer state")
            stride0 = indexbuf.strides[0]
            pindex = cython.cast(cython.p_char, indexbuf.buf) + startpos * stride0
            pindexend = cython.cast(cython.p_char, indexbuf.buf) + indexbuf.len - stride0 + 1
            return pindex < pindexend and cython.cast('numeric_A *', pindex)[0] == hkey
        finally:
            PyBuffer_Release(cython.address(indexbuf))
    finally:
        PyBuffer_Release(cython.address(pybuf))
    #lint:enable 
Example 11
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def _str_id_multi_has_gen(self, elem, pbkey, blen, startpos, hkey, pbuf_ptr, pbuf_len):
    #lint:disable
    nitems = self.index_elements
    PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
    try:
        if (indexbuf.strides == cython.NULL
               or indexbuf.ndim < 2
               or indexbuf.len < nitems * indexbuf.strides[0]):
            raise ValueError("Invalid buffer state")
        stride0 = indexbuf.strides[0]
        stride1 = indexbuf.strides[1]
        pindex = cython.cast(cython.p_char, indexbuf.buf) + startpos * stride0
        pindexend = cython.cast(cython.p_char, indexbuf.buf) + indexbuf.len - stride0 + 1
        while pindex < pindexend and cython.cast('numeric_A *', pindex)[0] == hkey:
            elem = cython.cast('numeric_A *', pindex + stride1)[0]
            if _compare_bytes_from_cbuffer(pbkey, blen, pbuf_ptr,
                    cython.cast(cython.size_t, elem), pbuf_len):
                return True
            pindex += stride0
        return False
    finally:
        PyBuffer_Release(cython.address(indexbuf))
    #lint:enable 
Example 12
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 6 votes vote down vote up
def map_buffer(cls, buf, offset = 0):
        """
        Builds a mapping proxy out of the data in ``buf`` at offset ``offset``.

        The way mappings are constructed requires metadata to be at a footer, and not a header.
        This means the buffer should end where the mapping ends, or the mapping won't be
        read correctly. If the mapping is embedded on a larger buffer, a slice must be taken
        prior to calling this method, so the caller needs to know the size of the mapping beforehand.

        :param buffer buf: A read buffer where the data is located. The mapping must end where the buffer ends.

        :param int offset: *(optional)* The offset where the mapping starts.
        """
        values_pos, = cls._Footer.unpack_from(buf, offset + len(buf) - cls._Footer.size)
        value_array = cls.ValueArray.map_buffer(buf, offset + values_pos)
        id_mapper = cls.IdMapper.map_buffer(buf, offset)
        return cls(value_array, id_mapper) 
Example 13
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _likebuffer(buf):
    """
    Takes a buffer object as parameter and returns a writable object with buffer protocol.
    """
    if (type(buf) is buffer or type(buf) is bytearray or type(buf) is bytes or
            isinstance(buf, (ctypes_Array, bytes))):
        return buf
    else:
        return buffer(buf) 
Example 14
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _likerobuffer(buf):
    """
    Takes a buffer object as parameter and returns a read-only object with buffer protocol.
    """
    if type(buf) is buffer or type(buf) is bytes or isinstance(buf, bytes):
        return buf
    else:
        return buffer(buf) 
Example 15
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def shared_id(obj):
    # (id(buf), offs) of buffer-mapped proxies
    if isinstance(obj, proxied_list):
        lobj = obj
        rv = (id(lobj.buf) << 68) | lobj.offs | PROXY_MASK
        if lobj.elem_step != 0:
            # Add slice arguments to the shared_id
            rv |= long(lobj.elem_step) << (68 + 64)
            rv |= long(lobj.elem_start) << (68 + 128)
            rv |= long(lobj.elem_end) << (68 + 192)
        return rv
    elif isinstance(obj, proxied_dict):
        dobj = obj
        return (id(dobj.buf) << 68) | dobj.offs | PROXY_MASK
    elif isinstance(obj, BufferProxyObject):
        oobj = obj
        return (id(oobj.buf) << 68) | oobj.offs | PROXY_MASK
    elif is_equality_key_compatible(obj):
        # For numeric keys, just add the LONG_MASK to differentiate them from regular ids
        if isinstance(obj, int):
            # Up to 64 bits, just add the LONG_MASK
            return obj | LONG_MASK
        elif isinstance(obj, long):
            # Real longs must make room for the flag bits
            return (obj & 0xFFFFFFFFFFFFFFFFL) | ((obj >> 64) << 68) | LONG_MASK

        # For other keys, use the object itself as key if hashable
        try:
            hash(obj)
        except TypeError:
            pass
        else:
            return obj

    # Otherwise plainly the id of the object
    return id(obj) 
Example 16
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def unpack_from(cls, buf, offs, idmap = None):
        buf = _likerobuffer(buf)
        if cython.compiled:
            PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_SIMPLE)  # lint:ok
            pbuf = cython.cast(cython.p_uchar, pybuf.buf)  # lint:ok
            if offs >= pybuf.len:
                PyBuffer_Release(cython.address(pybuf))  # lint
                raise IndexError("Offset out of range")
        else:
            pbuf = buf
        try:
            fs_type = pbuf[offs]
            if fs_type == 'm' or fs_type == 'M':
                # inline bitmap
                if fs_type == 'm':
                    fs_size = 7
                elif fs_type == 'M':
                    fs_size = 15
                else:
                    raise ValueError("Unknown set type %r" % fs_type)
                if cython.compiled and offs+fs_size >= pybuf.len:
                    raise IndexError("Object spans beyond buffer end")
                rv = []
                for i in xrange(fs_size):
                    b = ord(pbuf[offs+1+i])
                    if b:
                        for j in xrange(8):
                            if b & (1<<j):
                                rv.append(i*8+j)
                return frozenset(rv)
            else:
                # unpack a list, build a set from it
                return mapped_list.unpack_from(buf, offs, idmap, klass=frozenset)
        finally:
            if cython.compiled:
                if type(buf) is buffer:
                    PyBuffer_Release(cython.address(pybuf))  # lint:ok 
Example 17
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs buffer data from ``obj`` into ``buf``. Does not wrap it in :term:`RTTI`.

        See `mapped_object.pack_into` for argument details.
        """
        cur_offs = offs
        objlen = len(obj)
        _BUFFER_HEADER_PACKER.pack_into(buf, offs, objlen)
        cur_offs += _BUFFER_HEADER_SIZE

        end_offs = cur_offs + objlen
        buf[cur_offs:end_offs] = obj

        return end_offs 
Example 18
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def unpack_from(cls, buf, offs, idmap = None):
        """
        Unpacks buffer data from ``buf`` and returns a buffer slice of ``buf`` that contains the data.

        See `mapped_object.unpack_from` for argument details.
        """
        cur_offs = offs
        size, = _BUFFER_HEADER_PACKER.unpack_from(buf, offs)
        cur_offs += _BUFFER_HEADER_SIZE

        return buffer(buf, cur_offs, size) 
Example 19
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def buffer(self):
        return self.buf 
Example 20
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def unpack_from(cls, buf, offs, idmap = None):
        buf = _likerobuffer(buf)
        try:
            if cython.compiled:
                pybuf.buf = cython.NULL
                PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_SIMPLE)
                pbuf = cython.cast(cython.p_uchar, pybuf.buf)
                if offs >= pybuf.len:
                    raise IndexError("Offset out of range")
            else:
                pbuf = buf

            if pbuf[offs] == 'm':
                # inline bitmap (64 bits)
                if cython.compiled and offs+7 >= pybuf.len:
                    raise IndexError("Object spans beyond buffer end")
                bitrep_lo = cython.cast(cython.p_ulonglong, pbuf + offs)[0] >> 8
                return proxied_frozenset(None, bitrep_lo, 0)
            elif pbuf[offs] == 'M':
                # inline bitmap (128 bits)
                if cython.compiled and offs+15 >= pybuf.len:
                    raise IndexError("Object spans beyond buffer end")
                bitrep_lo = cython.cast(cython.p_ulonglong, pbuf + offs)[0] >> 8
                bitrep_hi = cython.cast(cython.p_ulonglong, pbuf + offs)[1]
                bitrep_lo |= (bitrep_hi & 0xff) << 56
                bitrep_hi >>= 8
                return proxied_frozenset(None, bitrep_lo, bitrep_hi)
            else:
                return proxied_frozenset(proxied_list.unpack_from(buf, offs, idmap))
        finally:
            if cython.compiled:
                if type(buf) is buffer and pybuf.buf != cython.NULL:
                    PyBuffer_Release(cython.address(pybuf)) 
Example 21
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs an :term:`RTTI`-wrapped value into ``buf`` at offset ``offs``.

        :param obj: The object to be packed. Must be one of the supported types, or an instance
            of a type registered with :meth:`register_schema`.

        :param buf: A writeable buffer onto which the object should be packed. Like a bytearray.

        :param offs: The offset within ``buf`` where to place the object.

        :param idmap: *(optional)* A mapping (dict-like or an instance of :class:`StrongIdMap`) used to
            deduplicate references to recurring objects. If not given, a temporary :class:`StrongIdMap` will
            be constructed for the operation if necessary. See :term:`idmap`.

        :param implicit_offs: *(optional)* The implicit offset of ``buf`` within a larger data structure.
            If either ``buf`` is a slice of a larger buffer or if its contents will be copied onto a larger
            buffer, this should be the starting point of ``buf``, so new entries on the ``idmap`` are
            created with the proper absolute offset. Otherwise, :term:`idmap` mixups are likely to corrupt the
            resulting buffer.

        :return: The offset where writing finished. Further objects can be placed at this offset when
            packing multiple instances.
        """
        if not isinstance(obj, cls):
            obj = cls(obj)
        typecode = obj.typecode
        endp = offs
        if typecode in cls.PACKERS:
            packer, padding = cls.PACKERS[typecode]
            packer.pack_into(buf, offs, typecode, obj.value)
            endp += packer.size + padding
        elif typecode in cls.OBJ_PACKERS:
            cpacker, cpadding = cls.CODE_PACKER
            cpacker.pack_into(buf, offs, typecode)
            endp += cpacker.size + cpadding
            packer = cls.OBJ_PACKERS[typecode][0]
            endp = packer(obj.value, buf, endp, idmap, implicit_offs)
        else:
            raise TypeError("Unsupported type %r: %r" % (typecode, obj.value))
        return endp 
Example 22
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _c_buffer_proxy_set_gen(self, obj, elem):
        if obj is None or (obj.none_bitmap & self.mask):
            return
        elif obj.pybuf.readonly:
            raise TypeError('cannot set attribute in read-only buffer')
        assert (obj.offs + self.offs + cython.sizeof(elem)) <= obj.pybuf.len   #lint:ok
        cython.cast('numeric_A *',
            cython.cast(cython.p_uchar, obj.pybuf.buf) + obj.offs + self.offs)[0] = elem   #lint:ok
        mfence_full()   # release 
Example 23
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def _c_buffer_proxy_atomic_cas(self, obj, exp_val, new_val):
        if obj is None or (obj.none_bitmap & self.mask):
            return False
        elif obj.pybuf.readonly:
            raise TypeError('cannot set attribute in read-only buffer')
        assert (obj.offs + self.offs + cython.sizeof(exp_val)) <= obj.pybuf.len   #lint:ok
        ptr = cython.cast('numeric_A *',
            cython.cast(cython.p_uchar, obj.pybuf.buf) + obj.offs + self.offs)
        if numeric_A is cython.float:
            return _c_atomic_cas_flt(ptr, exp_val, new_val)
        elif numeric_A is cython.double:
            return _c_atomic_cas_dbl(ptr, exp_val, new_val)
        else:
            return _c_atomic_cas(ptr, exp_val, new_val) 
Example 24
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def set_prewrite_hook(self, hook):
        """
        A callable that will be invoked with arguments (obj, buf, baseoffs)
        each time an object is about to be packed into a buffer.
        """
        self.prewrite_hook = hook 
Example 25
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def set_postwrite_hook(self, hook):
        """
        A callable that will be invoked with arguments (obj, buf, baseoffs, offs)
        each time after an object has been packed into a buffer.
        """
        self.postwrite_hook = hook 
Example 26
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def set_pack_buffer_size(self, newsize):
        """
        Sets a new :meth:`pack` buffer size. See :meth:`from_typed_slots`.
        """
        self.pack_buffer_size = newsize
        self._pack_buffer = bytearray(self.pack_buffer_size) 
Example 27
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def unpack(self, buf, idmap = None, factory_class_new = None, proxy_into = None):
        """
        Unpack data from ``buf``.

        See :meth:`unpack_from` for a description of the arguments.
        """
        return self.unpack_from(buffer(buf), 0, idmap, factory_class_new, proxy_into) 
Example 28
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self, buf, offset = 0, idmap = None, idmap_size = 1024):
        if idmap is None:
            idmap = Cache(idmap_size)
        self.offset = offset
        if offset != 0:
            self.buf = buf = buffer(buf, offset)
        else:
            self.buf = buf
        self.wr_buf = buf   # May be overriden by map_file
        self.total_size, self.index_offset, self.index_elements = self._Header.unpack_from(buf, 0)
        self.index = npfrombuffer(buf,
            offset = self.index_offset,
            dtype = numpy.uint64,
            count = self.index_elements)
        self.idmap = idmap

        if self.index_elements > 0 and self.index[0] >= (self._Header.size + self._NewHeader.size):
            # New version, most likely
            self.version, min_reader_version, self.schema_offset, self.schema_size = self._NewHeader.unpack_from(
                buf, self._Header.size)
            if self._CURRENT_VERSION < min_reader_version:
                raise ValueError((
                    "Incompatible buffer, this buffer needs a reader with support for version %d at least, "
                    "this reader supports up to version %d") % (
                        min_reader_version,
                        self._CURRENT_VERSION
                    ))
            if self.schema_offset and self.schema_size:
                if self.schema_offset > len(buf) or (self.schema_size + self.schema_offset) > len(buf):
                    raise ValueError("Corrupted input - bad schema location")
                stored_schema = cPickle.loads(bytes(buffer(buf, self.schema_offset, self.schema_size)))
                if not isinstance(stored_schema, Schema):
                    raise ValueError("Corrupted input - unrecognizable schema")
                if self.schema is None or not self.schema.compatible(stored_schema):
                    self.schema = stored_schema
            elif self.schema is None:
                raise ValueError("Cannot map schema-less buffer without specifying schema")
        elif self.index_elements > 0:
            raise ValueError("Cannot reliably map version-0 buffers") 
Example 29
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def map_buffer(cls, buf, offset = 0):
        """
        Build a mapped array instance mapping the array in ``buf`` at position ``offset``

        :param buf: Readable buffer to map the array from

        :param int offset: *(optional)* Position within the buffer where the array is located.
        """
        return cls(buf, offset) 
Example 30
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def map_file(cls, fileobj, offset = 0, size = None, read_only = True):
        """
        Build a mapped array instance mapping the given ``fileobj`` at position ``offset``.
        A size can optionally be given to map only the necessary portion of the file.

        :param file fileobj: Memory-mappable file where the array is located

        :param int offset: *(optional)* Position within the file where the array is located.

        :param int size: *(optional)* Size of the array data. If given, it will be used to reduce
            the mapped portion of the file to the minimum necessary mapping.

        :param bool read_only: *(optional)* Whether the mapping should be read-only, or if
            write access should also be requested. Defaults to true.
        """
        if isinstance(fileobj, zipfile.ZipExtFile):
            return cls.map_zipfile(fileobj, offset, size, read_only = read_only)

        fileobj.seek(offset)
        total_size = cls._Header.unpack(fileobj.read(cls._Header.size))[0]
        map_start = offset - offset % mmap.ALLOCATIONGRANULARITY
        if read_only:
            access = mmap.ACCESS_READ
        else:
            access = mmap.ACCESS_WRITE
        buf = mmap.mmap(fileobj.fileno(), total_size + offset - map_start,
            access = access, offset = map_start)
        rv = cls(buffer(buf, offset - map_start))
        rv._file = fileobj
        rv._mmap = buf
        if not read_only:
            offset -= map_start
            rv.wr_buf = (ctypes.c_char * (len(buf) - offset)).from_buffer(buf, offset)
        return rv 
Example 31
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self, buf, offset = 0):
        # Accelerate class attributes
        self._dtype = self.dtype

        # Initialize buffer
        if offset:
            self._buf = self._likebuf = buffer(buf, offset)
        else:
            self._buf = buf
            self._likebuf = _likebuffer(buf)

        # Parse header and map index
        self.index_elements, self.index_offset = self._Header.unpack_from(self._buf, 0)

        self.index = numpy.ndarray(buffer = self._buf,
            offset = self.index_offset,
            dtype = self.dtype,
            shape = (self.index_elements, 2))

        if len(self.index) > 0:
            self._index_min = self.index[0,0]
            self._index_max = self.index[-1,0]
        else:
            self._index_min = self._index_max = 0

        dtype = self.dtype
        try:
            self.dtypemax = ~dtype(0)
        except:
            try:
                self.dtypemax = ~dtype.type(0)
            except:
                self.dtypemax = ~0 
Example 32
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def map_buffer(cls, buf, offset = 0):
        """
        Build an :term:`Id Mapper` from the data in ``buf`` at position ``offset``

        :param buf: Readable buffer to map the :term:`Id Mapper` from

        :param int offset: *(optional)* Position within the buffer where the data is located.
        """
        return cls(buf, offset) 
Example 33
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def __init__(self, buf, offset = 0):
        # Accelerate class attributes
        self._dtype = self.dtype

        # Initialize buffer
        self._buf = buf
        self._likebuf = _likebuffer(buf)
        self._basepos = offset

        # Parse header and map index
        self.index_elements, self.index_offset = self._Header.unpack_from(self._buf, offset)

        self.index = numpy.ndarray(
            buffer = self._buf,
            offset = self.index_offset + offset,
            dtype = self.dtype,
            shape = (self.index_elements, 3))

        dtype = self.dtype
        try:
            dtypemax = int(~dtype(0))
        except:
            try:
                dtypemax = int(~dtype.type(0))
            except:
                dtypemax = ~0
        self._dtype_bits = 0
        while dtypemax:
            self._dtype_bits += 1
            dtypemax >>= 1 
Example 34
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 5 votes vote down vote up
def map_file(cls, fileobj, offset = 0, size = None, read_only = True):
        """
        Builds a mapping proxy out of the data in ``fileobj`` at offset ``offset``
        and size ``size``.

        The way mappings are constructed requires metadata to be at a footer, and not a header.
        This means the buffer should end where the mapping ends, or the mapping won't be
        read correctly. If the mapping is embedded on a larger buffer, a slice must be taken
        prior to calling this method, so the caller needs to know the size of the mapping beforehand.

        :param file fileobj: A file object where the data is located. The mapping must end where the file ends,
            or an explicit ``size`` must be given.

        :param int offset: *(optional)* The offset where the mapping starts.

        :param int size: *(optional)* The size of the mapping relative to its starting offset. It must be
            given if the mapping doesn't end at the EOF, to be able to locate the footer.

        :param bool read_only: *(optional)* Whether the mapping should be read-only, or if
            write access should also be requested. Defaults to true.
        """
        if isinstance(fileobj, zipfile.ZipExtFile):
            return cls.map_zipfile(fileobj, offset, size, read_only = read_only)

        # If no size is given, it's the whole file by default
        if size is None:
            fileobj.seek(0, os.SEEK_END)
            size = fileobj.tell() - offset

        # Read the footer
        fileobj.seek(offset + size - cls._Footer.size)
        values_pos, = cls._Footer.unpack(fileobj.read(cls._Footer.size))
        fileobj.seek(offset)

        # Map everything
        id_mapper = cls.IdMapper.map_file(fileobj, offset, size = values_pos, read_only = read_only)
        value_array = cls.ValueArray.map_file(fileobj, offset + values_pos,
            size = size - cls._Footer.size - values_pos, read_only = read_only)
        return cls(value_array, id_mapper) 
Example 35
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def pack_into(cls, obj, buf, offs, idmap = None, implicit_offs = 0):
        """
        Packs a byte string into ``buf``. Does not wrap it in :term:`RTTI`.

        See `mapped_object.pack_into` for argument details.
        """
        if idmap is not None:
            objid = shared_id(obj)
            idmap[objid] = offs + implicit_offs
            if isinstance(idmap, StrongIdMap):
                widmap = idmap
                widmap.link(objid, obj)
        objlen = len(obj)

        if objlen > MIN_COMPRESS_THRESHOLD:
            objcomp = lz4_compress(obj)
            objcomplen = len(objcomp)
            if objcomplen < (objlen - objlen/3):
                # Must get substantial compression to pay the price
                obj = objcomp
                objlen = objcomplen
                compressed = 0x8000
            else:
                compressed = 0
            del objcomp
        else:
            compressed = 0

        if (offs + 16 + len(obj)) > len(buf):
            raise struct.error('buffer too small')
        if cython.compiled:
            try:
                buf = _likebuffer(buf)
                PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_WRITABLE)  # lint:ok
                pbuf = cython.cast(cython.p_char, pybuf.buf) + offs  # lint:ok

                if objlen < 0x7FFF:
                    cython.cast('_varstr_header *', pbuf).shortlen = objlen | compressed
                    offs += cython.sizeof(cython.ushort)
                    pbuf += cython.sizeof(cython.ushort)
                else:
                    cython.cast('_varstr_header *', pbuf).shortlen = 0x7FFF | compressed
                    cython.cast('_varstr_header *', pbuf).biglen = objlen
                    offs += cython.sizeof('_varstr_header')
                    pbuf += cython.sizeof('_varstr_header')
                memcpy(pbuf, cython.cast(cython.p_char, obj), objlen)  # lint:ok
            finally:
                PyBuffer_Release(cython.address(pybuf))  # lint:ok
        else:
            if objlen < 0x7FFF:
                hpacker = struct.Struct('=H')
                hpacker.pack_into(buf, offs, objlen | compressed)
                offs += hpacker.size
            else:
                qpacker = struct.Struct('=HQ')
                qpacker.pack_into(buf, offs, 0x7FFF | compressed, objlen)
                offs += qpacker.size
            buf[offs:offs+objlen] = obj
        offs += objlen
        return offs 
Example 36
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def unpack_from(cls, buf, offs, idmap = None, proxy_into = None):
        """
        Unpacks an :term:`RTTI`-wrapped value from ``buf`` at offset ``offs``.

        :param buf: The buffer where the value is.

        :param offs: The position within ``buf`` where the value starts.

        :param idmap: *(optional)* A dict-like mapping that will be used to store references to already-unpacked
            objects, to preserve object identity (ie: to unpack the same offset into a reference to the same object).
            If object identity matters, be it for memory efficiency or some other reason, one should be provided.
            Otherwise it's not necessary for unpacking. See :term:`idmap`.

        :param proxy_into: *(optional)* An instance of :class:`BufferProxyObject` that will be turned into
            a proxy of the right kind, pointing at the right offset. This is slightly faster than building a new
            proxy every time. Otherwise a new proxy is constructed and returned.

        :return: The unpacked object or proxy, depending on the type.
        """
        cpadding = 7
        cpacker_size = 1
        buf = _likerobuffer(buf)
        typecode = buf[offs]

        unpacker_info = _mapped_object_PACKERS.get(typecode)
        if unpacker_info is not None:
            packer, padding = unpacker_info
            typecode, value = packer.unpack_from(buf, offs)
            return value

        unpacker_info = _mapped_object_OBJ_PACKERS.get(typecode)
        if unpacker_info is not None:
            offs += cpacker_size + cpadding
            if proxy_into is not None:
                typ = unpacker_info[2]
                if typ is _mapped_object or type(typ) is mapped_object_with_schema:
                    return unpacker_info[1](buf, offs, idmap, proxy_into)
                else:
                    return unpacker_info[1](buf, offs, idmap)
            else:
                return unpacker_info[1](buf, offs, idmap)
        else:
            raise ValueError("Inconsistent data") 
Example 37
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def iteritems(self):
        buf = self._buf
        dtype = self.dtype
        index = self.index
        if cython.compiled:
            #lint:disable
            buf = self._likebuf
            PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_SIMPLE)
            try:
                if dtype is npuint64:
                    PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                    try:
                        if ( indexbuf.strides == cython.NULL
                                or indexbuf.ndim < 2
                                or indexbuf.len < self.index_elements * indexbuf.strides[0] ):
                            raise ValueError("Invalid buffer state")
                        stride0 = indexbuf.strides[0]
                        stride1 = indexbuf.strides[1]
                        pindex = cython.cast(cython.p_char, indexbuf.buf)
                        for i in xrange(self.index_elements):
                            yield (
                                cython.cast(cython.p_ulonglong, pindex)[0],
                                cython.cast(cython.p_ulonglong, pindex + stride1)[0]
                            )
                            pindex += stride0
                    finally:
                        PyBuffer_Release(cython.address(indexbuf))
                elif dtype is npuint32:
                    PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                    try:
                        if ( indexbuf.strides == cython.NULL
                                or indexbuf.ndim < 2
                                or indexbuf.len < self.index_elements * indexbuf.strides[0] ):
                            raise ValueError("Invalid buffer state")
                        stride0 = indexbuf.strides[0]
                        stride1 = indexbuf.strides[1]
                        pindex = cython.cast(cython.p_char, indexbuf.buf)
                        for i in xrange(self.index_elements):
                            yield (
                                cython.cast(cython.p_uint, pindex)[0],
                                cython.cast(cython.p_uint, pindex + stride1)[0]
                            )
                            pindex += stride0
                    finally:
                        PyBuffer_Release(cython.address(indexbuf))
                else:
                    for i in xrange(self.index_elements):
                        yield (
                            index[i,0],
                            index[i,1]
                        )
            finally:
                PyBuffer_Release(cython.address(pybuf))
            #lint:enable
        else:
            for i in xrange(self.index_elements):
                yield (index[i,0], index[i,1]) 
Example 38
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def _search_hkey(self, hkey):
        hi = self.index_elements
        lo = 0
        if hi <= lo:
            return hi
        hikey = self._index_max
        lokey = self._index_min
        if hkey < lokey:
            return lo
        elif hkey > hikey:
            return hi
        if cython.compiled:
            dtype = self._dtype
            if dtype is npuint64 or dtype is npuint32 or dtype is npuint16 or dtype is npuint8:
                #lint:disable
                PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                try:
                    if ( indexbuf.strides == cython.NULL
                            or indexbuf.len < hi * indexbuf.strides[0] ):
                        raise ValueError("Invalid buffer state")
                    pindex = cython.cast(cython.p_char, indexbuf.buf)
                    stride0 = indexbuf.strides[0]

                    if dtype is npuint64:
                        # Must be careful about overflow
                        if hikey > cython.cast(cython.ulonglong, 0xFFFFFFFFFFFF):
                            hint = lo + ((hkey - lokey) >> 32) * (hi - lo) // max(((hikey - lokey) >> 32), 1)
                        elif hikey > cython.cast(cython.ulonglong, 0xFFFFFFFF):
                            hint = lo + ((hkey - lokey) >> 16) * (hi - lo) // max(((hikey - lokey) >> 16), 1)
                        else:
                            hint = lo + (hkey - lokey) * (hi - lo) // max((hikey - lokey), 1)
                        return _c_search_hkey_ui64(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint32:
                        hint = lo + (hkey - lokey) * (hi - lo) // max((hikey - lokey), 1)
                        return _c_search_hkey_ui32(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint16:
                        hint = lo + (hkey - lokey) * (hi - lo) // max((hikey - lokey), 1)
                        return _c_search_hkey_ui16(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint8:
                        hint = lo + (hkey - lokey) * (hi - lo) // max((hikey - lokey), 1)
                        return _c_search_hkey_ui8(hkey, pindex, stride0, hi, hint, True)
                    else:
                        raise AssertionError("Internal error")
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
                #lint:enable
            else:
                raise AssertionError("Internal error")
        else:
            dtype = self.dtype
            struct_dt = numpy.dtype([
                ('key', dtype),
                ('value', dtype),
            ])
            return self.index.view(struct_dt).reshape(self.index.shape[0]).searchsorted(
                numpy.array([(hkey,0)],dtype=struct_dt))[0] 
Example 39
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def iterkeys(self, make_sequential = False):
        buf = self._buf
        dtype = self.dtype
        if make_sequential:
            # Big collections don't fit in RAM, so it helps if they're accessed sequentially
            # We'll just have to copy and sort the index, no big deal though
            index = numpy.sort(self.index[:,1])
            stride = 1
            offs = 0
        else:
            index = self.index
            stride = 3
            offs = 1
        if cython.compiled:
            #lint:disable
            buf = self._likebuf
            if dtype is npuint64:
                PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_SIMPLE)
                try:
                    if indexbuf.len < (self.index_elements * stride * cython.sizeof(cython.ulonglong)):
                        raise ValueError("Invalid buffer state")
                    for i in xrange(self.index_elements):
                        yield self._unpack(self._buf,
                            cython.cast(cython.p_ulonglong, indexbuf.buf)[i*stride+offs])
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
            elif dtype is npuint32:
                PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_SIMPLE)
                try:
                    if indexbuf.len < (self.index_elements * stride * cython.sizeof(cython.uint)):
                        raise ValueError("Invalid buffer state")
                    for i in xrange(self.index_elements):
                        yield self._unpack(self._buf,
                            cython.cast(cython.p_uint, indexbuf.buf)[i*stride+offs])
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
            else:
                for i in xrange(self.index_elements):
                    if not make_sequential:
                        pos = index[i, offs]
                    else:
                        pos = index[i]
                    yield self._unpack(self._buf, pos)
            #lint:enable
        else:
            for i in xrange(self.index_elements):
                if not make_sequential:
                    pos = index[i, offs]
                else:
                    pos = index[i]
                yield self._unpack(buf, pos) 
Example 40
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def iteritems(self, make_sequential = False):
        buf = self._buf
        dtype = self.dtype
        if make_sequential:
            # Big collections don't fit in RAM, so it helps if they're accessed sequentially
            # We'll just have to copy and sort the index, no big deal though
            index = self.index[numpy.argsort(self.index[:,1])]
        else:
            index = self.index
        if cython.compiled:
            #lint:disable
            buf = self._likebuf
            if dtype is npuint64:
                PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                try:
                    if ( indexbuf.strides == cython.NULL
                            or indexbuf.ndim < 2
                            or indexbuf.len < self.index_elements * indexbuf.strides[0] ):
                        raise ValueError("Invalid buffer state")
                    stride0 = indexbuf.strides[0]
                    stride1 = indexbuf.strides[1]
                    pindex = cython.cast(cython.p_char, indexbuf.buf)
                    for i in xrange(self.index_elements):
                        yield (
                            self._unpack(buf,
                                cython.cast(cython.p_ulonglong, pindex + stride1)[0]),
                            cython.cast(cython.p_ulonglong, pindex + 2*stride1)[0]
                        )
                        pindex += stride0
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
            elif dtype is npuint32:
                PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                try:
                    if ( indexbuf.strides == cython.NULL
                            or indexbuf.ndim < 2
                            or indexbuf.len < self.index_elements * indexbuf.strides[0] ):
                        raise ValueError("Invalid buffer state")
                    stride0 = indexbuf.strides[0]
                    stride1 = indexbuf.strides[1]
                    pindex = cython.cast(cython.p_char, indexbuf.buf)
                    for i in xrange(self.index_elements):
                        yield (
                            self._unpack(buf,
                                cython.cast(cython.p_uint, pindex + stride1)[0]),
                            cython.cast(cython.p_uint, pindex + 2*stride1)[0]
                        )
                        pindex += stride0
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
            else:
                for i in xrange(self.index_elements):
                    yield (
                        self._unpack(self._buf, index[i,1]),
                        index[i,2]
                    )
            #lint:enable
        else:
            for i in xrange(self.index_elements):
                yield (self._unpack(buf, index[i,1]), index[i,2]) 
Example 41
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def _search_hkey(self, hkey):
        hi = self.index_elements
        lo = 0
        if hi <= lo:
            return hi
        if cython.compiled:
            dtype = self._dtype
            if dtype is npuint64 or dtype is npuint32 or dtype is npuint16 or dtype is npuint8:
                #lint:disable
                PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                try:
                    if ( indexbuf.strides == cython.NULL
                            or indexbuf.len < hi * indexbuf.strides[0] ):
                        raise ValueError("Invalid buffer state")
                    pindex = cython.cast(cython.p_char, indexbuf.buf)
                    stride0 = indexbuf.strides[0]

                    if dtype is npuint64:
                        # A quick guess assuming uniform distribution of keys over the 64-bit value range
                        hint = (((hkey >> 32) * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui64(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint32:
                        # A quick guess assuming uniform distribution of keys over the 64-bit value range
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui32(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint16:
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui16(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint8:
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui8(hkey, pindex, stride0, hi, hint, True)
                    else:
                        raise AssertionError("Internal error")
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
                #lint:enable
        else:
            dtype = self.dtype
            struct_dt = numpy.dtype([
                ('key_hash', dtype),
                ('key_offset', dtype),
                ('value', dtype),
            ])
            return self.index.view(struct_dt).reshape(self.index.shape[0]).searchsorted(
                numpy.array([(hkey,0,0)],dtype=struct_dt))[0] 
Example 42
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def iterkeys(self, make_sequential = True):
        buf = self._buf
        dtype = self.dtype
        if make_sequential:
            # Big collections don't fit in RAM, so it helps if they're accessed sequentially
            # We'll just have to copy and sort the index, no big deal though
            index = numpy.sort(self.index[:,1])
            stride = 1
            offs = 0
        else:
            index = self.index
            stride = 3
            offs = 1
        if cython.compiled:
            #lint:disable
            buf = self._likebuf
            PyObject_GetBuffer(buf, cython.address(pybuf), PyBUF_SIMPLE)
            try:
                if dtype is npuint64:
                    PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_SIMPLE)
                    try:
                        if indexbuf.len < (self.index_elements * stride * cython.sizeof(cython.ulonglong)):
                            raise ValueError("Invalid buffer state")
                        for i in xrange(self.index_elements):
                            yield _unpack_bytes_from_cbuffer(
                                cython.cast(cython.p_char, pybuf.buf),
                                cython.cast(cython.p_ulonglong, indexbuf.buf)[i*stride+offs],
                                pybuf.len, None)
                    finally:
                        PyBuffer_Release(cython.address(indexbuf))
                elif dtype is npuint32:
                    PyObject_GetBuffer(index, cython.address(indexbuf), PyBUF_SIMPLE)
                    try:
                        if indexbuf.len < (self.index_elements * stride * cython.sizeof(cython.uint)):
                            raise ValueError("Invalid buffer state")
                        for i in xrange(self.index_elements):
                            yield _unpack_bytes_from_cbuffer(
                                cython.cast(cython.p_char, pybuf.buf),
                                cython.cast(cython.p_uint, indexbuf.buf)[i*stride+offs],
                                pybuf.len, None)
                    finally:
                        PyBuffer_Release(cython.address(indexbuf))
                else:
                    for i in xrange(self.index_elements):
                        yield _unpack_bytes_from_cbuffer(
                            cython.cast(cython.p_char, pybuf.buf),
                            index[i],
                            pybuf.len, None)
            finally:
                PyBuffer_Release(cython.address(pybuf))
            #lint:enable
        else:
            for i in xrange(self.index_elements):
                yield _unpack_bytes_from_pybuffer(buf, index[i], None) 
Example 43
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def _search_hkey(self, hkey):
        hi = self.index_elements
        lo = 0
        if hi <= lo:
            return hi
        if cython.compiled:
            dtype = self._dtype
            if dtype is npuint64 or dtype is npuint32 or dtype is npuint16 or dtype is npuint8:
                #lint:disable
                PyObject_GetBuffer(self.index, cython.address(indexbuf), PyBUF_STRIDED_RO)
                try:
                    if ( indexbuf.strides == cython.NULL
                            or indexbuf.len < hi * indexbuf.strides[0] ):
                        raise ValueError("Invalid buffer state")
                    pindex = cython.cast(cython.p_char, indexbuf.buf)
                    stride0 = indexbuf.strides[0]

                    if dtype is npuint64:
                        # A quick guess assuming uniform distribution of keys over the 64-bit value range
                        hint = (((hkey >> 32) * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui64(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint32:
                        # A quick guess assuming uniform distribution of keys over the 64-bit value range
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui32(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint16:
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui16(hkey, pindex, stride0, hi, hint, True)
                    elif dtype is npuint8:
                        hint = ((hkey * (hi-lo)) >> 32) + lo
                        return _c_search_hkey_ui8(hkey, pindex, stride0, hi, hint, True)
                    else:
                        raise AssertionError("Internal error")
                finally:
                    PyBuffer_Release(cython.address(indexbuf))
                #lint:enable
        else:
            dtype = self.dtype
            struct_dt = numpy.dtype([
                ('key_hash', dtype),
                ('key_offset', dtype),
                ('value', dtype),
            ])
            return self.index.view(struct_dt).reshape(self.index.shape[0]).searchsorted(
                numpy.array([(hkey,0,0)],dtype=struct_dt))[0] 
Example 44
Project: sharedbuffers   Author: jampp   File: mapped_struct.py    BSD 3-Clause "New" or "Revised" License 4 votes vote down vote up
def build(cls, initializer, destfile = None, tempdir = None, idmap = None,
            value_array_kwargs = {},
            id_mapper_kwargs = {}):
        """
        Builds a mapping of keys to objects with a uniform :class:`Schema` into a memory mapped temporary file.

        :param iterable initializer: Content of the mapping.

        :param file destfile: *(optional)* An explicit file where the mapping should be built.
            This has to be an actual file, since it needs to be memory-mapped.
            The mapping will be written at the current position, and memory-mapped from it.

        :param str tempdir: *(optional)* A directory into which temporary files will be constructed. The build
            process needs temporary storage, so it will be used even when an explicit ``destfile`` is given.

        :type idmap: dict-like or StrongIdMap
        :param idmap: An :term:`idmap` to be used during the construction. If not given, a temporary
            :term:`idmap` is constructed for each object that is written, preventing instance deduplication
            across items but reducing memory usage.

        :param dict value_array_kwargs: Custom keyword arguments to be passed when invoking
            :attr:`ValueArray` . :meth:`~MappedArrayProxyBase.build`.

        :param dict id_mapper_kwargs: Custom keyword arguments to be passed when invoking
            :attr:`IdMapper` . :meth:`~NumericIdMapper.build`.

        :rtype: MappedMappingProxyBase
        :returns: The constructed mapping
        """
        if destfile is None:
            destfile = tempfile.NamedTemporaryFile(dir = tempdir)

        # Must dump values and keys to temporary files because
        # we don't know the size of the idmap before building it,
        # and it has to be at the beginning of the file
        with tempfile.NamedTemporaryFile(dir = tempdir, suffix = '.v',) as values_file:
            with tempfile.NamedTemporaryFile(dir = tempdir, suffix = '.k') as keys_file:
                initial_pos = destfile.tell()

                value_array = cls.ValueArray.build(
                    _iter_values_dump_keys(initializer, keys_file), values_file,
                    tempdir = tempdir, idmap = idmap, **value_array_kwargs)

                id_mapper = cls.IdMapper.build(
                    _iter_key_dump(keys_file), destfile,
                    tempdir = tempdir, **id_mapper_kwargs)

            # pad to multiple of 32 for better cache alignment
            pos = destfile.tell()
            if pos & 31:
                destfile.write("\x00" * (32 - (pos & 31)))

            values_pos = destfile.tell()

            blocklen = 1 << 20
            for start in xrange(0, len(value_array.buf), blocklen):
                destfile.write(buffer(value_array.buf, start, blocklen))
            destfile.write(cls._Footer.pack(values_pos - initial_pos))
            destfile.flush()

            return cls(value_array, id_mapper)