Python mmap() Examples

def create_shm_buffer(touch, width, height):
    stride = width * 4
    size = stride * height

    with tempfile.TemporaryFile() as f:
        f.write(b"\x64" * size)
        f.flush()

        fd = f.fileno()
        touch["data"] = mmap.mmap(
            fd, size, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE
        )
        pool = touch["shm"].create_pool(fd, size)
        touch["buffer"] = pool.create_buffer(
            0, width, height, stride, WlShm.format.argb8888.value
        )
        pool.destroy() 

def init(self):
        self.control = safe_socket(socket.AF_UNIX)
        self.control.settimeout(None)
        self.control.setblocking(1)
        while True:
            try:
                self.control.connect(self.control_filename)
                break
            except socket_error:
                pass

        self.kafl_shm_f     = os.open(self.bitmap_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        self.fs_shm_f       = os.open(self.payload_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(self.kafl_shm_f, self.bitmap_size)
        os.ftruncate(self.fs_shm_f, (128 << 10))

        self.kafl_shm       = mmap.mmap(self.kafl_shm_f, self.bitmap_size, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        self.fs_shm         = mmap.mmap(self.fs_shm_f, (128 << 10),  mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)

        return True 

def wipe(self):
    	filter_bitmap_fd = os.open("/dev/shm/kafl_filter0", os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(filter_bitmap_fd, self.config.config_values['BITMAP_SHM_SIZE'])
        filter_bitmap = mmap.mmap(filter_bitmap_fd, self.config.config_values['BITMAP_SHM_SIZE'], mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        for i in range(self.config.config_values['BITMAP_SHM_SIZE']):
        	filter_bitmap[i] = '\x00'
        filter_bitmap.close()
        os.close(filter_bitmap_fd)

        filter_bitmap_fd = os.open("/dev/shm/kafl_tfilter", os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(filter_bitmap_fd, 0x1000000)
        filter_bitmap = mmap.mmap(filter_bitmap_fd, 0x1000000, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        for i in range(0x1000000):
        	filter_bitmap[i] = '\x00'
        filter_bitmap.close()
        os.close(filter_bitmap_fd) 

def __set_binary(self, filename, binaryfile, max_size):
        shm_fd = os.open(filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(shm_fd, max_size)
        shm = mmap.mmap(shm_fd, max_size, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        shm.seek(0x0)
        shm.write('\x00' * max_size)
        shm.seek(0x0)

        f = open(binaryfile, "rb")
        bytes = f.read(1024)
        if bytes:
            shm.write(bytes)
        while bytes != "":
            bytes = f.read(1024)
            if bytes:
                shm.write(bytes)

        f.close()
        shm.close()
        os.close(shm_fd) 

def init(self):
        self.control = socket.socket(socket.AF_UNIX)
        while True:
            try:
                self.control.connect(self.control_filename)
                #self.control.connect(self.control_filename)
                break
            except socket_error:
                pass
                #time.sleep(0.01)

        self.kafl_shm_f     = os.open(self.bitmap_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        self.fs_shm_f       = os.open(self.payload_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        #argv_fd             = os.open(self.argv_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(self.kafl_shm_f, self.bitmap_size)
        os.ftruncate(self.fs_shm_f, (128 << 10))
        #os.ftruncate(argv_fd, (4 << 10))

        self.kafl_shm       = mmap.mmap(self.kafl_shm_f, self.bitmap_size, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        self.fs_shm         = mmap.mmap(self.fs_shm_f, (128 << 10),  mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)

        return True 

def setClockPython(newClock):
    with open("/dev/mem", os.O_RDWR) as f:
        CPPCR = (0x10 >> 2)

        m = newClock / 6
        if(m >= 200):
            m = 200
        
        if(m < 4):
            m = 4
        

        try:
            map.mmap(1024, mmap.MAP_SHARED, mmap.PROT_READ | mmap.PROT_WRITE, 0x10000000) 
            map.seek(CPPCR)
            map.write((m << 24) | 0x090520)
            map.close()
        except Exception as ex:
            print(str(ex)) 

def mmap(fd, offset, size):
    prot = MMAP.PROT_READ | MMAP.PROT_WRITE
    flags = MMAP.MAP_PRIVATE

    # When trying to map the contents of a file into memory, the offset must be a multiple of the page size (see
    # `man mmap`). So we need to align it before passing it to mmap(). Doing so also increases the size of the memory
    # area needed, so we need to account for that difference.
    aligned_offset = offset & ~0xFFF
    size += offset - aligned_offset

    if size & 0xFFF != 0:
        size = (size & ~0xFFF) + 0x1000
    assert size > 0

    result = mmap_function(0, size, prot, flags, fd, aligned_offset)
    assert result != ctypes.c_void_p(-1).value

    # Now when returning the pointer to the user, we need to skip the corrected offset so that the user doesn't end up
    # with a pointer to another region of the file than the one they requested.
    return ctypes.cast(result + offset - aligned_offset, ctypes.POINTER(ctypes.c_char)) 

def perf_events(trace_file: str) -> Generator[ct.Structure, None, None]:
    with open(trace_file, "rb+") as f:
        fd = f.fileno()
        size = os.fstat(fd).st_size
        with MMap(fd, size, mmap.PROT_READ, mmap.MAP_SHARED) as mm:
            header_size = ct.sizeof(perf_event_header)
            i = 0
            while i != size:
                assert (size - i) >= header_size
                ev = perf_event_header.from_address(mm.addr + i)
                struct_factory = EVENTS.get(ev.type)
                if struct_factory is None:
                    raise Exception("unexpected perf_event type: %d", ev.type)
                struct_type = struct_factory(ev.size)
                struct_size = ct.sizeof(struct_type)
                assert (size - i) >= struct_size
                struct = struct_type.from_address(mm.addr + i)

                yield struct

                i += ev.size 

def remoteSceneChanged(self, data):
        w, h, size, newfile = data
        #self._sizeHint = (whint, hhint)
        if self.shm is None or self.shm.size != size:
            if self.shm is not None:
                self.shm.close()
            if sys.platform.startswith('win'):
                self.shmtag = newfile   ## on windows, we create a new tag for every resize
                self.shm = mmap.mmap(-1, size, self.shmtag) ## can't use tmpfile on windows because the file can only be opened once.
            elif sys.platform == 'darwin':
                self.shmFile.close()
                self.shmFile = open(self._view.shmFileName(), 'r')
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
            else:
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
        self.shm.seek(0)
        data = self.shm.read(w*h*4)
        self._img = QtGui.QImage(data, w, h, QtGui.QImage.Format_ARGB32)
        self._img.data = data  # data must be kept alive or PySide 1.2.1 (and probably earlier) will crash.
        self.update() 

def get(self,request):
        n=12
        try:
            size = os.path.getsize(ACTIVITY_LOG)
            with open(ACTIVITY_LOG, "rb") as f:
                # for Windows the mmap parameters are different
                fm = mmap.mmap(f.fileno(), 0, mmap.MAP_SHARED, mmap.PROT_READ)
            for i in xrange(size - 1, -1, -1):
                if fm[i] == '\n':
                    n -= 1
                    if n == -1:
                        break
                lines = fm[i + 1 if i else 0:].splitlines()
            return JsonResponse({'status': "success", 'log': lines})
        except Exception as err:
            return JsonResponse({'status' : "error",'messagge' : err})
        finally:
            try:
                fm.close()
            except (UnboundLocalError, TypeError):
                return JsonResponse({'status':"error", 'message': "Activity log file is empty"})



#index 

def readelf(path):
    elf = {
        'elf_header': {},
        'sections': [],
        'strtabs': {},
        'dynamic': [],
    }
    f = open(path, 'rb')
    buffer = mmap.mmap(f.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ)
    read_header(elf, buffer)
    read_section_header(elf, buffer)
    read_strtab(elf, buffer)
    read_dynamic(elf, buffer)
    buffer.close()
    f.close()
    return elf 

def remoteSceneChanged(self, data):
        w, h, size, newfile = data
        #self._sizeHint = (whint, hhint)
        if self.shm is None or self.shm.size != size:
            if self.shm is not None:
                self.shm.close()
            if sys.platform.startswith('win'):
                self.shmtag = newfile   ## on windows, we create a new tag for every resize
                self.shm = mmap.mmap(-1, size, self.shmtag) ## can't use tmpfile on windows because the file can only be opened once.
            elif sys.platform == 'darwin':
                self.shmFile.close()
                self.shmFile = open(self._view.shmFileName(), 'r')
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
            else:
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
        self.shm.seek(0)
        data = self.shm.read(w*h*4)
        self._img = QtGui.QImage(data, w, h, QtGui.QImage.Format_ARGB32)
        self._img.data = data  # data must be kept alive or PySide 1.2.1 (and probably earlier) will crash.
        self.update() 

def wipe(self):
        filter_bitmap_fd = os.open("/dev/shm/kafl_filter0", os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(filter_bitmap_fd, self.config.config_values['BITMAP_SHM_SIZE'])
        filter_bitmap = mmap.mmap(filter_bitmap_fd, self.config.config_values['BITMAP_SHM_SIZE'], mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        for i in range(self.config.config_values['BITMAP_SHM_SIZE']):
            filter_bitmap[i] = '\x00'
        filter_bitmap.close()
        os.close(filter_bitmap_fd)

        filter_bitmap_fd = os.open("/dev/shm/kafl_tfilter", os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(filter_bitmap_fd, 0x1000000)
        filter_bitmap = mmap.mmap(filter_bitmap_fd, 0x1000000, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        for i in range(0x1000000):
            filter_bitmap[i] = '\x00'
        filter_bitmap.close()
        os.close(filter_bitmap_fd) 

def __set_binary(self, filename, binaryfile, max_size):
        shm_fd = os.open(filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(shm_fd, max_size)
        shm = mmap.mmap(shm_fd, max_size, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        shm.seek(0x0)
        shm.write('\x00' * max_size)
        shm.seek(0x0)

        f = open(binaryfile, "rb")
        bytes = f.read(1024)
        if bytes:
            shm.write(bytes)
        while bytes != "":
            bytes = f.read(1024)
            if bytes:
                shm.write(bytes)

        shm.flush()
        f.close()
        shm.close()
        os.close(shm_fd) 

def remoteSceneChanged(self, data):
        w, h, size, newfile = data
        #self._sizeHint = (whint, hhint)
        if self.shm is None or self.shm.size != size:
            if self.shm is not None:
                self.shm.close()
            if sys.platform.startswith('win'):
                self.shmtag = newfile   ## on windows, we create a new tag for every resize
                self.shm = mmap.mmap(-1, size, self.shmtag) ## can't use tmpfile on windows because the file can only be opened once.
            elif sys.platform == 'darwin':
                self.shmFile.close()
                self.shmFile = open(self._view.shmFileName(), 'r')
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
            else:
                self.shm = mmap.mmap(self.shmFile.fileno(), size, mmap.MAP_SHARED, mmap.PROT_READ)
        self.shm.seek(0)
        data = self.shm.read(w*h*4)
        self._img = QtGui.QImage(data, w, h, QtGui.QImage.Format_ARGB32)
        self._img.data = data  # data must be kept alive or PySide 1.2.1 (and probably earlier) will crash.
        self.update() 

def open_device():
    global _initialized
    if not _initialized:
        global _pwmfile
        _pwmfile = os.open(lms2012.PWM_DEVICE_NAME, os.O_RDWR | os.O_SYNC)
        global _motorfile
        _motorfile = os.open(lms2012.MOTOR_DEVICE_NAME, os.O_RDWR | os.O_SYNC)
        MOTORDATAArrray = lms2012.MOTORDATA * 4
        global _motormm
        _motormm = mmap(fileno=_motorfile, length=sizeof(MOTORDATAArrray),
                        flags=MAP_SHARED, prot=PROT_READ | PROT_WRITE, offset=0)
        global _motordata
        _motordata = MOTORDATAArrray.from_buffer(_motormm)
        _initialized = True
        os.write(_pwmfile, struct.pack('B', lms2012.opPROGRAM_START))

# This does not look required. Types are TACHO (for the big engines) and MINITACHO for the gun one
# and firmware should recognize them properly.
# def set_types(types):
#    os.write(_pwmfile, struct.pack('5B', lms2012.opOUTPUT_SET_TYPE, *types)) 

def __init__(self, path):
        super().__init__(path)
        if os.name == "nt":
            self._map_obj = mmap.mmap(self.path.fileno(), 0, access=mmap.ACCESS_READ)
        else:
            self._map_obj = mmap.mmap(self.path.fileno(), 0, prot=mmap.PROT_READ) 

def _open(self, physaddr, size):
        if not isinstance(physaddr, (int, long)):
            raise TypeError("Invalid physaddr type, should be integer.")
        if not isinstance(size, (int, long)):
            raise TypeError("Invalid size type, should be integer.")

        pagesize = os.sysconf(os.sysconf_names['SC_PAGESIZE'])

        self._physaddr = physaddr
        self._size = size
        self._aligned_physaddr = physaddr - (physaddr % pagesize)
        self._aligned_size = size + (physaddr - self._aligned_physaddr)

        try:
            fd = os.open("/dev/mem", os.O_RDWR | os.O_SYNC)
        except OSError as e:
            raise MMIOError(e.errno, "Opening /dev/mem: " + e.strerror)

        try:
            self.mapping = mmap.mmap(fd, self._aligned_size, flags=mmap.MAP_SHARED, prot=(mmap.PROT_READ | mmap.PROT_WRITE), offset=self._aligned_physaddr)
        except OSError as e:
            raise MMIOError(e.errno, "Mapping /dev/mem: " + e.strerror)

        try:
            os.close(fd)
        except OSError as e:
            raise MMIOError(e.errno, "Closing /dev/mem: " + e.strerror)

    # Methods 

def init(self):
        self.control = safe_socket(socket.AF_UNIX)
        self.control.settimeout(None)
        self.control.setblocking(1)
        while True:
            try:
                self.control.connect(self.control_filename)
                # self.control.connect(self.control_filename)
                break
            except socket_error:
                pass
                # time.sleep(0.01)

        self.kafl_shm_f = os.open(self.bitmap_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        self.fs_shm_f = os.open(self.payload_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)

        open(self.tracedump_filename, "wb").close()

        os.symlink(self.tracedump_filename, self.config.argument_values['work_dir'] + "/pt_trace_dump_" + self.qemu_id)

        os.symlink(self.payload_filename, self.config.argument_values['work_dir'] + "/payload_" + self.qemu_id)
        os.symlink(self.bitmap_filename, self.config.argument_values['work_dir'] + "/bitmap_" + self.qemu_id)

        # argv_fd             = os.open(self.argv_filename, os.O_RDWR | os.O_SYNC | os.O_CREAT)
        os.ftruncate(self.kafl_shm_f, self.bitmap_size)
        os.ftruncate(self.fs_shm_f, (128 << 10))
        # os.ftruncate(argv_fd, (4 << 10))

        self.kafl_shm = mmap.mmap(self.kafl_shm_f, self.bitmap_size, mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)
        self.c_bitmap = (ctypes.c_uint8 * self.bitmap_size).from_buffer(self.kafl_shm)
        self.last_bitmap_wrapper = ExecutionResult(None, None, None, None)
        self.fs_shm = mmap.mmap(self.fs_shm_f, (128 << 10), mmap.MAP_SHARED, mmap.PROT_WRITE | mmap.PROT_READ)

        return True 

def open_device():
    global _initialized
    if not _initialized:
        global _lcdfile
        _lcdfile = os.open(lms2012.LCD_DEVICE_NAME, os.O_RDWR)
        global lcdmm
        lcdmm = mmap(fileno=_lcdfile, length=MEM_WIDTH * SCREEN_HEIGHT,
                     flags=MAP_SHARED, prot=PROT_READ | PROT_WRITE, offset=0)
    _initialized = True 

def __init__(self, filename):
        """Read the given TrueType file.

        :Parameters:
            `filename`
                The name of any Windows, OS2 or Macintosh Truetype file.

        The object must be closed (see `close`) after use.

        An exception will be raised if the file does not exist or cannot
        be read.
        """
        if not filename: filename = ''
        len = os.stat(filename).st_size
        self._fileno = os.open(filename, os.O_RDONLY)
        if hasattr(mmap, 'MAP_SHARED'):
            self._data = mmap.mmap(self._fileno, len, mmap.MAP_SHARED,
                mmap.PROT_READ)
        else:
            self._data = mmap.mmap(self._fileno, len, None, mmap.ACCESS_READ)

        offsets = _read_offset_table(self._data, 0)
        self._tables = {}
        for table in _read_table_directory_entry.array(self._data,
            offsets.size, offsets.num_tables):
            self._tables[table.tag] = table

        self._names = None
        self._horizontal_metrics = None
        self._character_advances = None
        self._character_kernings = None
        self._glyph_kernings = None
        self._character_map = None
        self._glyph_map = None
        self._font_selection_flags = None

        self.header = \
            _read_head_table(self._data, self._tables['head'].offset)
        self.horizontal_header = \
            _read_horizontal_header(self._data, self._tables['hhea'].offset) 

def open_device():
    global _initialized
    if not _initialized:
        global _uifile
        _uifile = os.open(lms2012.UI_DEVICE_NAME, os.O_RDWR | os.O_SYNC)
        global _uimm
        _uimm = mmap(fileno=_uifile, length=sizeof(lms2012.UI),
                     flags=MAP_SHARED, prot=PROT_READ | PROT_WRITE, offset=0)
        global _ui
        _ui = lms2012.UI.from_buffer(_uimm)
        _initialized = True 

def open_device():
    global _initialized
    if not _initialized:
        global _devcon
        from . import devcon as _devcon
        global _iicfile
        _iicfile = os.open(lms2012.IIC_DEVICE_NAME, os.O_RDWR)
        global _iicmm
        _iicmm = mmap(fileno=_iicfile, length=sizeof(lms2012.IIC),
                      flags=MAP_SHARED, prot=PROT_READ | PROT_WRITE, offset=0)
        global _iic
        _iic = lms2012.IIC.from_buffer(_iicmm)
        _initialized = True 

def free(self, c_pointer, total_size):
        # unprotect it, since free() accesses it, I think...
        self.lib.mprotect(c_pointer, total_size,
                          ctypes.c_int(mmap.PROT_READ | mmap.PROT_WRITE))
        self.lib.free(c_pointer) 

def __enter__(self):
        self.lib.bcm_host_init()
        peri_addr = self.lib.bcm_host_get_peripheral_address()
        peri_size = self.lib.bcm_host_get_peripheral_size()
        page_size = os.sysconf("SC_PAGE_SIZE")
        assert(not (peri_addr & (page_size - 1)))
        fd = os.open('/dev/mem', os.O_RDWR)
        self.peri = mmap.mmap(fd, peri_size, mmap.MAP_SHARED, mmap.PROT_READ|mmap.PROT_WRITE, offset = peri_addr)
        self.peri_arr = np.frombuffer(self.peri, dtype=np.uint32)
        os.close(fd)
        return self 

def __init__(self, phys_address, size=PAGE_SIZE):
        """ Create object which maps physical memory to Python's mmap object.
        :param phys_address: based address of physical memory
        """
        self._size = size
        phys_address -= phys_address % PAGE_SIZE
        fd = self._open_dev("/dev/mem")
        self._memmap = mmap.mmap(fd, size, flags=mmap.MAP_SHARED,
                                 prot=mmap.PROT_READ | mmap.PROT_WRITE,
                                 offset=phys_address)
        self._close_dev(fd)
        atexit.register(self.cleanup) 

def get_map(cls, size):
        prot = mmap.PROT_EXEC | mmap.PROT_WRITE | mmap.PROT_READ
        return cls(-1, size, prot=prot) 

def __init__(self, vocab_filename, rows_filename, cols_filename=None):
    """Initializes the vectors from a text vocabulary and binary data."""
    with open(vocab_filename, 'r') as lines:
      self.vocab = [line.split()[0] for line in lines]
      self.word_to_idx = {word: idx for idx, word in enumerate(self.vocab)}

    n = len(self.vocab)

    with open(rows_filename, 'r') as rows_fh:
      rows_fh.seek(0, os.SEEK_END)
      size = rows_fh.tell()

      # Make sure that the file size seems reasonable.
      if size % (4 * n) != 0:
        raise IOError(
            'unexpected file size for binary vector file %s' % rows_filename)

      # Memory map the rows.
      dim = size / (4 * n)
      rows_mm = mmap.mmap(rows_fh.fileno(), 0, prot=mmap.PROT_READ)
      rows = np.matrix(
          np.frombuffer(rows_mm, dtype=np.float32).reshape(n, dim))

      # If column vectors were specified, then open them and add them to the
      # row vectors.
      if cols_filename:
        with open(cols_filename, 'r') as cols_fh:
          cols_mm = mmap.mmap(cols_fh.fileno(), 0, prot=mmap.PROT_READ)
          cols_fh.seek(0, os.SEEK_END)
          if cols_fh.tell() != size:
            raise IOError('row and column vector files have different sizes')

          cols = np.matrix(
              np.frombuffer(cols_mm, dtype=np.float32).reshape(n, dim))

          rows += cols
          cols_mm.close()

      # Normalize so that dot products are just cosine similarity.
      self.vecs = rows / np.linalg.norm(rows, axis=1).reshape(n, 1)
      rows_mm.close() 

def __init__(self, vocab_filename, rows_filename, cols_filename=None):
    """Initializes the vectors from a text vocabulary and binary data."""
    with open(vocab_filename, 'r') as lines:
      self.vocab = [line.split()[0] for line in lines]
      self.word_to_idx = {word: idx for idx, word in enumerate(self.vocab)}

    n = len(self.vocab)

    with open(rows_filename, 'r') as rows_fh:
      rows_fh.seek(0, os.SEEK_END)
      size = rows_fh.tell()

      # Make sure that the file size seems reasonable.
      if size % (4 * n) != 0:
        raise IOError(
            'unexpected file size for binary vector file %s' % rows_filename)

      # Memory map the rows.
      dim = size / (4 * n)
      rows_mm = mmap.mmap(rows_fh.fileno(), 0, prot=mmap.PROT_READ)
      rows = np.matrix(
          np.frombuffer(rows_mm, dtype=np.float32).reshape(n, dim))

      # If column vectors were specified, then open them and add them to the row
      # vectors.
      if cols_filename:
        with open(cols_filename, 'r') as cols_fh:
          cols_mm = mmap.mmap(cols_fh.fileno(), 0, prot=mmap.PROT_READ)
          cols_fh.seek(0, os.SEEK_END)
          if cols_fh.tell() != size:
            raise IOError('row and column vector files have different sizes')

          cols = np.matrix(
              np.frombuffer(cols_mm, dtype=np.float32).reshape(n, dim))

          rows += cols
          cols_mm.close()

      # Normalize so that dot products are just cosine similarity.
      self.vecs = rows / np.linalg.norm(rows, axis=1).reshape(n, 1)
      rows_mm.close() 

def __init__(self, vocab_filename, rows_filename, cols_filename=None):
    """Initializes the vectors from a text vocabulary and binary data."""
    with open(vocab_filename, 'r') as lines:
      self.vocab = [line.split()[0] for line in lines]
      self.word_to_idx = {word: idx for idx, word in enumerate(self.vocab)}

    n = len(self.vocab)

    with open(rows_filename, 'r') as rows_fh:
      rows_fh.seek(0, os.SEEK_END)
      size = rows_fh.tell()

      # Make sure that the file size seems reasonable.
      if size % (4 * n) != 0:
        raise IOError(
            'unexpected file size for binary vector file %s' % rows_filename)

      # Memory map the rows.
      dim = size / (4 * n)
      rows_mm = mmap.mmap(rows_fh.fileno(), 0, prot=mmap.PROT_READ)
      rows = np.matrix(
          np.frombuffer(rows_mm, dtype=np.float32).reshape(n, dim))

      # If column vectors were specified, then open them and add them to the
      # row vectors.
      if cols_filename:
        with open(cols_filename, 'r') as cols_fh:
          cols_mm = mmap.mmap(cols_fh.fileno(), 0, prot=mmap.PROT_READ)
          cols_fh.seek(0, os.SEEK_END)
          if cols_fh.tell() != size:
            raise IOError('row and column vector files have different sizes')

          cols = np.matrix(
              np.frombuffer(cols_mm, dtype=np.float32).reshape(n, dim))

          rows += cols
          cols_mm.close()

      # Normalize so that dot products are just cosine similarity.
      self.vecs = rows / np.linalg.norm(rows, axis=1).reshape(n, 1)
      rows_mm.close()