Python mmap.MAP_PRIVATE Examples

def mmap(fd, offset, size):
    prot = MMAP.PROT_READ | MMAP.PROT_WRITE
    flags = MMAP.MAP_PRIVATE

    # When trying to map the contents of a file into memory, the offset must be a multiple of the page size (see
    # `man mmap`). So we need to align it before passing it to mmap(). Doing so also increases the size of the memory
    # area needed, so we need to account for that difference.
    aligned_offset = offset & ~0xFFF
    size += offset - aligned_offset

    if size & 0xFFF != 0:
        size = (size & ~0xFFF) + 0x1000
    assert size > 0

    result = mmap_function(0, size, prot, flags, fd, aligned_offset)
    assert result != ctypes.c_void_p(-1).value

    # Now when returning the pointer to the user, we need to skip the corrected offset so that the user doesn't end up
    # with a pointer to another region of the file than the one they requested.
    return ctypes.cast(result + offset - aligned_offset, ctypes.POINTER(ctypes.c_char)) 

def readelf(path):
    elf = {
        'elf_header': {},
        'sections': [],
        'strtabs': {},
        'dynamic': [],
    }
    f = open(path, 'rb')
    buffer = mmap.mmap(f.fileno(), 0, mmap.MAP_PRIVATE, mmap.PROT_READ)
    read_header(elf, buffer)
    read_section_header(elf, buffer)
    read_strtab(elf, buffer)
    read_dynamic(elf, buffer)
    buffer.close()
    f.close()
    return elf 

def _get_buf(size):
    _buf = mmap.mmap(-1, size, flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS,
                    prot=mmap.PROT_READ|mmap.PROT_WRITE )
    _buf.seek(0,2) # move seek pointer to end
    return _buf 

def get_buffer(size):
    return mmap.mmap(-1, size, flags=mmap.MAP_PRIVATE|mmap.MAP_ANONYMOUS, 
                    prot=mmap.PROT_READ|mmap.PROT_WRITE )

# Also emulate a StringIO object. 

def mmap_read(f, sz = 0, close=True):
    """Create a read-only memory mapped region on file 'f'.
    If sz is 0, the region will cover the entire file.
    """
    return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ, close) 

def mmap_readwrite_private(f, sz = 0, close=True):
    """Create a read-write memory mapped region on file 'f'.
    If sz is 0, the region will cover the entire file.
    The map is private, which means the changes are never flushed back to the
    file.
    """
    return _mmap_do(f, sz, mmap.MAP_PRIVATE, mmap.PROT_READ|mmap.PROT_WRITE,
                    close) 

def open(self, mode, typeaccess=0, mmapmode = mmap.MAP_PRIVATE) :
		self.typeaccess = typeaccess
		try :
			self.__fd = open("/dev/mem", mode)
		except IOError :
			print "Mem::open IOErreur"
			sys.exit(-1)
		
		if(self.typeaccess != 0):
			try :
				self.__data = mmap.mmap(self.__fd.fileno(), 100 * 1024 * 1024, mmapmode)
				#print self.__data
			except TypeError :
				print "Mem::open TypeError"
				sys.exit(-1) 

def __init__(self, file, prune, num_docs, vocab_size, in_memory, gpu):
        self.file = file = open(file, 'rb')
        mmp = mmap.mmap(file.fileno(), 0, flags=mmap.MAP_PRIVATE, prot=mmap.PROT_READ)
        if in_memory:
            # file will be read in in full sequentially
            os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_SEQUENTIAL)
        else:
            # file will be read randomly as needed
            os.posix_fadvise(file.fileno(), 0, 0, os.POSIX_FADV_RANDOM)
        if prune != 0:
            S_DTYPE = 2
            if in_memory:
                mmp = np.empty((num_docs, prune), dtype='i2'), np.empty((num_docs, prune), dtype='f2')
                for did in logger.pbar(range(num_docs), desc='loading dvecs'):
                    try:
                        mmp[0][did] = np.frombuffer(file.read(prune*S_DTYPE), dtype='i2')
                        mmp[1][did] = np.frombuffer(file.read(prune*S_DTYPE), dtype='f2')
                    except ValueError:
                        pass
                file.close()
            self.lookup = self.dvec_lookup_pruned
        else:
            if in_memory:
                mmp = file.read()
                file.close()
            self.lookup = self.dvec_lookup_unpruned

        self.prune = prune
        self.num_docs = num_docs
        self.vocab_size = vocab_size
        self.mmp = mmp
        self.gpu = gpu
        self.in_memory = in_memory 

def compile(self):
		import ctypes

		machine_code = bytes.join(b'', self.machine_code)
		self.size = ctypes.c_size_t(len(machine_code))

		if is_windows:
			# Allocate a memory segment the size of the machine code, and make it executable
			size = len(machine_code)
			# Alloc at least 1 page to ensure we own all pages that we want to change protection on
			if size < 0x1000: size = 0x1000
			MEM_COMMIT = ctypes.c_ulong(0x1000)
			PAGE_READWRITE = ctypes.c_ulong(0x4)
			pfnVirtualAlloc = ctypes.windll.kernel32.VirtualAlloc
			pfnVirtualAlloc.restype = ctypes.c_void_p
			self.address = pfnVirtualAlloc(None, ctypes.c_size_t(size), MEM_COMMIT, PAGE_READWRITE)
			if not self.address:
				raise Exception("Failed to VirtualAlloc")

			# Copy the machine code into the memory segment
			memmove = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t)(ctypes._memmove_addr)
			if memmove(self.address, machine_code, size) < 0:
				raise Exception("Failed to memmove")

			# Enable execute permissions
			PAGE_EXECUTE = ctypes.c_ulong(0x10)
			old_protect = ctypes.c_ulong(0)
			pfnVirtualProtect = ctypes.windll.kernel32.VirtualProtect
			res = pfnVirtualProtect(ctypes.c_void_p(self.address), ctypes.c_size_t(size), PAGE_EXECUTE, ctypes.byref(old_protect))
			if not res:
				raise Exception("Failed VirtualProtect")

			# Flush Instruction Cache
			# First, get process Handle
			if not self.prochandle:
				pfnGetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
				pfnGetCurrentProcess.restype = ctypes.c_void_p
				self.prochandle = ctypes.c_void_p(pfnGetCurrentProcess())
			# Actually flush cache
			res = ctypes.windll.kernel32.FlushInstructionCache(self.prochandle, ctypes.c_void_p(self.address), ctypes.c_size_t(size))
			if not res:
				raise Exception("Failed FlushInstructionCache")
		else:
			from mmap import mmap, MAP_PRIVATE, MAP_ANONYMOUS, PROT_WRITE, PROT_READ, PROT_EXEC

			# Allocate a private and executable memory segment the size of the machine code
			machine_code = bytes.join(b'', self.machine_code)
			self.size = len(machine_code)
			self.mm = mmap(-1, self.size, flags=MAP_PRIVATE | MAP_ANONYMOUS, prot=PROT_WRITE | PROT_READ | PROT_EXEC)

			# Copy the machine code into the memory segment
			self.mm.write(machine_code)
			self.address = ctypes.addressof(ctypes.c_int.from_buffer(self.mm))

		# Cast the memory segment into a function
		functype = ctypes.CFUNCTYPE(self.restype, *self.argtypes)
		self.func = functype(self.address) 

def compile(self):
		machine_code = bytes.join(b'', self.machine_code)
		self.size = ctypes.c_size_t(len(machine_code))

		if DataSource.is_windows:
			# Allocate a memory segment the size of the machine code, and make it executable
			size = len(machine_code)
			# Alloc at least 1 page to ensure we own all pages that we want to change protection on
			if size < 0x1000: size = 0x1000
			MEM_COMMIT = ctypes.c_ulong(0x1000)
			PAGE_READWRITE = ctypes.c_ulong(0x4)
			pfnVirtualAlloc = ctypes.windll.kernel32.VirtualAlloc
			pfnVirtualAlloc.restype = ctypes.c_void_p
			self.address = pfnVirtualAlloc(None, ctypes.c_size_t(size), MEM_COMMIT, PAGE_READWRITE)
			if not self.address:
				raise Exception("Failed to VirtualAlloc")

			# Copy the machine code into the memory segment
			memmove = ctypes.CFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t)(ctypes._memmove_addr)
			if memmove(self.address, machine_code, size) < 0:
				raise Exception("Failed to memmove")

			# Enable execute permissions
			PAGE_EXECUTE = ctypes.c_ulong(0x10)
			old_protect = ctypes.c_ulong(0)
			pfnVirtualProtect = ctypes.windll.kernel32.VirtualProtect
			res = pfnVirtualProtect(ctypes.c_void_p(self.address), ctypes.c_size_t(size), PAGE_EXECUTE, ctypes.byref(old_protect))
			if not res:
				raise Exception("Failed VirtualProtect")

			# Flush Instruction Cache
			# First, get process Handle
			if not self.prochandle:
				pfnGetCurrentProcess = ctypes.windll.kernel32.GetCurrentProcess
				pfnGetCurrentProcess.restype = ctypes.c_void_p
				self.prochandle = ctypes.c_void_p(pfnGetCurrentProcess())
			# Actually flush cache
			res = ctypes.windll.kernel32.FlushInstructionCache(self.prochandle, ctypes.c_void_p(self.address), ctypes.c_size_t(size))
			if not res:
				raise Exception("Failed FlushInstructionCache")
		else:
			from mmap import mmap, MAP_PRIVATE, MAP_ANONYMOUS, PROT_WRITE, PROT_READ, PROT_EXEC

			# Allocate a private and executable memory segment the size of the machine code
			machine_code = bytes.join(b'', self.machine_code)
			self.size = len(machine_code)
			self.mm = mmap(-1, self.size, flags=MAP_PRIVATE | MAP_ANONYMOUS, prot=PROT_WRITE | PROT_READ | PROT_EXEC)

			# Copy the machine code into the memory segment
			self.mm.write(machine_code)
			self.address = ctypes.addressof(ctypes.c_int.from_buffer(self.mm))

		# Cast the memory segment into a function
		functype = ctypes.CFUNCTYPE(self.restype, *self.argtypes)
		self.func = functype(self.address)