Author: Edd Barrett <vex...@gmail.com> Branch: refactor_rmmap Changeset: r86169:a80e988edfd0 Date: 2016-08-12 11:26 +0100 http://bitbucket.org/pypy/pypy/changeset/a80e988edfd0/
Log: Refactor rmmap.py JIT support into its own file. And test with a W^X patch to the build system. diff --git a/rpython/jit/backend/llsupport/asmmemmgr.py b/rpython/jit/backend/llsupport/asmmemmgr.py --- a/rpython/jit/backend/llsupport/asmmemmgr.py +++ b/rpython/jit/backend/llsupport/asmmemmgr.py @@ -1,7 +1,7 @@ import sys from rpython.rlib.rarithmetic import intmask, r_uint, LONG_BIT from rpython.rlib.objectmodel import we_are_translated -from rpython.rlib import rmmap +from rpython.jit.backend.llsupport import rmmap from rpython.rlib.debug import debug_start, debug_print, debug_stop from rpython.rlib.debug import have_debug_prints from rpython.rtyper.lltypesystem import lltype, rffi diff --git a/rpython/jit/backend/llsupport/rmmap.py b/rpython/jit/backend/llsupport/rmmap.py new file mode 100644 --- /dev/null +++ b/rpython/jit/backend/llsupport/rmmap.py @@ -0,0 +1,145 @@ +"""mmap for the JIT + +Derived from rlib.rmmap +""" + +# borrow a few bits from our rlib cousin, but we must not share functions +from rpython.rlib.rmmap import _POSIX, _MS_WINDOWS, _CYGWIN, constants, CConfig +if _POSIX: + from rpython.rlib.rmmap import ( + MAP_PRIVATE, MAP_ANONYMOUS, PROT_EXEC, PROT_READ, PROT_WRITE, PTR) +if _MS_WINDOWS: + from rpython.rlib.rwin32 import LPDWORD, DWORD, BOOL + from rpython.rlib.rmmap import ( + MEM_COMMIT, MEM_RESERVE, PAGE_EXECUTE_READWRITE, MEM_RELEASE) + +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rlib import rposix +from rpython.rtyper.tool import rffi_platform + + +locals().update(constants) + + +def safe_external(name, args, result, save_err_on_unsafe=0, save_err_on_safe=0, + **kwargs): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + sandboxsafe=True, releasegil=False, + save_err=save_err_on_safe, **kwargs) + + +def safe_winexternal(name, args, result, **kwargs): + return rffi.llexternal(name, args, result, + compilation_info=CConfig._compilation_info_, + calling_conv='win', sandboxsafe=True, + releasegil=False, **kwargs) + + +if _POSIX: + c_mmap_safe = safe_external( + 'mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, + macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO) + + c_munmap_safe = safe_external('munmap', [PTR, size_t], rffi.INT) + + +if _CYGWIN: + c_free_safe = safe_external('free', [PTR], lltype.Void, macro=True) + c_malloc_safe = safe_external('malloc', [size_t], PTR, macro=True) + +if _MS_WINDOWS: + VirtualAlloc_safe = safe_winexternal( + 'VirtualAlloc', [rffi.VOIDP, rffi.SIZE_T, DWORD, DWORD], rffi.VOIDP) + + _VirtualProtect_safe = safe_winexternal( + 'VirtualProtect', [rffi.VOIDP, rffi.SIZE_T, DWORD, LPDWORD], BOOL) + + def VirtualProtect(addr, size, mode, oldmode_ptr): + return _VirtualProtect_safe( + addr, rffi.cast(rffi.SIZE_T, size), rffi.cast(DWORD, mode), + oldmode_ptr) + VirtualProtect._annspecialcase_ = 'specialize:ll' + + VirtualFree_safe = safe_winexternal( + 'VirtualFree', [rffi.VOIDP, rffi.SIZE_T, DWORD], BOOL) + + +if _POSIX: + def alloc_hinted(hintp, map_size): + flags = MAP_PRIVATE | MAP_ANONYMOUS + prot = PROT_EXEC | PROT_READ | PROT_WRITE + return c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + + # XXX is this really necessary? + class Hint: + pos = -0x4fff0000 # for reproducible results + hint = Hint() + + def alloc(map_size): + """Allocate memory. This is intended to be used by the JIT, + so the memory has the executable bit set and gets allocated + internally in case of a sandboxed process. + """ + from errno import ENOMEM + from rpython.rlib import debug + + if _CYGWIN: + # XXX: JIT memory should be using mmap MAP_PRIVATE with + # PROT_EXEC but Cygwin's fork() fails. mprotect() + # cannot be used, but seems to be unnecessary there. + res = c_malloc_safe(map_size) + if res == rffi.cast(PTR, 0): + raise MemoryError + return res + res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size) + if res == rffi.cast(PTR, -1): + # some systems (some versions of OS/X?) complain if they + # are passed a non-zero address. Try again. + res = alloc_hinted(rffi.cast(PTR, 0), map_size) + if res == rffi.cast(PTR, -1): + # ENOMEM simply raises MemoryError, but other errors are fatal + if rposix.get_saved_errno() != ENOMEM: + debug.fatalerror_notb( + "Got an unexpected error trying to allocate some " + "memory for the JIT (tried to do mmap() with " + "PROT_EXEC|PROT_READ|PROT_WRITE). This can be caused " + "by a system policy like PAX. You need to find how " + "to work around the policy on your system.") + raise MemoryError + else: + hint.pos += map_size + return res + alloc._annenforceargs_ = (int,) + + if _CYGWIN: + free = c_free_safe + else: + free = c_munmap_safe + +elif _MS_WINDOWS: + class Hint: + pos = -0x4fff0000 # for reproducible results + hint = Hint() + # XXX this has no effect on windows + + def alloc(map_size): + """Allocate memory. This is intended to be used by the JIT, + so the memory has the executable bit set. + XXX implement me: it should get allocated internally in + case of a sandboxed process + """ + null = lltype.nullptr(rffi.VOIDP.TO) + res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, + PAGE_EXECUTE_READWRITE) + if not res: + raise MemoryError + arg = lltype.malloc(LPDWORD.TO, 1, zero=True, flavor='raw') + VirtualProtect(res, map_size, PAGE_EXECUTE_READWRITE, arg) + lltype.free(arg, flavor='raw') + # ignore errors, just try + return res + alloc._annenforceargs_ = (int,) + + def free(ptr, map_size): + VirtualFree_safe(ptr, 0, MEM_RELEASE) diff --git a/rpython/jit/backend/x86/detect_feature.py b/rpython/jit/backend/x86/detect_feature.py --- a/rpython/jit/backend/x86/detect_feature.py +++ b/rpython/jit/backend/x86/detect_feature.py @@ -1,17 +1,20 @@ import sys import struct from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rlib.rmmap import alloc, free +from rpython.rlib.rmmap import alloc, free, set_pages_executable + +CPU_INFO_SZ = 4096 def cpu_info(instr): - data = alloc(4096) + data = alloc(CPU_INFO_SZ, no_exec=True) pos = 0 for c in instr: data[pos] = c pos += 1 + set_pages_executable(data, CPU_INFO_SZ) fnptr = rffi.cast(lltype.Ptr(lltype.FuncType([], lltype.Signed)), data) code = fnptr() - free(data, 4096) + free(data, CPU_INFO_SZ) return code def detect_sse2(): diff --git a/rpython/rlib/rmmap.py b/rpython/rlib/rmmap.py --- a/rpython/rlib/rmmap.py +++ b/rpython/rlib/rmmap.py @@ -155,6 +155,8 @@ c_mmap, c_mmap_safe = external('mmap', [PTR, size_t, rffi.INT, rffi.INT, rffi.INT, off_t], PTR, macro=True, save_err_on_unsafe=rffi.RFFI_SAVE_ERRNO) + c_mprotect, _ = external('mprotect', + [PTR, size_t, rffi.INT], rffi.INT) # 'mmap' on linux32 is a macro that calls 'mmap64' _, c_munmap_safe = external('munmap', [PTR, size_t], rffi.INT) c_msync, _ = external('msync', [PTR, size_t, rffi.INT], rffi.INT, @@ -705,14 +707,21 @@ m.setdata(res, map_size) return m - def alloc_hinted(hintp, map_size): + def alloc_hinted(hintp, map_size, no_exec=False): flags = MAP_PRIVATE | MAP_ANONYMOUS - prot = PROT_EXEC | PROT_READ | PROT_WRITE + prot = PROT_READ | PROT_WRITE + if not no_exec: + prot |= PROT_EXEC if we_are_translated(): flags = NonConstant(flags) prot = NonConstant(prot) return c_mmap_safe(hintp, map_size, prot, flags, -1, 0) + def set_pages_executable(addr, size): + rv = c_mprotect(addr, size, PROT_EXEC) + if rv < 0: + debug.fatalerror_notb("set_pages_executable failed") + def clear_large_memory_chunk_aligned(addr, map_size): addr = rffi.cast(PTR, addr) flags = MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS @@ -728,7 +737,7 @@ pos = -0x4fff0000 # for reproducible results hint = Hint() - def alloc(map_size): + def alloc(map_size, no_exec=False): """Allocate memory. This is intended to be used by the JIT, so the memory has the executable bit set and gets allocated internally in case of a sandboxed process. @@ -744,11 +753,11 @@ if res == rffi.cast(PTR, 0): raise MemoryError return res - res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size) + res = alloc_hinted(rffi.cast(PTR, hint.pos), map_size, no_exec=no_exec) if res == rffi.cast(PTR, -1): # some systems (some versions of OS/X?) complain if they # are passed a non-zero address. Try again. - res = alloc_hinted(rffi.cast(PTR, 0), map_size) + res = alloc_hinted(rffi.cast(PTR, 0), map_size, no_exec=no_exec) if res == rffi.cast(PTR, -1): # ENOMEM simply raises MemoryError, but other errors are fatal if rposix.get_saved_errno() != ENOMEM: @@ -762,7 +771,7 @@ else: hint.pos += map_size return res - alloc._annenforceargs_ = (int,) + alloc._annenforceargs_ = (int, bool) if _CYGWIN: free = c_free_safe @@ -933,11 +942,13 @@ hint = Hint() # XXX this has no effect on windows - def alloc(map_size): + def alloc(map_size, no_exec=no_exec): """Allocate memory. This is intended to be used by the JIT, so the memory has the executable bit set. XXX implement me: it should get allocated internally in case of a sandboxed process + + XXX no_exec not implemented on windows """ null = lltype.nullptr(rffi.VOIDP.TO) res = VirtualAlloc_safe(null, map_size, MEM_COMMIT | MEM_RESERVE, @@ -949,7 +960,10 @@ lltype.free(arg, flavor='raw') # ignore errors, just try return res - alloc._annenforceargs_ = (int,) + alloc._annenforceargs_ = (int, bool) + + def set_pages_executable(addr, size): + pass # XXX not implemented on windows def free(ptr, map_size): VirtualFree_safe(ptr, 0, MEM_RELEASE) _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit