Author: Manuel Jacob <m...@manueljacob.de> Branch: py3k Changeset: r79128:61296bb0341b Date: 2015-08-22 01:22 +0200 http://bitbucket.org/pypy/pypy/changeset/61296bb0341b/
Log: hg merge default diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.2.0 +Version: 1.2.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.2.0" -__version_info__ = (1, 2, 0) +__version__ = "1.2.1" +__version_info__ = (1, 2, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -46,7 +46,7 @@ # endif #else # include <stdint.h> -# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) +# if (defined (__SVR4) && defined (__sun)) || defined(_AIX) || defined(__hpux) # include <alloca.h> # endif #endif diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -15,9 +15,11 @@ except ImportError: lock = None -_r_comment = re.compile(r"/\*.*?\*/|//.*?$", re.DOTALL | re.MULTILINE) -_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)\s+(.*?)$", - re.MULTILINE) +_r_comment = re.compile(r"/\*.*?\*/|//([^\n\\]|\\.)*?$", + re.DOTALL | re.MULTILINE) +_r_define = re.compile(r"^\s*#\s*define\s+([A-Za-z_][A-Za-z_0-9]*)" + r"\b((?:[^\n\\]|\\.)*?)$", + re.DOTALL | re.MULTILINE) _r_partial_enum = re.compile(r"=\s*\.\.\.\s*[,}]|\.\.\.\s*\}") _r_enum_dotdotdot = re.compile(r"__dotdotdot\d+__$") _r_partial_array = re.compile(r"\[\s*\.\.\.\s*\]") @@ -39,6 +41,7 @@ macros = {} for match in _r_define.finditer(csource): macroname, macrovalue = match.groups() + macrovalue = macrovalue.replace('\\\n', '').strip() macros[macroname] = macrovalue csource = _r_define.sub('', csource) # Replace "[...]" with "[__dotdotdotarray__]" @@ -423,13 +426,10 @@ raise api.CDefError( "%s: a function with only '(...)' as argument" " is not correct C" % (funcname or 'in expression')) - elif (len(params) == 1 and - isinstance(params[0].type, pycparser.c_ast.TypeDecl) and - isinstance(params[0].type.type, pycparser.c_ast.IdentifierType) - and list(params[0].type.type.names) == ['void']): - del params[0] args = [self._as_func_arg(self._get_type(argdeclnode.type)) for argdeclnode in params] + if not ellipsis and args == [model.void_type]: + args = [] result = self._get_type(typenode.type) return model.RawFunctionType(tuple(args), result, ellipsis) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -4,11 +4,6 @@ VERSION = "0x2601" -try: - int_type = (int, long) -except NameError: # Python 3 - int_type = int - class GlobalExpr: def __init__(self, name, address, type_op, size=0, check_value=0): diff --git a/lib_pypy/cffi/setuptools_ext.py b/lib_pypy/cffi/setuptools_ext.py --- a/lib_pypy/cffi/setuptools_ext.py +++ b/lib_pypy/cffi/setuptools_ext.py @@ -81,10 +81,16 @@ allsources.extend(kwds.pop('sources', [])) ext = Extension(name=module_name, sources=allsources, **kwds) - def make_mod(tmpdir): + def make_mod(tmpdir, pre_run=None): c_file = os.path.join(tmpdir, module_name + source_extension) log.info("generating cffi module %r" % c_file) mkpath(tmpdir) + # a setuptools-only, API-only hook: called with the "ext" and "ffi" + # arguments just before we turn the ffi into C code. To use it, + # subclass the 'distutils.command.build_ext.build_ext' class and + # add a method 'def pre_run(self, ext, ffi)'. + if pre_run is not None: + pre_run(ext, ffi) updated = recompiler.make_c_source(ffi, module_name, source, c_file) if not updated: log.info("already up-to-date") @@ -98,7 +104,8 @@ class build_ext_make_mod(base_class): def run(self): if ext.sources[0] == '$PLACEHOLDER': - ext.sources[0] = make_mod(self.build_temp) + pre_run = getattr(self, 'pre_run', None) + ext.sources[0] = make_mod(self.build_temp, pre_run) base_class.run(self) dist.cmdclass['build_ext'] = build_ext_make_mod # NB. multiple runs here will create multiple 'build_ext_make_mod' diff --git a/pypy/doc/index-of-whatsnew.rst b/pypy/doc/index-of-whatsnew.rst --- a/pypy/doc/index-of-whatsnew.rst +++ b/pypy/doc/index-of-whatsnew.rst @@ -7,6 +7,7 @@ .. toctree:: whatsnew-head.rst + whatsnew-2.6.1.rst whatsnew-2.6.0.rst whatsnew-2.5.1.rst whatsnew-2.5.0.rst diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-2.6.1.rst copy from pypy/doc/whatsnew-head.rst copy to pypy/doc/whatsnew-2.6.1.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-2.6.1.rst @@ -1,6 +1,6 @@ -======================= -What's new in PyPy 2.6+ -======================= +======================== +What's new in PyPy 2.6.1 +======================== .. this is a revision shortly after release-2.6.0 .. startrev: 91904d5c5188 @@ -32,7 +32,10 @@ ``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.) .. branch: cffi-callback-onerror +Part of cffi 1.2. + .. branch: cffi-new-allocator +Part of cffi 1.2. .. branch: unicode-dtype diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -2,68 +2,6 @@ What's new in PyPy 2.6+ ======================= -.. this is a revision shortly after release-2.6.0 -.. startrev: 91904d5c5188 +.. this is a revision shortly after release-2.6.1 +.. startrev: 83ebc73d4fcb -.. branch: use_min_scalar -Correctly resolve the output dtype of ufunc(array, scalar) calls. - -.. branch: stdlib-2.7.10 - -Update stdlib to version 2.7.10 - -.. branch: issue2062 - -.. branch: disable-unroll-for-short-loops -The JIT no longer performs loop unrolling if the loop compiles to too much code. - -.. branch: run-create_cffi_imports - -Build cffi import libraries as part of translation by monkey-patching an -additional task into translation - -.. branch: int-float-list-strategy - -Use a compact strategy for Python lists that mix integers and floats, -at least if the integers fit inside 32 bits. These lists are now -stored as an array of floats, like lists that contain only floats; the -difference is that integers are stored as tagged NaNs. (This should -have no visible effect! After ``lst = [42, 42.5]``, the value of -``lst[0]`` is still *not* the float ``42.0`` but the integer ``42``.) - -.. branch: cffi-callback-onerror -.. branch: cffi-new-allocator - -.. branch: unicode-dtype - -Partial implementation of unicode dtype and unicode scalars. - -.. branch: dtypes-compatability - -Improve compatibility with numpy dtypes; handle offsets to create unions, -fix str() and repr(), allow specifying itemsize, metadata and titles, add flags, -allow subclassing dtype - -.. branch: indexing - -Refactor array indexing to support ellipses. - -.. branch: numpy-docstrings - -Allow the docstrings of built-in numpy objects to be set at run-time. - -.. branch: nditer-revisited - -Implement nditer 'buffered' flag and fix some edge cases - -.. branch: ufunc-reduce - -Allow multiple axes in ufunc.reduce() - -.. branch: fix-tinylang-goals - -Update tinylang goals to match current rpython - -.. branch: vmprof-review - -Clean up of vmprof, notably to handle correctly multiple threads diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -2,7 +2,7 @@ from pypy.interpreter.mixedmodule import MixedModule from rpython.rlib import rdynload -VERSION = "1.2.0" +VERSION = "1.2.1" class Module(MixedModule): diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -3427,4 +3427,4 @@ def test_version(): # this test is here mostly for PyPy - assert __version__ == "1.2.0" + assert __version__ == "1.2.1" diff --git a/pypy/module/_vmprof/test/test__vmprof.py b/pypy/module/_vmprof/test/test__vmprof.py --- a/pypy/module/_vmprof/test/test__vmprof.py +++ b/pypy/module/_vmprof/test/test__vmprof.py @@ -21,11 +21,12 @@ i = 0 count = 0 i += 5 * WORD # header - assert s[i] == 4 - i += 1 # marker - assert s[i] == 4 - i += 1 # length - i += len('pypy') + assert s[i ] == 5 # MARKER_HEADER + assert s[i + 1] == 0 # 0 + assert s[i + 2] == 1 # VERSION_THREAD_ID + assert s[i + 3] == 4 # len('pypy') + assert s[i + 4: i + 8] == 'pypy' + i += 8 while i < len(s): if s[i] == 3: break diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -42,7 +42,7 @@ } -""" + open(str(srcdir.join("rvmprof_get_custom_offset.h"))).read()) +""" + open(str(srcdir.join("vmprof_get_custom_offset.h"))).read()) class TestDirect(object): def test_infrastructure(self): diff --git a/pypy/module/cpyext/include/object.h b/pypy/module/cpyext/include/object.h --- a/pypy/module/cpyext/include/object.h +++ b/pypy/module/cpyext/include/object.h @@ -364,6 +364,8 @@ PyObject *ht_name, *ht_slots; } PyHeapTypeObject; +#define PyObject_Bytes PyObject_Str + /* Flag bits for printing: */ #define Py_PRINT_RAW 1 /* No string quotes etc. */ diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_parsing.py @@ -160,6 +160,35 @@ assert func.name == 'sin' assert func.BType == '<func (<double>, <double>), <double>, False>' +def test_remove_line_continuation_comments(): + ffi = FFI(backend=FakeBackend()) + ffi.cdef(""" + double // blah \\ + more comments + x(void); + double // blah\\\\ + y(void); + double // blah\\ \ + etc + z(void); + """) + m = ffi.dlopen(lib_m) + m.x + m.y + m.z + +def test_line_continuation_in_defines(): + ffi = FFI(backend=FakeBackend()) + ffi.cdef(""" + #define ABC\\ + 42 + #define BCD \\ + 43 + """) + m = ffi.dlopen(lib_m) + assert m.ABC == 42 + assert m.BCD == 43 + def test_define_not_supported_for_now(): ffi = FFI(backend=FakeBackend()) e = py.test.raises(CDefError, ffi.cdef, '#define FOO "blah"') @@ -238,6 +267,13 @@ ffi = FFI() ffi.cdef("typedef _Bool bool; void f(bool);") +def test_void_renamed_as_only_arg(): + ffi = FFI() + ffi.cdef("typedef void void_t1;" + "typedef void_t1 void_t;" + "typedef int (*func_t)(void_t);") + assert ffi.typeof("func_t").args == () + def test_win_common_types(): from cffi.commontypes import COMMON_TYPES, _CACHE from cffi.commontypes import win_common_types, resolve_common_type diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -25,6 +25,9 @@ if 1: # test the .cpp mode too kwds.setdefault('source_extension', '.cpp') source = 'extern "C" {\n%s\n}' % (source,) + else: + kwds['extra_compile_args'] = (kwds.get('extra_compile_args', []) + + ['-Werror']) return recompiler._verify(ffi, module_name, source, *args, **kwds) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py @@ -318,15 +318,32 @@ import cffi ffi = cffi.FFI() ffi.set_source("pack3.mymod", "/*code would be here*/") + ffi._hi_there = 42 """) with open("setup.py", "w") as f: - f.write("""if 1: + f.write("from __future__ import print_function\n" + """if 1: from setuptools import setup + from distutils.command.build_ext import build_ext + import os + + class TestBuildExt(build_ext): + def pre_run(self, ext, ffi): + print('_make_setuptools_api: in pre_run:', end=" ") + assert ffi._hi_there == 42 + assert ext.name == "pack3.mymod" + fn = os.path.join(os.path.dirname(self.build_lib), + '..', 'see_me') + print('creating %r' % (fn,)) + open(fn, 'w').close() + setup(name='example1', version='0.1', packages=['pack3'], package_dir={'': 'src1'}, - cffi_modules=["src1/pack3/_build.py:ffi"]) + cffi_modules=["src1/pack3/_build.py:ffi"], + cmdclass={'build_ext': TestBuildExt}, + ) """) @chdir_to_tmp @@ -335,6 +352,7 @@ self.run(["setup.py", "build"]) self.check_produced_files({'setup.py': None, 'build': '?', + 'see_me': None, 'src1': {'pack3': {'__init__.py': None, '_build.py': None}}}) @@ -344,6 +362,7 @@ self.run(["setup.py", "build_ext", "-i"]) self.check_produced_files({'setup.py': None, 'build': '?', + 'see_me': None, 'src1': {'pack3': {'__init__.py': None, '_build.py': None, 'mymod.SO': None}}}) diff --git a/rpython/flowspace/objspace.py b/rpython/flowspace/objspace.py --- a/rpython/flowspace/objspace.py +++ b/rpython/flowspace/objspace.py @@ -13,6 +13,11 @@ def _assert_rpythonic(func): """Raise ValueError if ``func`` is obviously not RPython""" + try: + func.func_code.co_cellvars + except AttributeError: + raise ValueError("%r is not RPython: it is likely an unexpected " + "built-in function or type" % (func,)) if func.func_doc and func.func_doc.lstrip().startswith('NOT_RPYTHON'): raise ValueError("%r is tagged as NOT_RPYTHON" % (func,)) if func.func_code.co_cellvars: diff --git a/rpython/flowspace/test/test_objspace.py b/rpython/flowspace/test/test_objspace.py --- a/rpython/flowspace/test/test_objspace.py +++ b/rpython/flowspace/test/test_objspace.py @@ -1363,6 +1363,15 @@ simplify_graph(graph) assert self.all_operations(graph) == {'bool': 1, 'inplace_add': 1} + def test_unexpected_builtin_function(self): + import itertools + e = py.test.raises(ValueError, build_flow, itertools.permutations) + assert ' is not RPython:' in str(e.value) + e = py.test.raises(ValueError, build_flow, itertools.tee) + assert ' is not RPython:' in str(e.value) + e = py.test.raises(ValueError, build_flow, Exception.__init__) + assert ' is not RPython:' in str(e.value) + DATA = {'x': 5, 'y': 6} diff --git a/rpython/jit/backend/arm/runner.py b/rpython/jit/backend/arm/runner.py --- a/rpython/jit/backend/arm/runner.py +++ b/rpython/jit/backend/arm/runner.py @@ -64,12 +64,6 @@ operations, original_loop_token, log=log) - def clear_latest_values(self, count): - setitem = self.assembler.fail_boxes_ptr.setitem - null = lltype.nullptr(llmemory.GCREF.TO) - for index in range(count): - setitem(index, null) - def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) return CPU_ARM.cast_adr_to_int(adr) diff --git a/rpython/jit/backend/llsupport/regalloc.py b/rpython/jit/backend/llsupport/regalloc.py --- a/rpython/jit/backend/llsupport/regalloc.py +++ b/rpython/jit/backend/llsupport/regalloc.py @@ -636,8 +636,7 @@ assert isinstance(box, Box) loc = self.fm.get_new_loc(box) locs.append(loc.value - base_ofs) - if looptoken.compiled_loop_token is not None: - # for tests + if looptoken.compiled_loop_token is not None: # <- for tests looptoken.compiled_loop_token._ll_initial_locs = locs def can_merge_with_next_guard(self, op, i, operations): diff --git a/rpython/jit/backend/llsupport/src/codemap.c b/rpython/jit/backend/llsupport/src/codemap.c --- a/rpython/jit/backend/llsupport/src/codemap.c +++ b/rpython/jit/backend/llsupport/src/codemap.c @@ -6,9 +6,9 @@ #endif #ifdef RPYTHON_VMPROF -RPY_EXTERN void rpython_vmprof_ignore_signals(int ignored); +RPY_EXTERN void vmprof_ignore_signals(int ignored); static void pypy_codemap_invalid_set(int ignored) { - rpython_vmprof_ignore_signals(ignored); + vmprof_ignore_signals(ignored); } #else static void pypy_codemap_invalid_set(int ignored) { diff --git a/rpython/jit/backend/x86/runner.py b/rpython/jit/backend/x86/runner.py --- a/rpython/jit/backend/x86/runner.py +++ b/rpython/jit/backend/x86/runner.py @@ -100,12 +100,6 @@ return self.assembler.assemble_bridge(faildescr, inputargs, operations, original_loop_token, log, logger) - def clear_latest_values(self, count): - setitem = self.assembler.fail_boxes_ptr.setitem - null = lltype.nullptr(llmemory.GCREF.TO) - for index in range(count): - setitem(index, null) - def cast_ptr_to_int(x): adr = llmemory.cast_ptr_to_adr(x) return CPU386.cast_adr_to_int(adr) diff --git a/rpython/jit/metainterp/test/test_ajit.py b/rpython/jit/metainterp/test/test_ajit.py --- a/rpython/jit/metainterp/test/test_ajit.py +++ b/rpython/jit/metainterp/test/test_ajit.py @@ -268,6 +268,7 @@ y -= 1 return res res = self.meta_interp(f, [6, sys.maxint, 48]) + self.check_trace_count(6) assert res == f(6, sys.maxint, 48) def test_loop_invariant_mul_bridge_ovf2(self): diff --git a/rpython/rlib/jit.py b/rpython/rlib/jit.py --- a/rpython/rlib/jit.py +++ b/rpython/rlib/jit.py @@ -1087,6 +1087,16 @@ """ assert type(value) is cls +def ll_record_exact_class(ll_value, ll_cls): + from rpython.rlib.debug import ll_assert + from rpython.rtyper.lltypesystem.lloperation import llop + from rpython.rtyper.lltypesystem import lltype + from rpython.rtyper.rclass import ll_type + ll_assert(ll_value == lltype.nullptr(lltype.typeOf(ll_value).TO), "record_exact_class called with None argument") + ll_assert(ll_type(ll_value) is ll_cls, "record_exact_class called with invalid arguments") + llop.jit_record_exact_class(lltype.Void, ll_value, ll_cls) + + class Entry(ExtRegistryEntry): _about_ = record_exact_class @@ -1100,12 +1110,10 @@ from rpython.rtyper import rclass classrepr = rclass.get_type_repr(hop.rtyper) - - hop.exception_cannot_occur() v_inst = hop.inputarg(hop.args_r[0], arg=0) v_cls = hop.inputarg(classrepr, arg=1) - return hop.genop('jit_record_exact_class', [v_inst, v_cls], - resulttype=lltype.Void) + hop.exception_is_here() + return hop.gendirectcall(ll_record_exact_class, v_inst, v_cls) def _jit_conditional_call(condition, function, *args): pass diff --git a/rpython/rlib/rvmprof/cintf.py b/rpython/rlib/rvmprof/cintf.py --- a/rpython/rlib/rvmprof/cintf.py +++ b/rpython/rlib/rvmprof/cintf.py @@ -40,24 +40,20 @@ **eci_kwds)) - vmprof_init = rffi.llexternal("rpython_vmprof_init", [rffi.INT], rffi.CCHARP, - compilation_info=eci) - vmprof_enable = rffi.llexternal("rpython_vmprof_enable", [rffi.LONG], rffi.INT, + vmprof_init = rffi.llexternal("vmprof_init", + [rffi.INT, rffi.DOUBLE, rffi.CCHARP], + rffi.CCHARP, compilation_info=eci) + vmprof_enable = rffi.llexternal("vmprof_enable", [], rffi.INT, compilation_info=eci, save_err=rffi.RFFI_SAVE_ERRNO) - vmprof_disable = rffi.llexternal("rpython_vmprof_disable", [], rffi.INT, + vmprof_disable = rffi.llexternal("vmprof_disable", [], rffi.INT, compilation_info=eci, save_err=rffi.RFFI_SAVE_ERRNO) - vmprof_write_buf = rffi.llexternal("rpython_vmprof_write_buf", - [rffi.CCHARP, rffi.LONG], - lltype.Void, compilation_info=eci) - - ## vmprof_register_virtual_function = rffi.llexternal( - ## "vmprof_register_virtual_function", - ## [rffi.CCHARP, rffi.VOIDP, rffi.VOIDP], lltype.Void, - ## compilation_info=eci, _nowrapper=True) - - vmprof_ignore_signals = rffi.llexternal("rpython_vmprof_ignore_signals", + vmprof_register_virtual_function = rffi.llexternal( + "vmprof_register_virtual_function", + [rffi.CCHARP, rffi.LONG, rffi.INT], + rffi.INT, compilation_info=eci) + vmprof_ignore_signals = rffi.llexternal("vmprof_ignore_signals", [rffi.INT], lltype.Void, compilation_info=eci) return CInterface(locals()) @@ -83,10 +79,20 @@ cont_name = 'rpyvmprof_f_%s_%s' % (name, token) tramp_name = 'rpyvmprof_t_%s_%s' % (name, token) + orig_tramp_name = tramp_name func.c_name = cont_name func._dont_inline_ = True + if sys.platform == 'darwin': + # according to internet "At the time UNIX was written in 1974...." + # "... all C functions are prefixed with _" + cont_name = '_' + cont_name + tramp_name = '_' + tramp_name + PLT = "" + else: + PLT = "@PLT" + assert detect_cpu.autodetect().startswith(detect_cpu.MODEL_X86_64), ( "rvmprof only supports x86-64 CPUs for now") @@ -111,17 +117,15 @@ target.write("""\ \t.text \t.globl\t%(tramp_name)s -\t.type\t%(tramp_name)s, @function %(tramp_name)s: \t.cfi_startproc \tpushq\t%(reg)s \t.cfi_def_cfa_offset 16 -\tcall %(cont_name)s@PLT +\tcall %(cont_name)s%(PLT)s \taddq\t$8, %%rsp \t.cfi_def_cfa_offset 8 \tret \t.cfi_endproc -\t.size\t%(tramp_name)s, .-%(tramp_name)s """ % locals()) def tok2cname(tok): @@ -133,7 +137,7 @@ header = 'RPY_EXTERN %s %s(%s);\n' % ( tok2cname(restok), - tramp_name, + orig_tramp_name, ', '.join([tok2cname(tok) for tok in token] + ['long'])) header += """\ @@ -147,7 +151,7 @@ #endif #define VMPROF_ADDR_OF_TRAMPOLINE cmp_%s } -""" % (tramp_name, tramp_name, tramp_name) +""" % (tramp_name, orig_tramp_name, tramp_name) eci = ExternalCompilationInfo( post_include_bits = [header], @@ -155,7 +159,7 @@ ) return rffi.llexternal( - tramp_name, + orig_tramp_name, [token2lltype(tok) for tok in token] + [lltype.Signed], token2lltype(restok), compilation_info=eci, diff --git a/rpython/rlib/rvmprof/rvmprof.py b/rpython/rlib/rvmprof/rvmprof.py --- a/rpython/rlib/rvmprof/rvmprof.py +++ b/rpython/rlib/rvmprof/rvmprof.py @@ -1,14 +1,12 @@ import sys, os from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rlib.rstring import StringBuilder from rpython.rlib import jit, rgc, rposix from rpython.rlib.rvmprof import cintf from rpython.rtyper.annlowlevel import cast_instance_to_gcref from rpython.rtyper.annlowlevel import cast_base_ptr_to_instance from rpython.rtyper.lltypesystem import rffi -MAX_CODES = 8000 - 255 -MAX_FUNC_NAME = 255 +MAX_FUNC_NAME = 1023 # ____________________________________________________________ @@ -34,8 +32,6 @@ def _cleanup_(self): self.is_enabled = False - self.fileno = -1 - self._current_codes = None @specialize.argtype(1) def register_code(self, code, full_name_func): @@ -102,18 +98,13 @@ assert fileno >= 0 if self.is_enabled: raise VMProfError("vmprof is already enabled") - if not (1e-6 <= interval < 1.0): - raise VMProfError("bad value for 'interval'") - interval_usec = int(interval * 1000000.0) - p_error = self.cintf.vmprof_init(fileno) + p_error = self.cintf.vmprof_init(fileno, interval, "pypy") if p_error: raise VMProfError(rffi.charp2str(p_error)) - self.fileno = fileno - self._write_header(interval_usec) self._gather_all_code_objs() - res = self.cintf.vmprof_enable(interval_usec) + res = self.cintf.vmprof_enable() if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) self.is_enabled = True @@ -125,9 +116,6 @@ if not self.is_enabled: raise VMProfError("vmprof is not enabled") self.is_enabled = False - if self._current_codes is not None: - self._flush_codes() - self.fileno = -1 res = self.cintf.vmprof_disable() if res < 0: raise VMProfError(os.strerror(rposix.get_saved_errno())) @@ -136,48 +124,8 @@ assert name.count(':') == 3 and len(name) <= MAX_FUNC_NAME, ( "the name must be 'class:func_name:func_line:filename' " "and at most %d characters; got '%s'" % (MAX_FUNC_NAME, name)) - b = self._current_codes - if b is None: - b = self._current_codes = StringBuilder() - b.append('\x02') - _write_long_to_string_builder(uid, b) - _write_long_to_string_builder(len(name), b) - b.append(name) - if b.getlength() >= MAX_CODES: - self._flush_codes() - - def _flush_codes(self): - buf = self._current_codes.build() - self._current_codes = None - self.cintf.vmprof_write_buf(buf, len(buf)) - # NOTE: keep in mind that vmprof_write_buf() can only write - # a maximum of 8184 bytes. This should be guaranteed here because: - assert MAX_CODES + 17 + MAX_FUNC_NAME <= 8184 - - def _write_header(self, interval_usec): - b = StringBuilder() - _write_long_to_string_builder(0, b) - _write_long_to_string_builder(3, b) - _write_long_to_string_builder(0, b) - _write_long_to_string_builder(interval_usec, b) - _write_long_to_string_builder(0, b) - b.append('\x04') # interp name - b.append(chr(len('pypy'))) - b.append('pypy') - buf = b.build() - self.cintf.vmprof_write_buf(buf, len(buf)) - - -def _write_long_to_string_builder(l, b): - b.append(chr(l & 0xff)) - b.append(chr((l >> 8) & 0xff)) - b.append(chr((l >> 16) & 0xff)) - b.append(chr((l >> 24) & 0xff)) - if sys.maxint > 2147483647: - b.append(chr((l >> 32) & 0xff)) - b.append(chr((l >> 40) & 0xff)) - b.append(chr((l >> 48) & 0xff)) - b.append(chr((l >> 56) & 0xff)) + if self.cintf.vmprof_register_virtual_function(name, uid, 500000) < 0: + raise VMProfError("vmprof buffers full! disk full or too slow") def vmprof_execute_code(name, get_code_fn, result_class=None): diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/rvmprof.c --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/rvmprof.c @@ -1,22 +1,3 @@ -/* VMPROF - * - * statistical sampling profiler specifically designed to profile programs - * which run on a Virtual Machine and/or bytecode interpreter, such as Python, - * etc. - * - * The logic to dump the C stack traces is partly stolen from the code in - * gperftools. - * The file "getpc.h" has been entirely copied from gperftools. - * - * Tested only on gcc, linux, x86_64. - * - * Copyright (C) 2014-2015 - * Antonio Cuni - anto.c...@gmail.com - * Maciej Fijalkowski - fij...@gmail.com - * Armin Rigo - ar...@tunes.org - * - */ - #define _GNU_SOURCE 1 @@ -39,431 +20,4 @@ #endif -#include <dlfcn.h> -#include <assert.h> -#include <pthread.h> -#include <sys/time.h> -#include <errno.h> -#include <unistd.h> -#include <stdio.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include "rvmprof_getpc.h" -#include "rvmprof_unwind.h" -#include "rvmprof_mt.h" - - -/************************************************************/ - -// functions copied from libunwind using dlopen - -static int (*unw_get_reg)(unw_cursor_t*, int, unw_word_t*) = NULL; -static int (*unw_step)(unw_cursor_t*) = NULL; -static int (*unw_init_local)(unw_cursor_t *, unw_context_t *) = NULL; -static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; - -static int profile_file = -1; - - -RPY_EXTERN -char *rpython_vmprof_init(int fd) -{ - if (!unw_get_reg) { - void *libhandle; - - if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) - goto error; - if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) - goto error; - if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))) - goto error; - if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) - goto error; - if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) - goto error; - } - if (prepare_concurrent_bufs() < 0) - return "out of memory"; - - assert(fd >= 0); - profile_file = fd; - return NULL; - - error: - return dlerror(); -} - -/************************************************************/ - -/* value: last bit is 1 if signals must be ignored; all other bits - are a counter for how many threads are currently in a signal handler */ -static long volatile signal_handler_value = 1; - -RPY_EXTERN -void rpython_vmprof_ignore_signals(int ignored) -{ - if (!ignored) { - __sync_fetch_and_and(&signal_handler_value, ~1L); - } - else { - /* set the last bit, and wait until concurrently-running signal - handlers finish */ - while (__sync_or_and_fetch(&signal_handler_value, 1L) != 1L) { - usleep(1); - } - } -} - - -/* ************************************************************* - * functions to write a profile file compatible with gperftools - * ************************************************************* - */ - -#define MAX_FUNC_NAME 128 -#define MAX_STACK_DEPTH \ - ((SINGLE_BUF_SIZE - sizeof(struct prof_stacktrace_s)) / sizeof(void *)) - -#define MARKER_STACKTRACE '\x01' -#define MARKER_VIRTUAL_IP '\x02' -#define MARKER_TRAILER '\x03' - -struct prof_stacktrace_s { - char padding[sizeof(long) - 1]; - char marker; - long count, depth; - void *stack[]; -}; - -static long profile_interval_usec = 0; -static char atfork_hook_installed = 0; - - -/* ****************************************************** - * libunwind workaround for process JIT frames correctly - * ****************************************************** - */ - -#include "rvmprof_get_custom_offset.h" - -typedef struct { - void* _unused1; - void* _unused2; - void* sp; - void* ip; - void* _unused3[sizeof(unw_cursor_t)/sizeof(void*) - 4]; -} vmprof_hacked_unw_cursor_t; - -static int vmprof_unw_step(unw_cursor_t *cp, int first_run) -{ - void* ip; - void* sp; - ptrdiff_t sp_offset; - unw_get_reg (cp, UNW_REG_IP, (unw_word_t*)&ip); - unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp); - if (!first_run) { - // make sure we're pointing to the CALL and not to the first - // instruction after. If the callee adjusts the stack for us - // it's not safe to be at the instruction after - ip -= 1; - } - sp_offset = vmprof_unw_get_custom_offset(ip, cp); - - if (sp_offset == -1) { - // it means that the ip is NOT in JITted code, so we can use the - // stardard unw_step - return unw_step(cp); - } - else { - // this is a horrible hack to manually walk the stack frame, by - // setting the IP and SP in the cursor - vmprof_hacked_unw_cursor_t *cp2 = (vmprof_hacked_unw_cursor_t*)cp; - void* bp = (void*)sp + sp_offset; - cp2->sp = bp; - bp -= sizeof(void*); - cp2->ip = ((void**)bp)[0]; - // the ret is on the top of the stack minus WORD - return 1; - } -} - - -/* ************************************************************* - * functions to dump the stack trace - * ************************************************************* - */ - -static int get_stack_trace(void** result, int max_depth, ucontext_t *ucontext) -{ - void *ip; - int n = 0; - unw_cursor_t cursor; - unw_context_t uc = *ucontext; - - int ret = unw_init_local(&cursor, &uc); - assert(ret >= 0); - (void)ret; - - while (n < max_depth) { - if (unw_get_reg(&cursor, UNW_REG_IP, (unw_word_t *) &ip) < 0) { - break; - } - - unw_proc_info_t pip; - unw_get_proc_info(&cursor, &pip); - - /* if n==0, it means that the signal handler interrupted us while we - were in the trampoline, so we are not executing (yet) the real main - loop function; just skip it */ - if (VMPROF_ADDR_OF_TRAMPOLINE((void*)pip.start_ip) && n > 0) { - // found main loop stack frame - void* sp; - unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); - void *arg_addr = (char*)sp /* + mainloop_sp_offset */; - void **arg_ptr = (void**)arg_addr; - /* if (mainloop_get_virtual_ip) { - ip = mainloop_get_virtual_ip(*arg_ptr); - } else { */ - ip = *arg_ptr; - } - - int first_run = (n == 0); - result[n++] = ip; - n = vmprof_write_header_for_jit_addr(result, n, ip, max_depth); - if (vmprof_unw_step(&cursor, first_run) <= 0) - break; - } - return n; -} - - -/* ************************************************************* - * the signal handler - * ************************************************************* - */ - -static void sigprof_handler(int sig_nr, siginfo_t* info, void *ucontext) -{ - long val = __sync_fetch_and_add(&signal_handler_value, 2L); - - if ((val & 1) == 0) { - int saved_errno = errno; - int fd = profile_file; - assert(fd >= 0); - - struct profbuf_s *p = reserve_buffer(fd); - if (p == NULL) { - /* ignore this signal: there are no free buffers right now */ - } - else { - int depth; - struct prof_stacktrace_s *st = (struct prof_stacktrace_s *)p->data; - st->marker = MARKER_STACKTRACE; - st->count = 1; - st->stack[0] = GetPC((ucontext_t*)ucontext); - depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext); - depth++; // To account for pc value in stack[0]; - st->depth = depth; - p->data_offset = offsetof(struct prof_stacktrace_s, marker); - p->data_size = (depth * sizeof(void *) + - sizeof(struct prof_stacktrace_s) - - offsetof(struct prof_stacktrace_s, marker)); - commit_buffer(fd, p); - } - - errno = saved_errno; - } - - __sync_sub_and_fetch(&signal_handler_value, 2L); -} - - -/* ************************************************************* - * the setup and teardown functions - * ************************************************************* - */ - -static int install_sigprof_handler(void) -{ - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction = sigprof_handler; - sa.sa_flags = SA_RESTART | SA_SIGINFO; - if (sigemptyset(&sa.sa_mask) == -1 || - sigaction(SIGPROF, &sa, NULL) == -1) - return -1; - return 0; -} - -static int remove_sigprof_handler(void) -{ - if (signal(SIGPROF, SIG_DFL) == SIG_ERR) - return -1; - return 0; -} - -static int install_sigprof_timer(void) -{ - static struct itimerval timer; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = profile_interval_usec; - timer.it_value = timer.it_interval; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) - return -1; - return 0; -} - -static int remove_sigprof_timer(void) { - static struct itimerval timer; - timer.it_interval.tv_sec = 0; - timer.it_interval.tv_usec = 0; - timer.it_value.tv_sec = 0; - timer.it_value.tv_usec = 0; - if (setitimer(ITIMER_PROF, &timer, NULL) != 0) - return -1; - return 0; -} - -static void atfork_disable_timer(void) { - if (profile_interval_usec > 0) { - remove_sigprof_timer(); - } -} - -static void atfork_enable_timer(void) { - if (profile_interval_usec > 0) { - install_sigprof_timer(); - } -} - -static int install_pthread_atfork_hooks(void) { - /* this is needed to prevent the problems described there: - - http://code.google.com/p/gperftools/issues/detail?id=278 - - http://lists.debian.org/debian-glibc/2010/03/msg00161.html - - TL;DR: if the RSS of the process is large enough, the clone() syscall - will be interrupted by the SIGPROF before it can complete, then - retried, interrupted again and so on, in an endless loop. The - solution is to disable the timer around the fork, and re-enable it - only inside the parent. - */ - if (atfork_hook_installed) - return 0; - int ret = pthread_atfork(atfork_disable_timer, atfork_enable_timer, NULL); - if (ret != 0) - return -1; - atfork_hook_installed = 1; - return 0; -} - -RPY_EXTERN -int rpython_vmprof_enable(long interval_usec) -{ - assert(profile_file >= 0); - assert(interval_usec > 0); - profile_interval_usec = interval_usec; - - if (install_pthread_atfork_hooks() == -1) - goto error; - if (install_sigprof_handler() == -1) - goto error; - if (install_sigprof_timer() == -1) - goto error; - rpython_vmprof_ignore_signals(0); - return 0; - - error: - profile_file = -1; - profile_interval_usec = 0; - return -1; -} - -static int _write_all(const void *buf, size_t bufsize) -{ - while (bufsize > 0) { - ssize_t count = write(profile_file, buf, bufsize); - if (count <= 0) - return -1; /* failed */ - buf += count; - bufsize -= count; - } - return 0; -} - -static int close_profile(void) -{ - char buf[4096]; - ssize_t size; - unsigned char marker = MARKER_TRAILER; - - if (_write_all(&marker, 1) < 0) - return -1; - -#ifdef __linux__ - // copy /proc/self/maps to the end of the profile file - int srcfd = open("/proc/self/maps", O_RDONLY); - if (srcfd < 0) - return -1; - - while ((size = read(srcfd, buf, sizeof buf)) > 0) { - if (_write_all(buf, size) < 0) { - close(srcfd); - return -1; - } - } - close(srcfd); -#else - // freebsd and mac - sprintf(buf, "procstat -v %d", getpid()); - FILE *srcf = popen(buf, "r"); - if (!srcf) - return -1; - - while ((size = fread(buf, 1, sizeof buf, src))) { - if (_write_all(buf, size) < 0) { - pclose(srcf); - return -1; - } - } - pclose(srcf); -#endif - - /* don't close() the file descriptor from here */ - profile_file = -1; - return 0; -} - -RPY_EXTERN -int rpython_vmprof_disable(void) -{ - rpython_vmprof_ignore_signals(1); - profile_interval_usec = 0; - - if (remove_sigprof_timer() == -1) - return -1; - if (remove_sigprof_handler() == -1) - return -1; - if (shutdown_concurrent_bufs(profile_file) < 0) - return -1; - return close_profile(); -} - -RPY_EXTERN -void rpython_vmprof_write_buf(char *buf, long size) -{ - struct profbuf_s *p; - - while ((p = reserve_buffer(profile_file)) == NULL) { - /* spin loop waiting for a buffer to be ready; should almost never - be the case */ - usleep(1); - } - - if (size > SINGLE_BUF_SIZE) - size = SINGLE_BUF_SIZE; - memcpy(p->data, buf, size); - p->data_size = size; - - commit_buffer(profile_file, p); -} +#include "vmprof_main.h" diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,6 +1,6 @@ -RPY_EXTERN char *rpython_vmprof_init(int); -RPY_EXTERN void rpython_vmprof_ignore_signals(int); -RPY_EXTERN int rpython_vmprof_enable(long); -RPY_EXTERN int rpython_vmprof_disable(void); -RPY_EXTERN void rpython_vmprof_write_buf(char *, long); +RPY_EXTERN char *vmprof_init(int, double, char *); +RPY_EXTERN void vmprof_ignore_signals(int); +RPY_EXTERN int vmprof_enable(void); +RPY_EXTERN int vmprof_disable(void); +RPY_EXTERN int vmprof_register_virtual_function(char *, long, int); diff --git a/rpython/rlib/rvmprof/src/rvmprof_config.h b/rpython/rlib/rvmprof/src/vmprof_config.h rename from rpython/rlib/rvmprof/src/rvmprof_config.h rename to rpython/rlib/rvmprof/src/vmprof_config.h diff --git a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h rename from rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h rename to rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h --- a/rpython/rlib/rvmprof/src/rvmprof_get_custom_offset.h +++ b/rpython/rlib/rvmprof/src/vmprof_get_custom_offset.h @@ -7,12 +7,69 @@ #endif +#ifdef CPYTHON_GET_CUSTOM_OFFSET +static void *tramp_start, *tramp_end; +#endif + + static ptrdiff_t vmprof_unw_get_custom_offset(void* ip, void *cp) { -#ifdef PYPY_JIT_CODEMAP + +#if defined(PYPY_JIT_CODEMAP) + intptr_t ip_l = (intptr_t)ip; return pypy_jit_stack_depth_at_loc(ip_l); + +#elif defined(CPYTHON_GET_CUSTOM_OFFSET) + + if (ip >= tramp_start && ip <= tramp_end) { + // XXX the return value is wrong for all the places before push and + // after pop, fix + void *bp; + void *sp; + + /* This is a stage2 trampoline created by hotpatch: + + push %rbx + push %rbp + mov %rsp,%rbp + and $0xfffffffffffffff0,%rsp // make sure the stack is aligned + movabs $0x7ffff687bb10,%rbx + callq *%rbx + leaveq + pop %rbx + retq + + the stack layout is like this: + + +-----------+ high addresses + | ret addr | + +-----------+ + | saved rbx | start of the function frame + +-----------+ + | saved rbp | + +-----------+ + | ........ | <-- rbp + +-----------+ low addresses + + So, the trampoline frame starts at rbp+16, and the return address, + is at rbp+24. The vmprof API requires us to return the offset of + the frame relative to sp, hence we have this weird computation. + + XXX (antocuni): I think we could change the API to return directly + the frame address instead of the offset; however, this require a + change in the PyPy code too + */ + + unw_get_reg (cp, UNW_REG_SP, (unw_word_t*)&sp); + unw_get_reg (cp, UNW_X86_64_RBP, (unw_word_t*)&bp); + return bp+16+8-sp; + } + return -1; + #else + return -1; + #endif } diff --git a/rpython/rlib/rvmprof/src/rvmprof_getpc.h b/rpython/rlib/rvmprof/src/vmprof_getpc.h rename from rpython/rlib/rvmprof/src/rvmprof_getpc.h rename to rpython/rlib/rvmprof/src/vmprof_getpc.h --- a/rpython/rlib/rvmprof/src/rvmprof_getpc.h +++ b/rpython/rlib/rvmprof/src/vmprof_getpc.h @@ -44,7 +44,7 @@ #ifndef BASE_GETPC_H_ #define BASE_GETPC_H_ -#include "rvmprof_config.h" +#include "vmprof_config.h" // On many linux systems, we may need _GNU_SOURCE to get access to // the defined constants that define the register we want to see (eg @@ -53,7 +53,9 @@ // If #define _GNU_SOURCE causes problems, this might work instead. // It will cause problems for FreeBSD though!, because it turns off // the needed __BSD_VISIBLE. -//#define _XOPEN_SOURCE 500 +#ifdef __APPLE__ +#define _XOPEN_SOURCE 500 +#endif #include <string.h> // for memcmp #if defined(HAVE_SYS_UCONTEXT_H) @@ -179,7 +181,11 @@ // configure.ac (or set it manually in your config.h). #else void* GetPC(ucontext_t *signal_ucontext) { +#ifdef __APPLE__ + return (void*)(signal_ucontext->uc_mcontext->__ss.__rip); +#else return (void*)signal_ucontext->PC_FROM_UCONTEXT; // defined in config.h +#endif } #endif diff --git a/rpython/rlib/rvmprof/src/rvmprof.c b/rpython/rlib/rvmprof/src/vmprof_main.h copy from rpython/rlib/rvmprof/src/rvmprof.c copy to rpython/rlib/rvmprof/src/vmprof_main.h --- a/rpython/rlib/rvmprof/src/rvmprof.c +++ b/rpython/rlib/rvmprof/src/vmprof_main.h @@ -19,26 +19,6 @@ #define _GNU_SOURCE 1 - -#ifdef RPYTHON_LL2CTYPES - /* only for testing: ll2ctypes sets RPY_EXTERN from the command-line */ -# ifndef RPY_EXTERN -# define RPY_EXTERN RPY_EXPORTED -# endif -# define RPY_EXPORTED extern __attribute__((visibility("default"))) -# define VMPROF_ADDR_OF_TRAMPOLINE(addr) 0 - -#else - -# include "common_header.h" -# include "rvmprof.h" -# ifndef VMPROF_ADDR_OF_TRAMPOLINE -# error "RPython program using rvmprof, but not calling vmprof_execute_code()" -# endif - -#endif - - #include <dlfcn.h> #include <assert.h> #include <pthread.h> @@ -47,11 +27,12 @@ #include <unistd.h> #include <stdio.h> #include <sys/types.h> +#include <signal.h> #include <sys/stat.h> #include <fcntl.h> -#include "rvmprof_getpc.h" -#include "rvmprof_unwind.h" -#include "rvmprof_mt.h" +#include "vmprof_getpc.h" +#include "vmprof_unwind.h" +#include "vmprof_mt.h" /************************************************************/ @@ -64,23 +45,40 @@ static int (*unw_get_proc_info)(unw_cursor_t *, unw_proc_info_t *) = NULL; static int profile_file = -1; +static long prepare_interval_usec; +static struct profbuf_s *volatile current_codes; +static void *(*mainloop_get_virtual_ip)(char *) = 0; +static int opened_profile(char *interp_name); +static void flush_codes(void); + +#ifdef __APPLE__ +#define UNWIND_NAME "/usr/lib/system/libunwind.dylib" +#define UNW_PREFIX "unw" +#else +#define UNWIND_NAME "libunwind.so" +#define UNW_PREFIX "_ULx86_64" +#endif RPY_EXTERN -char *rpython_vmprof_init(int fd) +char *vmprof_init(int fd, double interval, char *interp_name) { + if (interval < 1e-6 || interval >= 1.0) + return "bad value for 'interval'"; + prepare_interval_usec = (int)(interval * 1000000.0); + if (!unw_get_reg) { void *libhandle; - if (!(libhandle = dlopen("libunwind.so", RTLD_LAZY | RTLD_LOCAL))) + if (!(libhandle = dlopen(UNWIND_NAME, RTLD_LAZY | RTLD_LOCAL))) goto error; - if (!(unw_get_reg = dlsym(libhandle, "_ULx86_64_get_reg"))) + if (!(unw_get_reg = dlsym(libhandle, UNW_PREFIX "_get_reg"))) goto error; - if (!(unw_get_proc_info = dlsym(libhandle, "_ULx86_64_get_proc_info"))) + if (!(unw_get_proc_info = dlsym(libhandle, UNW_PREFIX "_get_proc_info"))) goto error; - if (!(unw_init_local = dlsym(libhandle, "_ULx86_64_init_local"))) + if (!(unw_init_local = dlsym(libhandle, UNW_PREFIX "_init_local"))) goto error; - if (!(unw_step = dlsym(libhandle, "_ULx86_64_step"))) + if (!(unw_step = dlsym(libhandle, UNW_PREFIX "_step"))) goto error; } if (prepare_concurrent_bufs() < 0) @@ -88,6 +86,10 @@ assert(fd >= 0); profile_file = fd; + if (opened_profile(interp_name) < 0) { + profile_file = -1; + return strerror(errno); + } return NULL; error: @@ -101,7 +103,7 @@ static long volatile signal_handler_value = 1; RPY_EXTERN -void rpython_vmprof_ignore_signals(int ignored) +void vmprof_ignore_signals(int ignored) { if (!ignored) { __sync_fetch_and_and(&signal_handler_value, ~1L); @@ -128,6 +130,11 @@ #define MARKER_STACKTRACE '\x01' #define MARKER_VIRTUAL_IP '\x02' #define MARKER_TRAILER '\x03' +#define MARKER_INTERP_NAME '\x04' /* deprecated */ +#define MARKER_HEADER '\x05' + +#define VERSION_BASE '\x00' +#define VERSION_THREAD_ID '\x01' struct prof_stacktrace_s { char padding[sizeof(long) - 1]; @@ -145,7 +152,7 @@ * ****************************************************** */ -#include "rvmprof_get_custom_offset.h" +#include "vmprof_get_custom_offset.h" typedef struct { void* _unused1; @@ -220,12 +227,10 @@ // found main loop stack frame void* sp; unw_get_reg(&cursor, UNW_REG_SP, (unw_word_t *) &sp); - void *arg_addr = (char*)sp /* + mainloop_sp_offset */; - void **arg_ptr = (void**)arg_addr; - /* if (mainloop_get_virtual_ip) { - ip = mainloop_get_virtual_ip(*arg_ptr); - } else { */ - ip = *arg_ptr; + if (mainloop_get_virtual_ip) + ip = mainloop_get_virtual_ip((char *)sp); + else + ip = *(void **)sp; } int first_run = (n == 0); @@ -237,6 +242,23 @@ return n; } +static void *get_current_thread_id(void) +{ + /* xxx This function is a hack on two fronts: + + - It assumes that pthread_self() is async-signal-safe. This + should be true on Linux. I hope it is also true elsewhere. + + - It abuses pthread_self() by assuming it just returns an + integer. According to comments in CPython's source code, the + platforms where it is not the case are rare nowadays. + + An alternative would be to try to look if the information is + available in the ucontext_t in the caller. + */ + return (void *)pthread_self(); +} + /* ************************************************************* * the signal handler @@ -262,9 +284,10 @@ st->marker = MARKER_STACKTRACE; st->count = 1; st->stack[0] = GetPC((ucontext_t*)ucontext); - depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-1, ucontext); + depth = get_stack_trace(st->stack+1, MAX_STACK_DEPTH-2, ucontext); depth++; // To account for pc value in stack[0]; st->depth = depth; + st->stack[depth++] = get_current_thread_id(); p->data_offset = offsetof(struct prof_stacktrace_s, marker); p->data_size = (depth * sizeof(void *) + sizeof(struct prof_stacktrace_s) - @@ -358,11 +381,11 @@ } RPY_EXTERN -int rpython_vmprof_enable(long interval_usec) +int vmprof_enable(void) { assert(profile_file >= 0); - assert(interval_usec > 0); - profile_interval_usec = interval_usec; + assert(prepare_interval_usec > 0); + profile_interval_usec = prepare_interval_usec; if (install_pthread_atfork_hooks() == -1) goto error; @@ -370,7 +393,7 @@ goto error; if (install_sigprof_timer() == -1) goto error; - rpython_vmprof_ignore_signals(0); + vmprof_ignore_signals(0); return 0; error: @@ -391,6 +414,29 @@ return 0; } +static int opened_profile(char *interp_name) +{ + struct { + long hdr[5]; + char interp_name[259]; + } header; + + size_t namelen = strnlen(interp_name, 255); + current_codes = NULL; + + header.hdr[0] = 0; + header.hdr[1] = 3; + header.hdr[2] = 0; + header.hdr[3] = prepare_interval_usec; + header.hdr[4] = 0; + header.interp_name[0] = MARKER_HEADER; + header.interp_name[1] = '\x00'; + header.interp_name[2] = VERSION_THREAD_ID; + header.interp_name[3] = namelen; + memcpy(&header.interp_name[4], interp_name, namelen); + return _write_all(&header, 5 * sizeof(long) + 4 + namelen); +} + static int close_profile(void) { char buf[4096]; @@ -415,12 +461,16 @@ close(srcfd); #else // freebsd and mac +#if defined(__APPLE__) + sprintf(buf, "vmmap %d", getpid()); +#else sprintf(buf, "procstat -v %d", getpid()); +#endif FILE *srcf = popen(buf, "r"); if (!srcf) return -1; - while ((size = fread(buf, 1, sizeof buf, src))) { + while ((size = fread(buf, 1, sizeof buf, srcf))) { if (_write_all(buf, size) < 0) { pclose(srcf); return -1; @@ -435,35 +485,85 @@ } RPY_EXTERN -int rpython_vmprof_disable(void) +int vmprof_disable(void) { - rpython_vmprof_ignore_signals(1); + vmprof_ignore_signals(1); profile_interval_usec = 0; if (remove_sigprof_timer() == -1) return -1; if (remove_sigprof_handler() == -1) return -1; + flush_codes(); if (shutdown_concurrent_bufs(profile_file) < 0) return -1; return close_profile(); } RPY_EXTERN -void rpython_vmprof_write_buf(char *buf, long size) +int vmprof_register_virtual_function(char *code_name, long code_uid, + int auto_retry) { + long namelen = strnlen(code_name, 1023); + long blocklen = 1 + 2 * sizeof(long) + namelen; struct profbuf_s *p; + char *t; - while ((p = reserve_buffer(profile_file)) == NULL) { - /* spin loop waiting for a buffer to be ready; should almost never - be the case */ - usleep(1); + retry: + p = current_codes; + if (p != NULL) { + if (__sync_bool_compare_and_swap(¤t_codes, p, NULL)) { + /* grabbed 'current_codes': we will append the current block + to it if it contains enough room */ + size_t freesize = SINGLE_BUF_SIZE - p->data_size; + if (freesize < blocklen) { + /* full: flush it */ + commit_buffer(profile_file, p); + p = NULL; + } + } + else { + /* compare-and-swap failed, don't try again */ + p = NULL; + } } - if (size > SINGLE_BUF_SIZE) - size = SINGLE_BUF_SIZE; - memcpy(p->data, buf, size); - p->data_size = size; + if (p == NULL) { + p = reserve_buffer(profile_file); + if (p == NULL) { + /* can't get a free block; should almost never be the + case. Spin loop if allowed, or return a failure code + if not (e.g. we're in a signal handler) */ + if (auto_retry > 0) { + auto_retry--; + usleep(1); + goto retry; + } + return -1; + } + } - commit_buffer(profile_file, p); + t = p->data + p->data_size; + p->data_size += blocklen; + assert(p->data_size <= SINGLE_BUF_SIZE); + *t++ = MARKER_VIRTUAL_IP; + memcpy(t, &code_uid, sizeof(long)); t += sizeof(long); + memcpy(t, &namelen, sizeof(long)); t += sizeof(long); + memcpy(t, code_name, namelen); + + /* try to reattach 'p' to 'current_codes' */ + if (!__sync_bool_compare_and_swap(¤t_codes, NULL, p)) { + /* failed, flush it */ + commit_buffer(profile_file, p); + } + return 0; } + +static void flush_codes(void) +{ + struct profbuf_s *p = current_codes; + if (p != NULL) { + current_codes = NULL; + commit_buffer(profile_file, p); + } +} diff --git a/rpython/rlib/rvmprof/src/rvmprof_mt.h b/rpython/rlib/rvmprof/src/vmprof_mt.h rename from rpython/rlib/rvmprof/src/rvmprof_mt.h rename to rpython/rlib/rvmprof/src/vmprof_mt.h --- a/rpython/rlib/rvmprof/src/rvmprof_mt.h +++ b/rpython/rlib/rvmprof/src/vmprof_mt.h @@ -66,7 +66,7 @@ unprepare_concurrent_bufs(); profbuf_all_buffers = mmap(NULL, sizeof(struct profbuf_s) * MAX_NUM_BUFFERS, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, + MAP_PRIVATE | MAP_ANON, -1, 0); if (profbuf_all_buffers == MAP_FAILED) { profbuf_all_buffers = NULL; @@ -190,10 +190,17 @@ } } +static void cancel_buffer(struct profbuf_s *buf) +{ + long i = buf - profbuf_all_buffers; + assert(profbuf_state[i] == PROFBUF_FILLING); + profbuf_state[i] = PROFBUF_UNUSED; +} + static int shutdown_concurrent_bufs(int fd) { /* no signal handler can be running concurrently here, because we - already did rpython_vmprof_ignore_signals(1) */ + already did vmprof_ignore_signals(1) */ assert(profbuf_write_lock == 0); profbuf_write_lock = 2; diff --git a/rpython/rlib/rvmprof/src/rvmprof_unwind.h b/rpython/rlib/rvmprof/src/vmprof_unwind.h rename from rpython/rlib/rvmprof/src/rvmprof_unwind.h rename to rpython/rlib/rvmprof/src/vmprof_unwind.h diff --git a/rpython/rlib/rvmprof/test/test_ztranslation.py b/rpython/rlib/rvmprof/test/test_ztranslation.py --- a/rpython/rlib/rvmprof/test/test_ztranslation.py +++ b/rpython/rlib/rvmprof/test/test_ztranslation.py @@ -1,4 +1,7 @@ -import time, os, sys, py +import time, os, sys +if __name__ == '__main__': + sys.path += ['../../../..'] # for subprocess in test_interpreted +import py from rpython.tool.udir import udir from rpython.rlib import rvmprof from rpython.translator.c.test.test_genc import compile @@ -57,7 +60,8 @@ def test_interpreted(): # takes forever if the Python process is already big... import subprocess - subprocess.check_call([sys.executable, __file__]) + subprocess.check_call([sys.executable, os.path.basename(__file__)], + cwd=(os.path.dirname(__file__) or '.')) def test_compiled(): fn = compile(main, [], gcpolicy="minimark") diff --git a/rpython/rtyper/rclass.py b/rpython/rtyper/rclass.py --- a/rpython/rtyper/rclass.py +++ b/rpython/rtyper/rclass.py @@ -616,26 +616,33 @@ while rbase.classdef is not None: immutable_fields.update(rbase.immutable_field_set) rbase = rbase.rbase - self._parse_field_list(immutable_fields, accessor) + self._parse_field_list(immutable_fields, accessor, hints) - def _parse_field_list(self, fields, accessor): + def _parse_field_list(self, fields, accessor, hints): ranking = {} for name in fields: + quasi = False if name.endswith('?[*]'): # a quasi-immutable field pointing to name = name[:-4] # an immutable array rank = IR_QUASIIMMUTABLE_ARRAY + quasi = True elif name.endswith('[*]'): # for virtualizables' lists name = name[:-3] rank = IR_IMMUTABLE_ARRAY elif name.endswith('?'): # a quasi-immutable field name = name[:-1] rank = IR_QUASIIMMUTABLE + quasi = True else: # a regular immutable/green field rank = IR_IMMUTABLE try: mangled_name, r = self._get_field(name) except KeyError: continue + if quasi and hints.get("immutable"): + raise TyperError( + "can't have _immutable_ = True and a quasi-immutable field " + "%s in class %s" % (name, self.classdef)) ranking[mangled_name] = rank accessor.initialize(self.object_type, ranking) return ranking diff --git a/rpython/rtyper/rvirtualizable.py b/rpython/rtyper/rvirtualizable.py --- a/rpython/rtyper/rvirtualizable.py +++ b/rpython/rtyper/rvirtualizable.py @@ -38,8 +38,8 @@ else: self._super()._setup_repr(hints = hints) c_vfields = self.classdef.classdesc.classdict['_virtualizable_'] - self.my_redirected_fields = self._parse_field_list(c_vfields.value, - self.accessor) + self.my_redirected_fields = self._parse_field_list( + c_vfields.value, self.accessor, hints) else: self._super()._setup_repr() # ootype needs my_redirected_fields even for subclass. lltype does diff --git a/rpython/rtyper/test/test_rclass.py b/rpython/rtyper/test/test_rclass.py --- a/rpython/rtyper/test/test_rclass.py +++ b/rpython/rtyper/test/test_rclass.py @@ -943,6 +943,19 @@ found.append(op.args[1].value) assert found == ['mutate_a', 'mutate_a', 'mutate_b'] + def test_quasi_immutable_clashes_with_immutable(self): + from rpython.jit.metainterp.typesystem import deref + class A(object): + _immutable_ = True + _immutable_fields_ = ['a?'] + def f(): + a = A() + a.x = 42 + a.a = 142 + return A() + with py.test.raises(TyperError): + self.gengraph(f, []) + def test_quasi_immutable_array(self): from rpython.jit.metainterp.typesystem import deref class A(object): diff --git a/rpython/translator/platform/posix.py b/rpython/translator/platform/posix.py --- a/rpython/translator/platform/posix.py +++ b/rpython/translator/platform/posix.py @@ -181,6 +181,7 @@ ('all', '$(DEFAULT_TARGET)', []), ('$(TARGET)', '$(OBJECTS)', '$(CC_LINK) $(LDFLAGSEXTRA) -o $@ $(OBJECTS) $(LIBDIRS) $(LIBS) $(LINKFILES) $(LDFLAGS)'), ('%.o', '%.c', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), + ('%.o', '%.s', '$(CC) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), ('%.o', '%.cxx', '$(CXX) $(CFLAGS) $(CFLAGSEXTRA) -o $@ -c $< $(INCLUDEDIRS)'), ] _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
