Author: Matti Picus <matti.pi...@gmail.com> Branch: buffer-interface2 Changeset: r87582:ed7457dceb76 Date: 2016-10-04 20:33 +0300 http://bitbucket.org/pypy/pypy/changeset/ed7457dceb76/
Log: merge default into branch diff too long, truncating to 2000 out of 3661 lines diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -43,6 +43,7 @@ try: if detect_cpu.autodetect().startswith('x86'): working_modules.add('_vmprof') + working_modules.add('faulthandler') except detect_cpu.ProcessorAutodetectError: pass @@ -89,6 +90,7 @@ ('objspace.usemodules.thread', True)], 'cpyext': [('objspace.usemodules.array', True)], 'cppyy': [('objspace.usemodules.cpyext', True)], + 'faulthandler': [('objspace.usemodules._vmprof', True)], } module_suggests = { # the reason you want _rawffi is for ctypes, which @@ -114,7 +116,8 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], - "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "_vmprof" : ["pypy.module._vmprof.interp_vmprof"], + "faulthandler" : ["pypy.module._vmprof.interp_vmprof"], } def get_module_validator(modname): diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -33,7 +33,7 @@ from somewhere else. You no longer have to do the same with the ``pypy`` executable, as long as it finds its ``libpypy-c.so`` library. -.. branch: _warning +.. branch: _warnings CPython allows warning.warn(('something', 1), Warning), on PyPy this produced a "expected a readable buffer object" error. Test and fix. @@ -42,3 +42,16 @@ CPython rejects 'a'.strip(buffer(' ')); only None, str or unicode are allowed as arguments. Test and fix for str and unicode + +.. branch: faulthandler + +Port the 'faulthandler' module to PyPy default. This module is standard +in Python 3.3 but can also be installed from CPython >= 2.6 from PyPI. + +.. branch: test-cpyext + +Refactor cpyext testing to be more pypy3-friendly. + +.. branch: better-error-missing-self + +Improve the error message when the user forgot the "self" argument of a method. diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -33,6 +33,7 @@ --info : print translation information about this PyPy executable -X track-resources : track the creation of files and sockets and display a warning if they are not closed explicitly +-X faulthandler : attempt to display tracebacks when PyPy crashes """ # Missing vs CPython: PYTHONHOME, PYTHONCASEOK USAGE2 = """ @@ -233,12 +234,22 @@ import pypyjit pypyjit.set_param(jitparam) +def run_faulthandler(): + if 'faulthandler' in sys.builtin_module_names: + import faulthandler + try: + faulthandler.enable(2) # manually set to stderr + except ValueError: + pass # ignore "2 is not a valid file descriptor" + def set_runtime_options(options, Xparam, *args): if Xparam == 'track-resources': sys.pypy_set_track_resources(True) + elif Xparam == 'faulthandler': + run_faulthandler() else: print >> sys.stderr, 'usage: %s -X [options]' % (get_sys_executable(),) - print >> sys.stderr, '[options] can be: track-resources' + print >> sys.stderr, '[options] can be: track-resources, faulthandler' raise SystemExit class CommandLineError(Exception): @@ -527,6 +538,9 @@ print >> sys.stderr, ( "Warning: pypy does not implement py3k warnings") + if os.getenv('PYTHONFAULTHANDLER'): + run_faulthandler() + ## if not we_are_translated(): ## for key in sorted(options): ## print '%40s: %s' % (key, options[key]) diff --git a/pypy/interpreter/argument.py b/pypy/interpreter/argument.py --- a/pypy/interpreter/argument.py +++ b/pypy/interpreter/argument.py @@ -21,7 +21,8 @@ ### Construction ### def __init__(self, space, args_w, keywords=None, keywords_w=None, - w_stararg=None, w_starstararg=None, keyword_names_w=None): + w_stararg=None, w_starstararg=None, keyword_names_w=None, + methodcall=False): self.space = space assert isinstance(args_w, list) self.arguments_w = args_w @@ -41,6 +42,9 @@ # a flag that specifies whether the JIT can unroll loops that operate # on the keywords self._jit_few_keywords = self.keywords is None or jit.isconstant(len(self.keywords)) + # a flag whether this is likely a method call, which doesn't change the + # behaviour but produces better error messages + self.methodcall = methodcall def __repr__(self): """ NOT_RPYTHON """ @@ -207,7 +211,7 @@ starargs_w = [] scope_w[co_argcount] = self.space.newtuple(starargs_w) elif avail > co_argcount: - raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0) + raise self.argerrcount(avail, num_kwds, signature, defaults_w, 0) # if a **kwargs argument is needed, create the dict w_kwds = None @@ -241,7 +245,7 @@ kwds_mapping, self.keyword_names_w, self._jit_few_keywords) else: if co_argcount == 0: - raise ArgErrCount(avail, num_kwds, signature, defaults_w, 0) + raise self.argerrcount(avail, num_kwds, signature, defaults_w, 0) raise ArgErrUnknownKwds(self.space, num_remainingkwds, keywords, kwds_mapping, self.keyword_names_w) @@ -265,9 +269,12 @@ else: missing += 1 if missing: - raise ArgErrCount(avail, num_kwds, signature, defaults_w, missing) + raise self.argerrcount(avail, num_kwds, signature, defaults_w, missing) - + def argerrcount(self, *args): + if self.methodcall: + return ArgErrCountMethod(*args) + return ArgErrCount(*args) def parse_into_scope(self, w_firstarg, scope_w, fnname, signature, defaults_w=None): @@ -478,6 +485,22 @@ num_args) return msg +class ArgErrCountMethod(ArgErrCount): + """ A subclass of ArgErrCount that is used if the argument matching is done + as part of a method call, in which case more information is added to the + error message, if the cause of the error is likely a forgotten `self` + argument. + """ + + def getmsg(self): + msg = ArgErrCount.getmsg(self) + n = self.signature.num_argnames() + if (self.num_args == n + 1 and + (n == 0 or self.signature.argnames[0] != "self")): + msg += ". Did you forget 'self' in the function definition?" + return msg + + class ArgErrMultipleValues(ArgErr): def __init__(self, argname): diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -1127,7 +1127,8 @@ args = Arguments(self, list(args_w)) return self.call_args(w_func, args) - def call_valuestack(self, w_func, nargs, frame): + def call_valuestack(self, w_func, nargs, frame, methodcall=False): + # methodcall is only used for better error messages in argument.py from pypy.interpreter.function import Function, Method, is_builtin_code if frame.get_is_being_profiled() and is_builtin_code(w_func): # XXX: this code is copied&pasted :-( from the slow path below @@ -1144,13 +1145,15 @@ # reuse callable stack place for w_inst frame.settopvalue(w_inst, nargs) nargs += 1 + methodcall = True elif nargs > 0 and ( self.abstract_isinstance_w(frame.peekvalue(nargs-1), # :-( w_func.w_class)): w_func = w_func.w_function if isinstance(w_func, Function): - return w_func.funccall_valuestack(nargs, frame) + return w_func.funccall_valuestack( + nargs, frame, methodcall=methodcall) # end of hack for performance args = frame.make_arguments(nargs) diff --git a/pypy/interpreter/function.py b/pypy/interpreter/function.py --- a/pypy/interpreter/function.py +++ b/pypy/interpreter/function.py @@ -117,7 +117,8 @@ list(args_w[1:]))) return self.call_args(Arguments(self.space, list(args_w))) - def funccall_valuestack(self, nargs, frame): # speed hack + def funccall_valuestack(self, nargs, frame, methodcall=False): # speed hack + # methodcall is only for better error messages from pypy.interpreter import gateway from pypy.interpreter.pycode import PyCode @@ -164,7 +165,7 @@ args = frame.make_arguments(nargs-1) return code.funcrun_obj(self, w_obj, args) - args = frame.make_arguments(nargs) + args = frame.make_arguments(nargs, methodcall=methodcall) return self.call_args(args) @jit.unroll_safe diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -403,11 +403,14 @@ depth -= 1 self.valuestackdepth = finaldepth - def make_arguments(self, nargs): - return Arguments(self.space, self.peekvalues(nargs)) + def make_arguments(self, nargs, methodcall=False): + return Arguments( + self.space, self.peekvalues(nargs), methodcall=methodcall) - def argument_factory(self, arguments, keywords, keywords_w, w_star, w_starstar): - return Arguments(self.space, arguments, keywords, keywords_w, w_star, w_starstar) + def argument_factory(self, arguments, keywords, keywords_w, w_star, w_starstar, methodcall=False): + return Arguments( + self.space, arguments, keywords, keywords_w, w_star, + w_starstar, methodcall=methodcall) @jit.dont_look_inside def descr__reduce__(self, space): diff --git a/pypy/interpreter/test/test_argument.py b/pypy/interpreter/test/test_argument.py --- a/pypy/interpreter/test/test_argument.py +++ b/pypy/interpreter/test/test_argument.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import py from pypy.interpreter.argument import (Arguments, ArgErr, ArgErrUnknownKwds, - ArgErrMultipleValues, ArgErrCount) + ArgErrMultipleValues, ArgErrCount, ArgErrCountMethod) from pypy.interpreter.signature import Signature from pypy.interpreter.error import OperationError @@ -573,6 +573,10 @@ s = err.getmsg() assert s == "takes exactly 1 argument (0 given)" + sig = Signature(['self', 'b'], None, None) + err = ArgErrCount(3, 0, sig, [], 0) + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" sig = Signature(['a', 'b'], None, None) err = ArgErrCount(3, 0, sig, [], 0) s = err.getmsg() @@ -607,6 +611,57 @@ s = err.getmsg() assert s == "takes at most 1 non-keyword argument (2 given)" + def test_missing_args_method(self): + # got_nargs, nkwds, expected_nargs, has_vararg, has_kwarg, + # defaults_w, missing_args + sig = Signature([], None, None) + err = ArgErrCountMethod(1, 0, sig, None, 0) + s = err.getmsg() + assert s == "takes no arguments (1 given). Did you forget 'self' in the function definition?" + + sig = Signature(['a'], None, None) + err = ArgErrCountMethod(0, 0, sig, [], 1) + s = err.getmsg() + assert s == "takes exactly 1 argument (0 given)" + + sig = Signature(['self', 'b'], None, None) + err = ArgErrCountMethod(3, 0, sig, [], 0) + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given)" + sig = Signature(['a', 'b'], None, None) + err = ArgErrCountMethod(3, 0, sig, [], 0) + s = err.getmsg() + assert s == "takes exactly 2 arguments (3 given). Did you forget 'self' in the function definition?" + err = ArgErrCountMethod(3, 0, sig, ['a'], 0) + s = err.getmsg() + assert s == "takes at most 2 arguments (3 given). Did you forget 'self' in the function definition?" + + sig = Signature(['a', 'b'], '*', None) + err = ArgErrCountMethod(1, 0, sig, [], 1) + s = err.getmsg() + assert s == "takes at least 2 arguments (1 given)" + err = ArgErrCountMethod(0, 1, sig, ['a'], 1) + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" + + sig = Signature(['a'], None, '**') + err = ArgErrCountMethod(2, 1, sig, [], 0) + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (2 given). Did you forget 'self' in the function definition?" + err = ArgErrCountMethod(0, 1, sig, [], 1) + s = err.getmsg() + assert s == "takes exactly 1 non-keyword argument (0 given)" + + sig = Signature(['a'], '*', '**') + err = ArgErrCountMethod(0, 1, sig, [], 1) + s = err.getmsg() + assert s == "takes at least 1 non-keyword argument (0 given)" + + sig = Signature(['a'], None, '**') + err = ArgErrCountMethod(2, 1, sig, ['a'], 0) + s = err.getmsg() + assert s == "takes at most 1 non-keyword argument (2 given). Did you forget 'self' in the function definition?" + def test_bad_type_for_star(self): space = self.space try: @@ -674,6 +729,45 @@ exc = raises(TypeError, (lambda a, b, **kw: 0), a=1) assert exc.value.message == "<lambda>() takes exactly 2 non-keyword arguments (0 given)" + @py.test.mark.skipif("config.option.runappdirect") + def test_error_message_method(self): + class A(object): + def f0(): + pass + def f1(a): + pass + exc = raises(TypeError, lambda : A().f0()) + assert exc.value.message == "f0() takes no arguments (1 given). Did you forget 'self' in the function definition?" + exc = raises(TypeError, lambda : A().f1(1)) + assert exc.value.message == "f1() takes exactly 1 argument (2 given). Did you forget 'self' in the function definition?" + def f0(): + pass + exc = raises(TypeError, f0, 1) + # does not contain the warning about missing self + assert exc.value.message == "f0() takes no arguments (1 given)" + + @py.test.mark.skipif("config.option.runappdirect") + def test_error_message_module_function(self): + import operator # use repeat because it's defined at applevel + exc = raises(TypeError, lambda : operator.repeat(1, 2, 3)) + # does not contain the warning about missing self + assert exc.value.message == "repeat() takes exactly 2 arguments (3 given)" + + @py.test.mark.skipif("config.option.runappdirect") + def test_error_message_bound_method(self): + class A(object): + def f0(): + pass + def f1(a): + pass + m0 = A().f0 + exc = raises(TypeError, lambda : m0()) + assert exc.value.message == "f0() takes no arguments (1 given). Did you forget 'self' in the function definition?" + m1 = A().f1 + exc = raises(TypeError, lambda : m1(1)) + assert exc.value.message == "f1() takes exactly 1 argument (2 given). Did you forget 'self' in the function definition?" + + def test_unicode_keywords(self): def f(**kwargs): assert kwargs[u"美"] == 42 diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py --- a/pypy/module/__pypy__/__init__.py +++ b/pypy/module/__pypy__/__init__.py @@ -2,6 +2,8 @@ from pypy.interpreter.mixedmodule import MixedModule from pypy.module.imp.importing import get_pyc_magic +from rpython.rlib import rtime + class BuildersModule(MixedModule): appleveldefs = {} @@ -14,16 +16,11 @@ class TimeModule(MixedModule): appleveldefs = {} interpleveldefs = {} - if sys.platform.startswith("linux") or 'bsd' in sys.platform: - from pypy.module.__pypy__ import interp_time + if rtime.HAS_CLOCK_GETTIME: interpleveldefs["clock_gettime"] = "interp_time.clock_gettime" interpleveldefs["clock_getres"] = "interp_time.clock_getres" - for name in [ - "CLOCK_REALTIME", "CLOCK_MONOTONIC", "CLOCK_MONOTONIC_RAW", - "CLOCK_PROCESS_CPUTIME_ID", "CLOCK_THREAD_CPUTIME_ID" - ]: - if getattr(interp_time, name) is not None: - interpleveldefs[name] = "space.wrap(interp_time.%s)" % name + for name in rtime.ALL_DEFINED_CLOCKS: + interpleveldefs[name] = "space.wrap(%d)" % getattr(rtime, name) class ThreadModule(MixedModule): diff --git a/pypy/module/__pypy__/interp_time.py b/pypy/module/__pypy__/interp_time.py --- a/pypy/module/__pypy__/interp_time.py +++ b/pypy/module/__pypy__/interp_time.py @@ -4,71 +4,28 @@ from pypy.interpreter.error import exception_from_saved_errno from pypy.interpreter.gateway import unwrap_spec from rpython.rtyper.lltypesystem import rffi, lltype -from rpython.rtyper.tool import rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rlib import rtime +from rpython.rlib.rtime import HAS_CLOCK_GETTIME -if sys.platform == 'linux2': - libraries = ["rt"] -else: - libraries = [] - - -class CConfig: - _compilation_info_ = ExternalCompilationInfo( - includes=["time.h"], - libraries=libraries, - ) - - HAS_CLOCK_GETTIME = rffi_platform.Has('clock_gettime') - - CLOCK_REALTIME = rffi_platform.DefinedConstantInteger("CLOCK_REALTIME") - CLOCK_MONOTONIC = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC") - CLOCK_MONOTONIC_RAW = rffi_platform.DefinedConstantInteger("CLOCK_MONOTONIC_RAW") - CLOCK_PROCESS_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_PROCESS_CPUTIME_ID") - CLOCK_THREAD_CPUTIME_ID = rffi_platform.DefinedConstantInteger("CLOCK_THREAD_CPUTIME_ID") - -cconfig = rffi_platform.configure(CConfig) - -HAS_CLOCK_GETTIME = cconfig["HAS_CLOCK_GETTIME"] - -CLOCK_REALTIME = cconfig["CLOCK_REALTIME"] -CLOCK_MONOTONIC = cconfig["CLOCK_MONOTONIC"] -CLOCK_MONOTONIC_RAW = cconfig["CLOCK_MONOTONIC_RAW"] -CLOCK_PROCESS_CPUTIME_ID = cconfig["CLOCK_PROCESS_CPUTIME_ID"] -CLOCK_THREAD_CPUTIME_ID = cconfig["CLOCK_THREAD_CPUTIME_ID"] if HAS_CLOCK_GETTIME: - #redo it for timespec - CConfig.TIMESPEC = rffi_platform.Struct("struct timespec", [ - ("tv_sec", rffi.TIME_T), - ("tv_nsec", rffi.LONG), - ]) - cconfig = rffi_platform.configure(CConfig) - TIMESPEC = cconfig['TIMESPEC'] - - c_clock_gettime = rffi.llexternal("clock_gettime", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) - c_clock_getres = rffi.llexternal("clock_getres", - [lltype.Signed, lltype.Ptr(TIMESPEC)], rffi.INT, - compilation_info=CConfig._compilation_info_, releasegil=False, - save_err=rffi.RFFI_SAVE_ERRNO - ) @unwrap_spec(clk_id="c_int") def clock_gettime(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_gettime(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_gettime(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) @unwrap_spec(clk_id="c_int") def clock_getres(space, clk_id): - with lltype.scoped_alloc(TIMESPEC) as tp: - ret = c_clock_getres(clk_id, tp) + with lltype.scoped_alloc(rtime.TIMESPEC) as tp: + ret = rtime.c_clock_getres(clk_id, tp) if ret != 0: raise exception_from_saved_errno(space, space.w_IOError) - return space.wrap(int(tp.c_tv_sec) + 1e-9 * int(tp.c_tv_nsec)) + t = (float(rffi.getintfield(tp, 'c_tv_sec')) + + float(rffi.getintfield(tp, 'c_tv_nsec')) * 0.000000001) + return space.wrap(t) diff --git a/pypy/module/_file/interp_file.py b/pypy/module/_file/interp_file.py --- a/pypy/module/_file/interp_file.py +++ b/pypy/module/_file/interp_file.py @@ -140,7 +140,11 @@ stream = dispatch_filename(streamio.open_file_as_stream)( self.space, w_name, mode, buffering, signal_checker(self.space)) fd = stream.try_to_find_file_descriptor() - self.check_not_dir(fd) + try: + self.check_not_dir(fd) + except: + stream.close() + raise self.fdopenstream(stream, fd, mode) def direct___enter__(self): diff --git a/pypy/module/_file/test/test_file_extra.py b/pypy/module/_file/test/test_file_extra.py --- a/pypy/module/_file/test/test_file_extra.py +++ b/pypy/module/_file/test/test_file_extra.py @@ -667,3 +667,20 @@ f2.close() s2.close() s1.close() + + def test_close_fd_if_dir_check_fails(self): + from errno import EMFILE + for i in range(1700): + try: + open('/') + except IOError as e: + assert e.errno != EMFILE + else: + assert False + + @py.test.mark.skipif("os.name != 'posix'") + def test_dont_close_fd_if_dir_check_fails_in_fdopen(self): + import posix + fd = posix.open('/', posix.O_RDONLY) + raises(IOError, posix.fdopen, fd) + posix.close(fd) diff --git a/pypy/module/cpyext/test/_sre.c b/pypy/module/cpyext/test/_sre.c --- a/pypy/module/cpyext/test/_sre.c +++ b/pypy/module/cpyext/test/_sre.c @@ -2608,8 +2608,8 @@ }; statichere PyTypeObject Pattern_Type = { - PyObject_HEAD_INIT(NULL) - 0, "_" SRE_MODULE ".SRE_Pattern", + PyVarObject_HEAD_INIT(NULL, 0) + "_" SRE_MODULE ".SRE_Pattern", sizeof(PatternObject), sizeof(SRE_CODE), (destructor)pattern_dealloc, /*tp_dealloc*/ 0, /* tp_print */ @@ -3794,8 +3794,8 @@ }; statichere PyTypeObject Scanner_Type = { - PyObject_HEAD_INIT(NULL) - 0, "_" SRE_MODULE ".SRE_Scanner", + PyVarObject_HEAD_INIT(NULL, 0) + "_" SRE_MODULE ".SRE_Scanner", sizeof(ScannerObject), 0, (destructor)scanner_dealloc, /*tp_dealloc*/ 0, /* tp_print */ diff --git a/pypy/module/cpyext/test/conftest.py b/pypy/module/cpyext/test/conftest.py --- a/pypy/module/cpyext/test/conftest.py +++ b/pypy/module/cpyext/test/conftest.py @@ -2,6 +2,12 @@ import pytest def pytest_configure(config): + if config.option.runappdirect: + import sys + import py + from pypy import pypydir + sys.path.append(str(py.path.local(pypydir) / 'tool' / 'cpyext')) + return from pypy.tool.pytest.objspace import gettestobjspace # For some reason (probably a ll2ctypes cache issue on linux64) # it's necessary to run "import time" at least once before any diff --git a/pypy/module/cpyext/test/foo.c b/pypy/module/cpyext/test/foo.c --- a/pypy/module/cpyext/test/foo.c +++ b/pypy/module/cpyext/test/foo.c @@ -259,8 +259,7 @@ }; PyTypeObject UnicodeSubtype = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.fuu", sizeof(UnicodeSubclassObject), 0, @@ -318,8 +317,7 @@ }; PyTypeObject UnicodeSubtype2 = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.fuu2", sizeof(UnicodeSubclassObject), 0, @@ -377,8 +375,7 @@ }; PyTypeObject UnicodeSubtype3 = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.fuu3", sizeof(UnicodeSubclassObject) }; @@ -386,8 +383,7 @@ /* A Metatype */ PyTypeObject MetaType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.Meta", sizeof(PyHeapTypeObject),/*tp_basicsize*/ 0, /*tp_itemsize*/ @@ -466,8 +462,7 @@ PyTypeObject InitErrType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.InitErrType", sizeof(PyObject),/*tp_basicsize*/ 0, /*tp_itemsize*/ @@ -550,8 +545,7 @@ PyTypeObject SimplePropertyType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.Property", sizeof(PyObject), 0, @@ -619,14 +613,13 @@ static PyObject *newCustom(PyObject *self, PyObject *args) { PyObject *obj = calloc(1, sizeof(PyObject)); - obj->ob_type = &CustomType; + Py_TYPE(obj) = &CustomType; _Py_NewReference(obj); return obj; } static PyTypeObject CustomType = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.Custom", /*tp_name*/ sizeof(PyObject), /*tp_size*/ 0, /*tp_itemsize*/ @@ -635,8 +628,7 @@ }; static PyTypeObject TupleLike = { - PyObject_HEAD_INIT(NULL) - 0, + PyVarObject_HEAD_INIT(NULL, 0) "foo.TupleLike", /*tp_name*/ sizeof(PyObject), /*tp_size*/ }; @@ -736,7 +728,7 @@ SimplePropertyType.tp_new = PyType_GenericNew; InitErrType.tp_new = PyType_GenericNew; - CustomType.ob_type = &MetaType; + Py_TYPE(&CustomType) = &MetaType; if (PyType_Ready(&CustomType) < 0) INITERROR; diff --git a/pypy/module/cpyext/test/support.py b/pypy/module/cpyext/test/support.py deleted file mode 100644 --- a/pypy/module/cpyext/test/support.py +++ /dev/null @@ -1,68 +0,0 @@ -import os -import py -from sys import platform - -if os.name != 'nt': - so_ext = 'so' -else: - so_ext = 'dll' - -def c_compile(cfilenames, outputfilename, - compile_extra=None, link_extra=None, - include_dirs=None, libraries=None, library_dirs=None): - compile_extra = compile_extra or [] - link_extra = link_extra or [] - include_dirs = include_dirs or [] - libraries = libraries or [] - library_dirs = library_dirs or [] - if platform == 'win32': - link_extra = link_extra + ['/DEBUG'] # generate .pdb file - if platform == 'darwin': - # support Fink & Darwinports - for s in ('/sw/', '/opt/local/'): - if (s + 'include' not in include_dirs - and os.path.exists(s + 'include')): - include_dirs.append(s + 'include') - if s + 'lib' not in library_dirs and os.path.exists(s + 'lib'): - library_dirs.append(s + 'lib') - - outputfilename = py.path.local(outputfilename).new(ext=so_ext) - saved_environ = os.environ.copy() - try: - _build( - cfilenames, outputfilename, - compile_extra, link_extra, - include_dirs, libraries, library_dirs) - finally: - # workaround for a distutils bugs where some env vars can - # become longer and longer every time it is used - for key, value in saved_environ.items(): - if os.environ.get(key) != value: - os.environ[key] = value - return outputfilename - -def _build(cfilenames, outputfilename, compile_extra, link_extra, - include_dirs, libraries, library_dirs): - from distutils.ccompiler import new_compiler - from distutils import sysconfig - compiler = new_compiler(force=1) - sysconfig.customize_compiler(compiler) # XXX - objects = [] - for cfile in cfilenames: - cfile = py.path.local(cfile) - old = cfile.dirpath().chdir() - try: - res = compiler.compile([cfile.basename], - include_dirs=include_dirs, extra_preargs=compile_extra) - assert len(res) == 1 - cobjfile = py.path.local(res[0]) - assert cobjfile.check() - objects.append(str(cobjfile)) - finally: - old.chdir() - - compiler.link_shared_object( - objects, str(outputfilename), - libraries=libraries, - extra_preargs=link_extra, - library_dirs=library_dirs) diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -1,24 +1,21 @@ import sys import weakref -import os -import py, pytest +import pytest -from pypy import pypydir -from pypy.interpreter import gateway +from pypy.tool.cpyext.extbuild import ( + SystemCompilationInfo, HERE, get_sys_info_app) +from pypy.interpreter.gateway import unwrap_spec, interp2app from rpython.rtyper.lltypesystem import lltype, ll2ctypes -from rpython.translator.gensupp import uniquemodulename -from rpython.tool.udir import udir from pypy.module.cpyext import api from pypy.module.cpyext.state import State from pypy.module.cpyext.pyobject import Py_DecRef from rpython.tool.identity_dict import identity_dict from rpython.tool import leakfinder from rpython.rlib import rawrefcount +from rpython.tool.udir import udir -from .support import c_compile - -only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" +only_pypy ="config.option.runappdirect and '__pypy__' not in sys.builtin_module_names" @api.cpython_api([], api.PyObject) def PyPy_Crash1(space): @@ -33,40 +30,19 @@ assert 'PyModule_Check' in api.FUNCTIONS assert api.FUNCTIONS['PyModule_Check'].argtypes == [api.PyObject] -def convert_sources_to_files(sources, dirname): - files = [] - for i, source in enumerate(sources): - filename = dirname / ('source_%d.c' % i) - with filename.open('w') as f: - f.write(str(source)) - files.append(filename) - return files -def create_so(modname, include_dirs, source_strings=None, source_files=None, - compile_extra=None, link_extra=None, libraries=None): - dirname = (udir/uniquemodulename('module')).ensure(dir=1) - if source_strings: - assert not source_files - files = convert_sources_to_files(source_strings, dirname) - source_files = files - soname = c_compile(source_files, outputfilename=str(dirname/modname), - compile_extra=compile_extra, link_extra=link_extra, - include_dirs=include_dirs, - libraries=libraries) - return soname +class SpaceCompiler(SystemCompilationInfo): + """Extension compiler for regular (untranslated PyPy) mode""" + def __init__(self, space, *args, **kwargs): + self.space = space + SystemCompilationInfo.__init__(self, *args, **kwargs) -class SystemCompilationInfo(object): - """Bundles all the generic information required to compile extensions. + def load_module(self, mod, name): + space = self.space + api.load_extension_module(space, mod, name) + return space.getitem( + space.sys.get('modules'), space.wrap(name)) - Note: here, 'system' means OS + target interpreter + test config + ... - """ - def __init__(self, include_extra=None, compile_extra=None, link_extra=None, - extra_libs=None, ext=None): - self.include_extra = include_extra or [] - self.compile_extra = compile_extra - self.link_extra = link_extra - self.extra_libs = extra_libs - self.ext = ext def get_cpyext_info(space): from pypy.module.imp.importing import get_so_extension @@ -88,7 +64,8 @@ link_extra = ["-g"] else: compile_extra = link_extra = None - return SystemCompilationInfo( + return SpaceCompiler(space, + builddir_base=udir, include_extra=api.include_dirs, compile_extra=compile_extra, link_extra=link_extra, @@ -96,59 +73,6 @@ ext=get_so_extension(space)) -def compile_extension_module(sys_info, modname, include_dirs=[], - source_files=None, source_strings=None): - """ - Build an extension module and return the filename of the resulting native - code file. - - modname is the name of the module, possibly including dots if it is a module - inside a package. - - Any extra keyword arguments are passed on to ExternalCompilationInfo to - build the module (so specify your source with one of those). - """ - modname = modname.split('.')[-1] - soname = create_so(modname, - include_dirs=sys_info.include_extra + include_dirs, - source_files=source_files, - source_strings=source_strings, - compile_extra=sys_info.compile_extra, - link_extra=sys_info.link_extra, - libraries=sys_info.extra_libs) - pydname = soname.new(purebasename=modname, ext=sys_info.ext) - soname.rename(pydname) - return str(pydname) - -def get_so_suffix(): - from imp import get_suffixes, C_EXTENSION - for suffix, mode, typ in get_suffixes(): - if typ == C_EXTENSION: - return suffix - else: - raise RuntimeError("This interpreter does not define a filename " - "suffix for C extensions!") - -def get_sys_info_app(): - from distutils.sysconfig import get_python_inc - if sys.platform == 'win32': - compile_extra = ["/we4013"] - link_extra = ["/LIBPATH:" + os.path.join(sys.exec_prefix, 'libs')] - elif sys.platform == 'darwin': - compile_extra = link_extra = None - pass - elif sys.platform.startswith('linux'): - compile_extra = [ - "-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"] - link_extra = None - ext = get_so_suffix() - return SystemCompilationInfo( - include_extra=[get_python_inc()], - compile_extra=compile_extra, - link_extra=link_extra, - ext=get_so_suffix()) - - def freeze_refcnts(self): rawrefcount._dont_free_any_more() return #ZZZ @@ -159,25 +83,9 @@ #state.print_refcounts() self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values()) -class FakeSpace(object): - """Like TinyObjSpace, but different""" - def __init__(self, config): - self.config = config - - def passthrough(self, arg): - return arg - listview = passthrough - str_w = passthrough - - def unwrap(self, args): - try: - return args.str_w(None) - except: - return args - class LeakCheckingTest(object): """Base class for all cpyext tests.""" - spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array', + spaceconfig = dict(usemodules=['cpyext', 'thread', 'struct', 'array', 'itertools', 'time', 'binascii', 'micronumpy', 'mmap' ]) @@ -265,9 +173,12 @@ cls.w_libc = cls.space.wrap(get_libc_name()) def setup_method(self, meth): - freeze_refcnts(self) + if not self.runappdirect: + freeze_refcnts(self) def teardown_method(self, meth): + if self.runappdirect: + return self.cleanup_references(self.space) # XXX: like AppTestCpythonExtensionBase.teardown_method: # find out how to disable check_and_print_leaks() if the @@ -293,21 +204,82 @@ skip("Windows Python >= 2.6 only") assert isinstance(sys.dllhandle, int) + +def _unwrap_include_dirs(space, w_include_dirs): + if w_include_dirs is None: + return None + else: + return [space.str_w(s) for s in space.listview(w_include_dirs)] + +def debug_collect(space): + rawrefcount._collect() + class AppTestCpythonExtensionBase(LeakCheckingTest): def setup_class(cls): space = cls.space - space.getbuiltinmodule("cpyext") - # 'import os' to warm up reference counts - w_import = space.builtin.getdictvalue(space, '__import__') - space.call_function(w_import, space.wrap("os")) - #state = cls.space.fromcache(RefcountState) ZZZ - #state.non_heaptypes_w[:] = [] + cls.w_here = space.wrap(str(HERE)) + cls.w_udir = space.wrap(str(udir)) + cls.w_runappdirect = space.wrap(cls.runappdirect) if not cls.runappdirect: - cls.w_runappdirect = space.wrap(cls.runappdirect) + cls.sys_info = get_cpyext_info(space) + space.getbuiltinmodule("cpyext") + # 'import os' to warm up reference counts + w_import = space.builtin.getdictvalue(space, '__import__') + space.call_function(w_import, space.wrap("os")) + #state = cls.space.fromcache(RefcountState) ZZZ + #state.non_heaptypes_w[:] = [] + cls.w_debug_collect = space.wrap(interp2app(debug_collect)) + else: + def w_import_module(self, name, init=None, body='', filename=None, + include_dirs=None, PY_SSIZE_T_CLEAN=False): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.import_module( + name, init=init, body=body, filename=filename, + include_dirs=include_dirs, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + cls.w_import_module = w_import_module + + def w_import_extension(self, modname, functions, prologue="", + include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.import_extension( + modname, functions, prologue=prologue, + include_dirs=include_dirs, more_init=more_init, + PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) + cls.w_import_extension = w_import_extension + + def w_compile_module(self, name, + source_files=None, source_strings=None): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.compile_extension_module(name, + source_files=source_files, source_strings=source_strings) + cls.w_compile_module = w_compile_module + + def w_load_module(self, mod, name): + from extbuild import get_sys_info_app + sys_info = get_sys_info_app(self.udir) + return sys_info.load_module(mod, name) + cls.w_load_module = w_load_module + + + def record_imported_module(self, name): + """ + Record a module imported in a test so that it can be cleaned up in + teardown before the check for leaks is done. + + name gives the name of the module in the space's sys.modules. + """ + self.imported_module_names.append(name) def setup_method(self, func): - @gateway.unwrap_spec(name=str) + if self.runappdirect: + return + + @unwrap_spec(name=str) def compile_module(space, name, w_source_files=None, w_source_strings=None): @@ -322,166 +294,54 @@ source_strings = space.listview_bytes(w_source_strings) else: source_strings = None - pydname = compile_extension_module( - self.sys_info, name, + pydname = self.sys_info.compile_extension_module( + name, source_files=source_files, source_strings=source_strings) + + # hackish, but tests calling compile_module() always end up + # importing the result + self.record_imported_module(name) + return space.wrap(pydname) - @gateway.unwrap_spec(name=str, init='str_or_None', body=str, - load_it=bool, filename='str_or_None', - PY_SSIZE_T_CLEAN=bool) - def import_module(space, name, init=None, body='', load_it=True, + @unwrap_spec(name=str, init='str_or_None', body=str, + filename='str_or_None', PY_SSIZE_T_CLEAN=bool) + def import_module(space, name, init=None, body='', filename=None, w_include_dirs=None, PY_SSIZE_T_CLEAN=False): - """ - init specifies the overall template of the module. + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_module( + name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN) + self.record_imported_module(name) + return w_result - if init is None, the module source will be loaded from a file in this - test direcory, give a name given by the filename parameter. - if filename is None, the module name will be used to construct the - filename. - """ - if w_include_dirs is None: - include_dirs = [] - else: - include_dirs = [space.str_w(s) for s in space.listview(w_include_dirs)] - if init is not None: - code = """ - %(PY_SSIZE_T_CLEAN)s - #include <Python.h> - /* fix for cpython 2.7 Python.h if running tests with -A - since pypy compiles with -fvisibility-hidden */ - #undef PyMODINIT_FUNC - #ifdef __GNUC__ - # define RPY_EXPORTED extern __attribute__((visibility("default"))) - #else - # define RPY_EXPORTED extern __declspec(dllexport) - #endif - #define PyMODINIT_FUNC RPY_EXPORTED void + @unwrap_spec(mod=str, name=str) + def load_module(space, mod, name): + return self.sys_info.load_module(mod, name) - %(body)s - - PyMODINIT_FUNC - init%(name)s(void) { - %(init)s - } - """ % dict(name=name, init=init, body=body, - PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN' - if PY_SSIZE_T_CLEAN else '') - kwds = dict(source_strings=[code]) - else: - assert not PY_SSIZE_T_CLEAN - if filename is None: - filename = name - filename = py.path.local(pypydir) / 'module' \ - / 'cpyext'/ 'test' / (filename + ".c") - kwds = dict(source_files=[filename]) - mod = compile_extension_module(self.sys_info, name, - include_dirs=include_dirs, **kwds) - - if load_it: - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - self.imported_module_names.append(name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - else: - path = os.path.dirname(mod) - if self.runappdirect: - return path - else: - return space.wrap(path) - - @gateway.unwrap_spec(mod=str, name=str) - def reimport_module(space, mod, name): - if self.runappdirect: - import imp - return imp.load_dynamic(name, mod) - else: - api.load_extension_module(space, mod, name) - return space.getitem( - space.sys.get('modules'), - space.wrap(name)) - - @gateway.unwrap_spec(modname=str, prologue=str, + @unwrap_spec(modname=str, prologue=str, more_init=str, PY_SSIZE_T_CLEAN=bool) def import_extension(space, modname, w_functions, prologue="", w_include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False): functions = space.unwrap(w_functions) - methods_table = [] - codes = [] - for funcname, flags, code in functions: - cfuncname = "%s_%s" % (modname, funcname) - methods_table.append("{\"%s\", %s, %s}," % - (funcname, cfuncname, flags)) - func_code = """ - static PyObject* %s(PyObject* self, PyObject* args) - { - %s - } - """ % (cfuncname, code) - codes.append(func_code) - - body = prologue + "\n".join(codes) + """ - static PyMethodDef methods[] = { - %s - { NULL } - }; - """ % ('\n'.join(methods_table),) - init = """Py_InitModule("%s", methods);""" % (modname,) - if more_init: - init += more_init - return import_module(space, name=modname, init=init, body=body, - w_include_dirs=w_include_dirs, - PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN) - - @gateway.unwrap_spec(name=str) - def record_imported_module(name): - """ - Record a module imported in a test so that it can be cleaned up in - teardown before the check for leaks is done. - - name gives the name of the module in the space's sys.modules. - """ - self.imported_module_names.append(name) - - def debug_collect(space): - rawrefcount._collect() + include_dirs = _unwrap_include_dirs(space, w_include_dirs) + w_result = self.sys_info.import_extension( + modname, functions, prologue, include_dirs, more_init, + PY_SSIZE_T_CLEAN) + self.record_imported_module(modname) + return w_result # A list of modules which the test caused to be imported (in # self.space). These will be cleaned up automatically in teardown. self.imported_module_names = [] - if self.runappdirect: - fake = FakeSpace(self.space.config) - def interp2app(func): - def run(*args, **kwargs): - for k in kwargs.keys(): - if k not in func.unwrap_spec and not k.startswith('w_'): - v = kwargs.pop(k) - kwargs['w_' + k] = v - return func(fake, *args, **kwargs) - return run - def wrap(func): - return func - self.sys_info = get_sys_info_app() - else: - interp2app = gateway.interp2app - wrap = self.space.wrap - self.sys_info = get_cpyext_info(self.space) + wrap = self.space.wrap self.w_compile_module = wrap(interp2app(compile_module)) + self.w_load_module = wrap(interp2app(load_module)) self.w_import_module = wrap(interp2app(import_module)) - self.w_reimport_module = wrap(interp2app(reimport_module)) self.w_import_extension = wrap(interp2app(import_extension)) - self.w_record_imported_module = wrap(interp2app(record_imported_module)) - self.w_here = wrap(str(py.path.local(pypydir)) + '/module/cpyext/test/') - self.w_debug_collect = wrap(interp2app(debug_collect)) # create the file lock before we count allocations self.space.call_method(self.space.sys.get("stdout"), "flush") @@ -498,6 +358,8 @@ self.space.delitem(w_modules, w_name) def teardown_method(self, func): + if self.runappdirect: + return for name in self.imported_module_names: self.unimport_module(name) self.cleanup_references(self.space) @@ -632,19 +494,15 @@ If `cherry.date` is an extension module which imports `apple.banana`, the latter is added to `sys.modules` for the `"apple.banana"` key. """ - if self.runappdirect: - skip('record_imported_module not supported in runappdirect mode') + import sys, types, os # Build the extensions. banana = self.compile_module( - "apple.banana", source_files=[self.here + 'banana.c']) - self.record_imported_module("apple.banana") + "apple.banana", source_files=[os.path.join(self.here, 'banana.c')]) date = self.compile_module( - "cherry.date", source_files=[self.here + 'date.c']) - self.record_imported_module("cherry.date") + "cherry.date", source_files=[os.path.join(self.here, 'date.c')]) # Set up some package state so that the extensions can actually be # imported. - import sys, types, os cherry = sys.modules['cherry'] = types.ModuleType('cherry') cherry.__path__ = [os.path.dirname(date)] @@ -652,7 +510,6 @@ apple.__path__ = [os.path.dirname(banana)] import cherry.date - import apple.banana assert sys.modules['apple.banana'].__name__ == 'apple.banana' assert sys.modules['cherry.date'].__name__ == 'cherry.date' @@ -989,7 +846,7 @@ f.write('not again!\n') f.close() m1 = sys.modules['foo'] - m2 = self.reimport_module(m1.__file__, name='foo') + m2 = self.load_module(m1.__file__, name='foo') assert m1 is m2 assert m1 is sys.modules['foo'] diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py --- a/pypy/module/cpyext/test/test_import.py +++ b/pypy/module/cpyext/test/test_import.py @@ -1,6 +1,6 @@ from pypy.module.cpyext.test.test_api import BaseApiTest from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase -from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem import rffi class TestImport(BaseApiTest): def test_import(self, space, api): @@ -39,9 +39,9 @@ class AppTestImportLogic(AppTestCpythonExtensionBase): def test_import_logic(self): - path = self.import_module(name='test_import_module', load_it=False) - import sys - sys.path.append(path) + import sys, os + path = self.compile_module('test_import_module', + source_files=[os.path.join(self.here, 'test_import_module.c')]) + sys.path.append(os.path.dirname(path)) import test_import_module assert test_import_module.TEST is None - diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py --- a/pypy/module/cpyext/test/test_intobject.py +++ b/pypy/module/cpyext/test/test_intobject.py @@ -120,8 +120,7 @@ }; PyTypeObject Enum_Type = { - PyObject_HEAD_INIT(0) - /*ob_size*/ 0, + PyVarObject_HEAD_INIT(NULL, 0) /*tp_name*/ "Enum", /*tp_basicsize*/ sizeof(EnumObject), /*tp_itemsize*/ 0, diff --git a/pypy/module/cpyext/test/test_pyfile.py b/pypy/module/cpyext/test/test_pyfile.py --- a/pypy/module/cpyext/test/test_pyfile.py +++ b/pypy/module/cpyext/test/test_pyfile.py @@ -101,7 +101,7 @@ w_stdout = space.sys.get("stdout") assert api.PyFile_SoftSpace(w_stdout, 1) == 0 assert api.PyFile_SoftSpace(w_stdout, 0) == 1 - + api.PyFile_SoftSpace(w_stdout, 1) w_ns = space.newdict() space.exec_("print 1,", w_ns, w_ns) @@ -117,11 +117,9 @@ class AppTestPyFile(AppTestCpythonExtensionBase): def setup_class(cls): + AppTestCpythonExtensionBase.setup_class.__func__(cls) from rpython.tool.udir import udir - if option.runappdirect: - cls.w_udir = str(udir) - else: - cls.w_udir = cls.space.wrap(str(udir)) + cls.w_udir = cls.space.wrap(str(udir)) def test_file_tell(self): module = self.import_extension('foo', [ @@ -158,4 +156,3 @@ t_py = fid.tell() assert t_c == t_py, 'after a fread, c level ftell(fp) %d but PyFile.tell() %d' % (t_c, t_py) - diff --git a/pypy/module/cpyext/test/test_sliceobject.py b/pypy/module/cpyext/test/test_sliceobject.py --- a/pypy/module/cpyext/test/test_sliceobject.py +++ b/pypy/module/cpyext/test/test_sliceobject.py @@ -15,10 +15,10 @@ def get_indices(w_start, w_stop, w_step, length): w_slice = space.newslice(w_start, w_stop, w_step) values = lltype.malloc(Py_ssize_tP.TO, 4, flavor='raw') - - res = api.PySlice_GetIndicesEx(w_slice, 100, values, - rffi.ptradd(values, 1), - rffi.ptradd(values, 2), + + res = api.PySlice_GetIndicesEx(w_slice, 100, values, + rffi.ptradd(values, 1), + rffi.ptradd(values, 2), rffi.ptradd(values, 3)) assert res == 0 rv = values[0], values[1], values[2], values[3] @@ -31,9 +31,9 @@ def get_indices(w_start, w_stop, w_step, length): w_slice = space.newslice(w_start, w_stop, w_step) values = lltype.malloc(Py_ssize_tP.TO, 3, flavor='raw') - - res = api.PySlice_GetIndices(w_slice, 100, values, - rffi.ptradd(values, 1), + + res = api.PySlice_GetIndices(w_slice, 100, values, + rffi.ptradd(values, 1), rffi.ptradd(values, 2)) assert res == 0 rv = values[0], values[1], values[2] @@ -47,7 +47,7 @@ ("clone", "METH_O", """ PySliceObject *slice = (PySliceObject *)args; - if (slice->ob_type != &PySlice_Type) { + if (Py_TYPE(slice) != &PySlice_Type) { PyErr_SetNone(PyExc_ValueError); return NULL; } diff --git a/pypy/module/cpyext/test/test_typeobject.py b/pypy/module/cpyext/test/test_typeobject.py --- a/pypy/module/cpyext/test/test_typeobject.py +++ b/pypy/module/cpyext/test/test_typeobject.py @@ -834,8 +834,7 @@ } PyTypeObject IntLike_Type = { - PyObject_HEAD_INIT(0) - /*ob_size*/ 0, + PyVarObject_HEAD_INIT(NULL, 0) /*tp_name*/ "IntLike", /*tp_basicsize*/ sizeof(IntLikeObject), }; @@ -930,8 +929,7 @@ } PyTypeObject IntLike_Type = { - PyObject_HEAD_INIT(0) - /*ob_size*/ 0, + PyVarObject_HEAD_INIT(NULL, 0) /*tp_name*/ "IntLike", /*tp_basicsize*/ sizeof(IntLikeObject), }; @@ -944,8 +942,7 @@ } IntLikeObjectNoOp; PyTypeObject IntLike_Type_NoOp = { - PyObject_HEAD_INIT(0) - /*ob_size*/ 0, + PyVarObject_HEAD_INIT(NULL, 0) /*tp_name*/ "IntLikeNoOp", /*tp_basicsize*/ sizeof(IntLikeObjectNoOp), }; diff --git a/pypy/module/faulthandler/__init__.py b/pypy/module/faulthandler/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/__init__.py @@ -0,0 +1,38 @@ +import sys +from pypy.interpreter.mixedmodule import MixedModule + + +class Module(MixedModule): + appleveldefs = { + } + + interpleveldefs = { + 'enable': 'handler.enable', + 'disable': 'handler.disable', + 'is_enabled': 'handler.is_enabled', +# 'register': 'interp_faulthandler.register', +# + 'dump_traceback': 'handler.dump_traceback', +# + '_read_null': 'handler.read_null', + '_sigsegv': 'handler.sigsegv', + '_sigfpe': 'handler.sigfpe', + '_sigabrt': 'handler.sigabrt', + '_stack_overflow': 'handler.stack_overflow', + } + + def setup_after_space_initialization(self): + """NOT_RPYTHON""" + if self.space.config.translation.thread: + self.extra_interpdef('dump_traceback_later', + 'handler.dump_traceback_later') + self.extra_interpdef('cancel_dump_traceback_later', + 'handler.cancel_dump_traceback_later') + if sys.platform != 'win32': + self.extra_interpdef('register', 'handler.register') + self.extra_interpdef('unregister', 'handler.unregister') + + def shutdown(self, space): + from pypy.module.faulthandler import handler + handler.finish(space) + MixedModule.shutdown(self, space) diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/cintf.py @@ -0,0 +1,99 @@ +import py +from rpython.rtyper.lltypesystem import lltype, llmemory, rffi, rstr +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +cwd = py.path.local(__file__).dirpath() +eci = ExternalCompilationInfo( + includes=[cwd.join('faulthandler.h')], + include_dirs=[str(cwd), cdir], + separate_module_files=[cwd.join('faulthandler.c')]) + +eci_later = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_LATER\n'])) +eci_user = eci.merge(ExternalCompilationInfo( + pre_include_bits=['#define PYPY_FAULTHANDLER_USER\n'])) + +def direct_llexternal(*args, **kwargs): + kwargs.setdefault('_nowrapper', True) + kwargs.setdefault('compilation_info', eci) + return rffi.llexternal(*args, **kwargs) + + +DUMP_CALLBACK = lltype.Ptr(lltype.FuncType( + [rffi.INT, rffi.SIGNEDP, lltype.Signed], lltype.Void)) + +pypy_faulthandler_setup = direct_llexternal( + 'pypy_faulthandler_setup', [DUMP_CALLBACK], rffi.CCHARP) + +pypy_faulthandler_teardown = direct_llexternal( + 'pypy_faulthandler_teardown', [], lltype.Void) + +pypy_faulthandler_enable = direct_llexternal( + 'pypy_faulthandler_enable', [rffi.INT, rffi.INT], rffi.CCHARP) + +pypy_faulthandler_disable = direct_llexternal( + 'pypy_faulthandler_disable', [], lltype.Void) + +pypy_faulthandler_is_enabled = direct_llexternal( + 'pypy_faulthandler_is_enabled', [], rffi.INT) + +pypy_faulthandler_write = direct_llexternal( + 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) + +pypy_faulthandler_write_int = direct_llexternal( + 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) + +pypy_faulthandler_dump_traceback = direct_llexternal( + 'pypy_faulthandler_dump_traceback', + [rffi.INT, rffi.INT, llmemory.Address], lltype.Void) + +pypy_faulthandler_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_dump_traceback_later', + [rffi.LONGLONG, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_later) + +pypy_faulthandler_cancel_dump_traceback_later = direct_llexternal( + 'pypy_faulthandler_cancel_dump_traceback_later', [], lltype.Void) + +pypy_faulthandler_check_signum = direct_llexternal( + 'pypy_faulthandler_check_signum', + [rffi.LONG], rffi.INT, + compilation_info=eci_user) + +pypy_faulthandler_register = direct_llexternal( + 'pypy_faulthandler_register', + [rffi.INT, rffi.INT, rffi.INT, rffi.INT], rffi.CCHARP, + compilation_info=eci_user) + +pypy_faulthandler_unregister = direct_llexternal( + 'pypy_faulthandler_unregister', + [rffi.INT], rffi.INT, + compilation_info=eci_user) + + +# for tests... + +pypy_faulthandler_read_null = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void) + +pypy_faulthandler_read_null_releasegil = direct_llexternal( + 'pypy_faulthandler_read_null', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigsegv = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void) + +pypy_faulthandler_sigsegv_releasegil = direct_llexternal( + 'pypy_faulthandler_sigsegv', [], lltype.Void, + _nowrapper=False, releasegil=True) + +pypy_faulthandler_sigfpe = direct_llexternal( + 'pypy_faulthandler_sigfpe', [], lltype.Void) + +pypy_faulthandler_sigabrt = direct_llexternal( + 'pypy_faulthandler_sigabrt', [], lltype.Void) + +pypy_faulthandler_stackoverflow = direct_llexternal( + 'pypy_faulthandler_stackoverflow', [lltype.Float], lltype.Float) diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/dumper.py @@ -0,0 +1,54 @@ +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib import rgc +from rpython.rlib.rvmprof import traceback + +from pypy.interpreter.pycode import PyCode +from pypy.module.faulthandler.cintf import pypy_faulthandler_write +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int + + +MAX_STRING_LENGTH = 500 + +global_buf = lltype.malloc(rffi.CCHARP.TO, MAX_STRING_LENGTH, flavor='raw', + immortal=True, zero=True) + +def _dump(fd, s): + assert isinstance(s, str) + l = len(s) + if l >= MAX_STRING_LENGTH: + l = MAX_STRING_LENGTH - 1 + i = 0 + while i < l: + global_buf[i] = s[i] + i += 1 + global_buf[l] = '\x00' + pypy_faulthandler_write(fd, global_buf) + +def _dump_int(fd, i): + pypy_faulthandler_write_int(fd, i) + + +def dump_code(pycode, loc, fd): + if pycode is None: + _dump(fd, " File ???") + else: + _dump(fd, ' File "') + _dump(fd, pycode.co_filename) + _dump(fd, '" in ') + _dump(fd, pycode.co_name) + _dump(fd, ", from line ") + _dump_int(fd, pycode.co_firstlineno) + if loc == traceback.LOC_JITTED: + _dump(fd, " [jitted]") + elif loc == traceback.LOC_JITTED_INLINED: + _dump(fd, " [jit inlined]") + _dump(fd, "\n") + + +@rgc.no_collect +def _dump_callback(fd, array_p, array_length): + """We are as careful as we can reasonably be here (i.e. not 100%, + but hopefully close enough). In particular, this is written as + RPython but shouldn't allocate anything. + """ + traceback.walk_traceback(PyCode, dump_code, fd, array_p, array_length) diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c new file mode 100644 --- /dev/null +++ b/pypy/module/faulthandler/faulthandler.c @@ -0,0 +1,679 @@ +#include "faulthandler.h" +#include <stdlib.h> +#include <stdio.h> +#include <signal.h> +#include <assert.h> +#include <errno.h> +#include <string.h> +#include <unistd.h> +#include <sys/resource.h> +#include <math.h> + +#ifdef RPYTHON_LL2CTYPES +# include "../../../rpython/rlib/rvmprof/src/rvmprof.h" +#else +# include "common_header.h" +# include "structdef.h" +# include "rvmprof.h" +#endif +#include "src/threadlocal.h" + +#define MAX_FRAME_DEPTH 100 +#define FRAME_DEPTH_N RVMPROF_TRACEBACK_ESTIMATE_N(MAX_FRAME_DEPTH) + + +typedef struct sigaction _Py_sighandler_t; + +typedef struct { + const int signum; + volatile int enabled; + const char* name; + _Py_sighandler_t previous; +} fault_handler_t; + +static struct { + int initialized; + int enabled; + volatile int fd, all_threads; + volatile pypy_faulthandler_cb_t dump_traceback; +} fatal_error; + +static stack_t stack; + + +static fault_handler_t faulthandler_handlers[] = { +#ifdef SIGBUS + {SIGBUS, 0, "Bus error", }, +#endif +#ifdef SIGILL + {SIGILL, 0, "Illegal instruction", }, +#endif + {SIGFPE, 0, "Floating point exception", }, + {SIGABRT, 0, "Aborted", }, + /* define SIGSEGV at the end to make it the default choice if searching the + handler fails in faulthandler_fatal_error() */ + {SIGSEGV, 0, "Segmentation fault", } +}; +static const int faulthandler_nsignals = + sizeof(faulthandler_handlers) / sizeof(fault_handler_t); + +RPY_EXTERN +void pypy_faulthandler_write(int fd, const char *str) +{ + (void)write(fd, str, strlen(str)); +} + +RPY_EXTERN +void pypy_faulthandler_write_int(int fd, long value) +{ + char buf[48]; + sprintf(buf, "%ld", value); + pypy_faulthandler_write(fd, buf); +} + + +RPY_EXTERN +void pypy_faulthandler_dump_traceback(int fd, int all_threads, + void *ucontext) +{ + pypy_faulthandler_cb_t fn; + intptr_t array_p[FRAME_DEPTH_N], array_length; + + fn = fatal_error.dump_traceback; + if (!fn) + return; + +#ifndef RPYTHON_LL2CTYPES + if (all_threads && _RPython_ThreadLocals_AcquireTimeout(10000) == 0) { + /* This is known not to be perfectly safe against segfaults if we + don't hold the GIL ourselves. Too bad. I suspect that CPython + has issues there too. + */ + struct pypy_threadlocal_s *my, *p; + int blankline = 0; + char buf[40]; + + my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); + p = _RPython_ThreadLocals_Head(); + p = _RPython_ThreadLocals_Enum(p); + while (p != NULL) { + if (blankline) + pypy_faulthandler_write(fd, "\n"); + blankline = 1; + + pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); + sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); + pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, " (most recent call first):\n"); + + array_length = vmprof_get_traceback(p->vmprof_tl_stack, + my == p ? ucontext : NULL, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + + p = _RPython_ThreadLocals_Enum(p); + } + _RPython_ThreadLocals_Release(); + } + else { + pypy_faulthandler_write(fd, "Stack (most recent call first):\n"); + array_length = vmprof_get_traceback(NULL, ucontext, + array_p, FRAME_DEPTH_N); + fn(fd, array_p, array_length); + } +#else + pypy_faulthandler_write(fd, "(no traceback when untranslated)\n"); +#endif +} + +static void +faulthandler_dump_traceback(int fd, int all_threads, void *ucontext) +{ + static volatile int reentrant = 0; + + if (reentrant) + return; + reentrant = 1; + pypy_faulthandler_dump_traceback(fd, all_threads, ucontext); + reentrant = 0; +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_LATER +#include "src/thread.h" +static struct { + int fd; + long long microseconds; + int repeat, exit; + /* The main thread always holds this lock. It is only released when + faulthandler_thread() is interrupted before this thread exits, or at + Python exit. */ + struct RPyOpaque_ThreadLock cancel_event; + /* released by child thread when joined */ + struct RPyOpaque_ThreadLock running; +} thread_later; + +static void faulthandler_thread(void) +{ +#ifndef _WIN32 + /* we don't want to receive any signal */ + sigset_t set; + sigfillset(&set); + pthread_sigmask(SIG_SETMASK, &set, NULL); +#endif + + RPyLockStatus st; + char buf[64]; + unsigned long hour, minutes, seconds, fraction; + long long t; + + do { + st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, + thread_later.microseconds, 0); + if (st == RPY_LOCK_ACQUIRED) { + RPyThreadReleaseLock(&thread_later.cancel_event); + break; + } + /* Timeout => dump traceback */ + assert(st == RPY_LOCK_FAILURE); + + /* getting to know which thread holds the GIL is not as simple + * as in CPython, so for now we don't */ + + t = thread_later.microseconds; + fraction = (unsigned long)(t % 1000000); + t /= 1000000; + seconds = (unsigned long)(t % 60); + t /= 60; + minutes = (unsigned long)(t % 60); + t /= 60; + hour = (unsigned long)t; + if (fraction == 0) + sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", + hour, minutes, seconds); + else + sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", + hour, minutes, seconds, fraction); + + pypy_faulthandler_write(thread_later.fd, buf); + pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + + if (thread_later.exit) + _exit(1); + } while (thread_later.repeat); + + /* The only way out */ + RPyThreadReleaseLock(&thread_later.running); +} + +RPY_EXTERN +char *pypy_faulthandler_dump_traceback_later(long long microseconds, int repeat, + int fd, int exit) +{ + pypy_faulthandler_cancel_dump_traceback_later(); + + thread_later.fd = fd; + thread_later.microseconds = microseconds; + thread_later.repeat = repeat; + thread_later.exit = exit; + + RPyThreadAcquireLock(&thread_later.running, 1); + + if (RPyThreadStart(&faulthandler_thread) == -1) { + RPyThreadReleaseLock(&thread_later.running); + return "unable to start watchdog thread"; + } + return NULL; +} +#endif /* PYPY_FAULTHANDLER_LATER */ + +RPY_EXTERN +void pypy_faulthandler_cancel_dump_traceback_later(void) +{ +#ifdef PYPY_FAULTHANDLER_LATER + /* Notify cancellation */ + RPyThreadReleaseLock(&thread_later.cancel_event); + + /* Wait for thread to join (or does nothing if no thread is running) */ + RPyThreadAcquireLock(&thread_later.running, 1); + RPyThreadReleaseLock(&thread_later.running); + + /* The main thread should always hold the cancel_event lock */ + RPyThreadAcquireLock(&thread_later.cancel_event, 1); +#endif /* PYPY_FAULTHANDLER_LATER */ +} + + +/************************************************************/ + + +#ifdef PYPY_FAULTHANDLER_USER +typedef struct { + int enabled; + int fd; + int all_threads; + int chain; + _Py_sighandler_t previous; +} user_signal_t; + +static user_signal_t *user_signals; + +#ifndef NSIG +# if defined(_NSIG) +# define NSIG _NSIG /* For BSD/SysV */ +# elif defined(_SIGMAX) +# define NSIG (_SIGMAX + 1) /* For QNX */ +# elif defined(SIGMAX) +# define NSIG (SIGMAX + 1) /* For djgpp */ +# else +# define NSIG 64 /* Use a reasonable default value */ +# endif +#endif + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext); + +static int +faulthandler_register(int signum, int chain, _Py_sighandler_t *p_previous) +{ + struct sigaction action; + action.sa_handler = faulthandler_user; + sigemptyset(&action.sa_mask); + /* if the signal is received while the kernel is executing a system + call, try to restart the system call instead of interrupting it and + return EINTR. */ + action.sa_flags = SA_RESTART | SA_SIGINFO; + if (chain) { + /* do not prevent the signal from being received from within its + own signal handler */ + action.sa_flags = SA_NODEFER; + } + if (stack.ss_sp != NULL) { + /* Call the signal handler on an alternate signal stack + provided by sigaltstack() */ + action.sa_flags |= SA_ONSTACK; + } + return sigaction(signum, &action, p_previous); +} + +static void faulthandler_user(int signum, siginfo_t *info, void *ucontext) +{ + int save_errno; + user_signal_t *user = &user_signals[signum]; + + if (!user->enabled) + return; + + save_errno = errno; + faulthandler_dump_traceback(user->fd, user->all_threads, ucontext); + + if (user->chain) { + (void)sigaction(signum, &user->previous, NULL); + errno = save_errno; + + /* call the previous signal handler */ + raise(signum); + + save_errno = errno; + (void)faulthandler_register(signum, user->chain, NULL); + } + + errno = save_errno; +} + +RPY_EXTERN +int pypy_faulthandler_check_signum(long signum) +{ + unsigned int i; + + for (i = 0; i < faulthandler_nsignals; i++) { + if (faulthandler_handlers[i].signum == signum) { + return -1; + } + } + if (signum < 1 || NSIG <= signum) { + return -2; + } + return 0; +} + +RPY_EXTERN +char *pypy_faulthandler_register(int signum, int fd, int all_threads, int chain) +{ + user_signal_t *user; + _Py_sighandler_t previous; + int err; + + if (user_signals == NULL) { + user_signals = malloc(NSIG * sizeof(user_signal_t)); + if (user_signals == NULL) + return "out of memory"; + memset(user_signals, 0, NSIG * sizeof(user_signal_t)); + } + + user = &user_signals[signum]; + user->fd = fd; + user->all_threads = all_threads; + user->chain = chain; + + if (!user->enabled) { + err = faulthandler_register(signum, chain, &previous); _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit