Amaury Forgeot d'Arc, 03.07.2012 14:55:
> 2012/7/3 Stefan Behnel
>> I've been working with one of the modules in the Python benchmark suite,
>> namely nbody, and tried to make it run a little faster when compiled with
>> Cython in PyPy. I managed to get a massive speed-up by avoiding some
>> borrowed references during list iteration and using PySequence_GetItem()
>> instead, but now some 50-60% of the runtime are spent in Py_DecRef().
>
> Don't forget that cpyext reference counts are quite different from CPython:
> PySequence_GetItem() needs to *create* a PyObject structure, and the
> returned object has a refcount of 1.
> Then Py_DECREF() will really *deallocate* the PyObject structure...
Sure. And using PySequence_GetItem() is still some 10x faster in PyPy than
taking a borrowed reference using PyList_GET_ITEM() and then calling
Py_INCREF() on it, which is what Cython does in CPython because it's way
faster there. The overhead of borrowed references is seriously huge in PyPy.
BTW, are PyObject structures currently cached in a free-list somewhere?
That would be really helpful for the iteration performance.
> This is quite more expensive than the simple refcount increment/decrement
> done by CPython.
>
>> OverflowError: array too large
>
> Looks like a ctypes bug to me. Which OS, Python, etc. are you using?
Ah - totally, sure. I accidentally ran the system Py2.5 on 64bit Linux.
Running it with Py2.7 fixes this specific problem, thanks for the hint!
Although it now names the extension module "nbody.so" instead of
"nbody.pypy-19.so". Comprend qui peut ...
After figuring out that I was supposed to enable cpyext manually and
running strace to see what extension module name it is actually looking
for, I failed to make it load the module it just built regardless of how I
named it, so I tried building it within the same run as follows:
pypy/bin/py.py --withmod-cpyext -c 'import setup; import nbody; \
nbody.test_nbody(1)' build_ext -i
Doing this, it then fails with the following error:
"""Traceback (most recent call last):
File ".../pypy/bin/py.py", line 187, in <module>
sys.exit(main_(sys.argv))
File ".../pypy/bin/py.py", line 158, in main_
verbose=interactiveconfig.verbose):
File ".../pypy/interpreter/main.py", line 103, in run_toplevel
f()
File ".../pypy/bin/py.py", line 133, in doit
main.run_string(command, space=space)
File ".../pypy/interpreter/main.py", line 59, in run_string
_run_eval_string(source, filename, space, False)
File ".../pypy/interpreter/main.py", line 48, in _run_eval_string
retval = pycode.exec_code(space, w_globals, w_globals)
File ".../pypy/interpreter/eval.py", line 33, in exec_code
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 834, in IMPORT_NAME
w_locals, w_fromlist)
File ".../pypy/interpreter/baseobjspace.py", line 986, in call_function
return w_func.funccall(*args_w)
File ".../pypy/interpreter/function.py", line 111, in funccall
return self.call_args(Arguments(self.space, list(args_w)))
File ".../pypy/interpreter/function.py", line 59, in call_args
return self.getcode().funcrun(self, args)
File ".../pypy/interpreter/gateway.py", line 614, in funcrun
return BuiltinCode.funcrun_obj(self, func, None, args)
File ".../pypy/interpreter/gateway.py", line 631, in funcrun_obj
self.handle_exception(space, e)
File ".../pypy/interpreter/gateway.py", line 648, in handle_exception
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/gateway.py", line 622, in funcrun_obj
w_result = activation._run(space, scope_w)
File "<6076-codegen .../pypy/tool/sourcetools.py:174>", line 3, in _run
File ".../pypy/module/imp/importing.py", line 271, in importhook
fromlist_w, tentative=True)
File ".../pypy/module/imp/importing.py", line 297, in absolute_import
fromlist_w, tentative)
File ".../pypy/module/imp/importing.py", line 306, in
absolute_import_with_lock
fromlist_w, tentative)
File ".../pypy/module/imp/importing.py", line 367, in _absolute_import
tentative=tentative)
File ".../pypy/module/imp/importing.py", line 646, in load_part
w_mod = load_module(space, w_modulename, find_info)
File ".../pypy/module/imp/importing.py", line 601, in load_module
find_info.stream.readall())
File ".../pypy/module/imp/importing.py", line 911, in load_source_module
exec_code_module(space, w_mod, code_w)
File ".../pypy/module/imp/importing.py", line 872, in exec_code_module
code_w.exec_code(space, w_dict, w_dict)
File ".../pypy/interpreter/eval.py", line 33, in exec_code
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1029, in CALL_FUNCTION
self.call_function(oparg)
File ".../pypy/interpreter/pyopcode.py", line 1012, in call_function
w_result = self.space.call_args(w_function, args)
File ".../pypy/objspace/descroperation.py", line 158, in call_args
return w_obj.call_args(args)
File ".../pypy/interpreter/function.py", line 59, in call_args
return self.getcode().funcrun(self, args)
File ".../pypy/interpreter/pycode.py", line 210, in funcrun
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1029, in CALL_FUNCTION
self.call_function(oparg)
File ".../pypy/interpreter/pyopcode.py", line 1012, in call_function
w_result = self.space.call_args(w_function, args)
File ".../pypy/objspace/descroperation.py", line 160, in call_args
return w_obj.call_args(args)
File ".../pypy/interpreter/function.py", line 471, in call_args
return space.call_obj_args(self.w_function, self.w_instance, args)
File ".../pypy/interpreter/baseobjspace.py", line 961, in call_obj_args
return w_callable.call_obj_args(w_obj, args)
File ".../pypy/interpreter/function.py", line 63, in call_obj_args
return self.getcode().funcrun_obj(self, w_obj, args)
File ".../pypy/interpreter/pycode.py", line 223, in funcrun_obj
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 147, in funccall_valuestack
return self._flat_pycall(code, nargs, frame)
File ".../pypy/interpreter/function.py", line 172, in _flat_pycall
return new_frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1029, in CALL_FUNCTION
self.call_function(oparg)
File ".../pypy/interpreter/pyopcode.py", line 1012, in call_function
w_result = self.space.call_args(w_function, args)
File ".../pypy/objspace/descroperation.py", line 158, in call_args
return w_obj.call_args(args)
File ".../pypy/interpreter/function.py", line 59, in call_args
return self.getcode().funcrun(self, args)
File ".../pypy/interpreter/pycode.py", line 210, in funcrun
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1029, in CALL_FUNCTION
self.call_function(oparg)
File ".../pypy/interpreter/pyopcode.py", line 1012, in call_function
w_result = self.space.call_args(w_function, args)
File ".../pypy/objspace/descroperation.py", line 158, in call_args
return w_obj.call_args(args)
File ".../pypy/interpreter/function.py", line 59, in call_args
return self.getcode().funcrun(self, args)
File ".../pypy/interpreter/pycode.py", line 210, in funcrun
return frame.run()
File ".../pypy/interpreter/pyframe.py", line 141, in run
return self.execute_frame()
File ".../pypy/interpreter/pyframe.py", line 175, in execute_frame
executioncontext)
File ".../pypy/interpreter/pyopcode.py", line 84, in dispatch
next_instr = self.handle_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 107, in handle_bytecode
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/pyopcode.py", line 90, in handle_bytecode
next_instr = self.dispatch_bytecode(co_code, next_instr, ec)
File ".../pypy/interpreter/pyopcode.py", line 265, in dispatch_bytecode
res = meth(oparg, next_instr)
File ".../pypy/interpreter/pyopcode.py", line 1022, in CALL_FUNCTION
w_result = self.space.call_valuestack(w_function, nargs, self)
File ".../pypy/interpreter/baseobjspace.py", line 1015, in call_valuestack
return w_func.funccall_valuestack(nargs, frame)
File ".../pypy/interpreter/function.py", line 128, in funccall_valuestack
return code.fastcall_0(self.space, self)
File ".../pypy/interpreter/gateway.py", line 705, in fastcall_0
self.handle_exception(space, e)
File ".../pypy/interpreter/gateway.py", line 648, in handle_exception
rstackovf.check_stack_overflow()
File ".../pypy/interpreter/gateway.py", line 700, in fastcall_0
w_result = self.fastfunc_0(space)
File ".../pypy/module/posix/interp_posix.py", line 706, in fork
run_fork_hooks('child', space)
File ".../pypy/module/posix/interp_posix.py", line 690, in run_fork_hooks
hook(space)
File ".../pypy/rpython/lltypesystem/rffi.py", line 241, in wrapper
res = call_external_function(*real_args)
File "<6063-codegen .../pypy/rpython/lltypesystem/rffi.py:168>", line 6,
in call_external_function
File ".../pypy/rpython/lltypesystem/lltype.py", line 1283, in __call__
return callb(*args)
File ".../pypy/rpython/lltypesystem/ll2ctypes.py", line 1164, in __call__
cfunc = get_ctypes_callable(self.funcptr, self.calling_conv)
File ".../pypy/rpython/lltypesystem/ll2ctypes.py", line 1138, in
get_ctypes_callable
funcname, place))
NotImplementedError: function 'PyThread_ReInitTLS' not found in library
'/tmp/usession-default-21/module_cache/pypyapi.so'
"""
I have no clue why it thinks it needs to call that function. Any idea?
Stefan
_______________________________________________
pypy-dev mailing list
pypy-dev@python.org
http://mail.python.org/mailman/listinfo/pypy-dev
