Author: Ronan Lamy <ronan.l...@gmail.com> Branch: py3k Changeset: r87601:943171b0ad54 Date: 2016-10-05 18:38 +0100 http://bitbucket.org/pypy/pypy/changeset/943171b0ad54/
Log: hg merge default diff --git a/lib-python/2.7/test/test_timeit.py b/lib-python/2.7/test/test_timeit.py --- a/lib-python/2.7/test/test_timeit.py +++ b/lib-python/2.7/test/test_timeit.py @@ -13,7 +13,7 @@ DEFAULT_NUMBER = 1000000 # timeit's default number of repetitions. -DEFAULT_REPEAT = 3 +DEFAULT_REPEAT = timeit.default_repeat # XXX: some tests are commented out that would improve the coverage but take a # long time to run because they test the default number of loops, which is @@ -204,7 +204,7 @@ t.print_exc(s) self.assert_exc_string(s.getvalue(), 'ZeroDivisionError') - MAIN_DEFAULT_OUTPUT = "10 loops, best of 3: 1 sec per loop\n" + MAIN_DEFAULT_OUTPUT = "1 loops, average of 7: 1 +- 0 sec per loop (using standard deviation)\n" def run_main(self, seconds_per_increment=1.0, switches=None, timer=None): if timer is None: @@ -230,33 +230,35 @@ def test_main_seconds(self): s = self.run_main(seconds_per_increment=5.5) - self.assertEqual(s, "10 loops, best of 3: 5.5 sec per loop\n") + self.assertIn("1 loops, average of 7: 5.5 +- 0 sec per loop (using standard deviation)\n", s) def test_main_milliseconds(self): s = self.run_main(seconds_per_increment=0.0055) - self.assertEqual(s, "100 loops, best of 3: 5.5 msec per loop\n") + self.assertIn("100 loops, average of 7: 5.5 +-", s) + self.assertIn("msec per loop", s) def test_main_microseconds(self): s = self.run_main(seconds_per_increment=0.0000025, switches=['-n100']) - self.assertEqual(s, "100 loops, best of 3: 2.5 usec per loop\n") + self.assertIn("100 loops, average of 7: 2.5", s) + self.assertIn("usec per loop", s) def test_main_fixed_iters(self): s = self.run_main(seconds_per_increment=2.0, switches=['-n35']) - self.assertEqual(s, "35 loops, best of 3: 2 sec per loop\n") + self.assertIn("35 loops, average of 7: 2 +- 0 sec per loop (using standard deviation)\n", s) def test_main_setup(self): s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s', 'print("CustomSetup")']) - self.assertEqual(s, "CustomSetup\n" * 3 + - "35 loops, best of 3: 2 sec per loop\n") + self.assertIn("CustomSetup\n" * DEFAULT_REPEAT + + "35 loops, average of 7: 2 +- 0 sec per loop (using standard deviation)\n", s) def test_main_fixed_reps(self): s = self.run_main(seconds_per_increment=60.0, switches=['-r9']) - self.assertEqual(s, "10 loops, best of 9: 60 sec per loop\n") + self.assertIn("1 loops, average of 9: 60 +- 0 sec per loop (using standard deviation)\n", s) def test_main_negative_reps(self): s = self.run_main(seconds_per_increment=60.0, switches=['-r-5']) - self.assertEqual(s, "10 loops, best of 1: 60 sec per loop\n") + self.assertIn("1 loops, average of 1: 60 +- 0 sec per loop (using standard deviation)\n", s) @unittest.skipIf(sys.flags.optimize >= 2, "need __doc__") def test_main_help(self): @@ -266,33 +268,34 @@ def test_main_using_time(self): fake_timer = FakeTimer() s = self.run_main(switches=['-t'], timer=fake_timer) - self.assertEqual(s, self.MAIN_DEFAULT_OUTPUT) + self.assertIn(self.MAIN_DEFAULT_OUTPUT, s) self.assertIs(fake_timer.saved_timer, time.time) def test_main_using_clock(self): fake_timer = FakeTimer() s = self.run_main(switches=['-c'], timer=fake_timer) - self.assertEqual(s, self.MAIN_DEFAULT_OUTPUT) + self.assertIn(self.MAIN_DEFAULT_OUTPUT, s) self.assertIs(fake_timer.saved_timer, time.clock) def test_main_verbose(self): s = self.run_main(switches=['-v']) - self.assertEqual(s, dedent("""\ - 10 loops -> 10 secs - raw times: 10 10 10 - 10 loops, best of 3: 1 sec per loop - """)) + self.assertIn(dedent("""\ + 1 loops -> 1 secs + raw times: 1 1 1 1 1 1 1 + 1 loops, average of 7: 1 +- 0 sec per loop (using standard deviation) + """), s) def test_main_very_verbose(self): s = self.run_main(seconds_per_increment=0.000050, switches=['-vv']) - self.assertEqual(s, dedent("""\ + self.assertIn(dedent("""\ + 1 loops -> 5e-05 secs 10 loops -> 0.0005 secs 100 loops -> 0.005 secs 1000 loops -> 0.05 secs 10000 loops -> 0.5 secs - raw times: 0.5 0.5 0.5 - 10000 loops, best of 3: 50 usec per loop - """)) + raw times: 0.5 0.5 0.5 0.5 0.5 0.5 0.5 + 10000 loops, average of 7: 50 +- 0 usec per loop (using standard deviation) + """), s) def test_main_exception(self): with captured_stderr() as error_stringio: @@ -304,6 +307,15 @@ s = self.run_main(switches=['-n1', '1.0/0.0']) self.assert_exc_string(error_stringio.getvalue(), 'ZeroDivisionError') + def test_main_recommends_perf(self): + s = self.run_main(seconds_per_increment=2.0, switches=['-n35', '-s', 'print("CustomSetup")']) + self.assertIn(dedent("""\ + WARNING: timeit is a very unreliable tool. use perf or something else for real measurements + pypy -m pip install perf + pypy -m perf timeit -n35 -s 'print("CustomSetup")' 'import timeit; timeit._fake_timer.inc()' + """), s) + + def test_main(): run_unittest(TestTimeit) diff --git a/lib-python/2.7/timeit.py b/lib-python/2.7/timeit.py --- a/lib-python/2.7/timeit.py +++ b/lib-python/2.7/timeit.py @@ -13,7 +13,7 @@ Options: -n/--number N: how many times to execute 'statement' (default: see below) - -r/--repeat N: how many times to repeat the timer (default 3) + -r/--repeat N: how many times to repeat the timer (default 7) -s/--setup S: statement to be executed once initially (default 'pass') -t/--time: use time.time() (default on Unix) -c/--clock: use time.clock() (default on Windows) @@ -53,6 +53,8 @@ """ import gc +import math +import os import sys import time @@ -60,7 +62,7 @@ dummy_src_name = "<timeit-src>" default_number = 1000000 -default_repeat = 3 +default_repeat = 7 if sys.platform == "win32": # On Windows, the best timer is time.clock() @@ -265,6 +267,7 @@ """ if args is None: args = sys.argv[1:] + origargs = args import getopt try: opts, args = getopt.getopt(args, "n:s:r:tcvh", @@ -281,6 +284,7 @@ repeat = default_repeat verbose = 0 precision = 3 + units = {"sec": 1, "msec": 1e3, "usec": 1e6, "ns": 1e9} for o, a in opts: if o in ("-n", "--number"): number = int(a) @@ -302,17 +306,25 @@ print __doc__, return 0 setup = "\n".join(setup) or "pass" + + print "WARNING: timeit is a very unreliable tool. use perf or something else for real measurements" + executable = os.path.basename(sys.executable) + print "%s -m pip install perf" % executable + print "%s -m perf timeit %s" % ( + executable, + " ".join([(arg if arg.startswith("-") else repr(arg)) + for arg in origargs]), ) + print "-" * 60 # Include the current directory, so that local imports work (sys.path # contains the directory of this script, rather than the current # directory) - import os sys.path.insert(0, os.curdir) if _wrap_timer is not None: timer = _wrap_timer(timer) t = Timer(stmt, setup, timer) if number == 0: # determine number so that 0.2 <= total time < 2.0 - for i in range(1, 10): + for i in range(0, 10): number = 10**i try: x = t.timeit(number) @@ -324,24 +336,34 @@ if x >= 0.2: break try: - r = t.repeat(repeat, number) + timings = t.repeat(repeat, number) except: t.print_exc() return 1 - best = min(r) if verbose: - print "raw times:", " ".join(["%.*g" % (precision, x) for x in r]) - print "%d loops," % number, - usec = best * 1e6 / number - if usec < 1000: - print "best of %d: %.*g usec per loop" % (repeat, precision, usec) - else: - msec = usec / 1000 - if msec < 1000: - print "best of %d: %.*g msec per loop" % (repeat, precision, msec) - else: - sec = msec / 1000 - print "best of %d: %.*g sec per loop" % (repeat, precision, sec) + print "raw times:", " ".join(["%.*g" % (precision, x) for x in timings]) + + timings = [dt / number for dt in timings] + + def _avg(l): + return math.fsum(l) / len(l) + def _stdev(l): + avg = _avg(l) + return (math.fsum([(x - avg) ** 2 for x in l]) / len(l)) ** 0.5 + + average = _avg(timings) + + scales = [(scale, unit) for unit, scale in units.items()] + scales.sort() + for scale, time_unit in scales: + if average * scale >= 1.0: + break + + stdev = _stdev(timings) + print("%s loops, average of %d: %.*g +- %.*g %s per loop (using standard deviation)" + % (number, repeat, + precision, average * scale, + precision, stdev * scale, time_unit)) return None if __name__ == "__main__": diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -72,6 +72,8 @@ if "cppyy" in working_modules: working_modules.remove("cppyy") # not tested on win32 + if "faulthandler" in working_modules: + working_modules.remove("faulthandler") # missing details # The _locale module is needed by site.py on Windows default_modules.add("_locale") diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -469,7 +469,12 @@ these concerns also exist on CPython, just less so. For this reason we explicitly don't implement ``sys.getsizeof()``. +* The ``timeit`` module behaves differently under PyPy: it prints the average + time and the standard deviation, instead of the minimum, since the minimum is + often misleading. + .. _`is ignored in PyPy`: http://bugs.python.org/issue14621 .. _`little point`: http://events.ccc.de/congress/2012/Fahrplan/events/5152.en.html .. _`#2072`: https://bitbucket.org/pypy/pypy/issue/2072/ + diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -61,3 +61,9 @@ .. branch: better-error-missing-self Improve the error message when the user forgot the "self" argument of a method. + + +.. fb6bb835369e +Change the ``timeit`` module: it now prints the average time and the standard +deviation over 7 runs by default, instead of the minimum. The minimum is often +misleading. diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -267,6 +267,13 @@ return sys_info.load_module(mod, name) cls.w_load_module = w_load_module + def w_debug_collect(self): + import gc + gc.collect() + gc.collect() + gc.collect() + cls.w_debug_collect = w_debug_collect + def record_imported_module(self, name): """ diff --git a/pypy/module/faulthandler/cintf.py b/pypy/module/faulthandler/cintf.py --- a/pypy/module/faulthandler/cintf.py +++ b/pypy/module/faulthandler/cintf.py @@ -42,8 +42,9 @@ pypy_faulthandler_write = direct_llexternal( 'pypy_faulthandler_write', [rffi.INT, rffi.CCHARP], lltype.Void) -pypy_faulthandler_write_int = direct_llexternal( - 'pypy_faulthandler_write_int', [rffi.INT, lltype.Signed], lltype.Void) +pypy_faulthandler_write_uint = direct_llexternal( + 'pypy_faulthandler_write_uint', [rffi.INT, lltype.Unsigned, rffi.INT], + lltype.Void) pypy_faulthandler_dump_traceback = direct_llexternal( 'pypy_faulthandler_dump_traceback', diff --git a/pypy/module/faulthandler/dumper.py b/pypy/module/faulthandler/dumper.py --- a/pypy/module/faulthandler/dumper.py +++ b/pypy/module/faulthandler/dumper.py @@ -4,7 +4,7 @@ from pypy.interpreter.pycode import PyCode from pypy.module.faulthandler.cintf import pypy_faulthandler_write -from pypy.module.faulthandler.cintf import pypy_faulthandler_write_int +from pypy.module.faulthandler.cintf import pypy_faulthandler_write_uint MAX_STRING_LENGTH = 500 @@ -24,8 +24,9 @@ global_buf[l] = '\x00' pypy_faulthandler_write(fd, global_buf) -def _dump_int(fd, i): - pypy_faulthandler_write_int(fd, i) +def _dump_nonneg_int(fd, i): + pypy_faulthandler_write_uint(fd, rffi.cast(lltype.Unsigned, i), + rffi.cast(rffi.INT, 1)) def dump_code(pycode, loc, fd): @@ -37,7 +38,7 @@ _dump(fd, '" in ') _dump(fd, pycode.co_name) _dump(fd, ", from line ") - _dump_int(fd, pycode.co_firstlineno) + _dump_nonneg_int(fd, pycode.co_firstlineno) if loc == traceback.LOC_JITTED: _dump(fd, " [jitted]") elif loc == traceback.LOC_JITTED_INLINED: diff --git a/pypy/module/faulthandler/faulthandler.c b/pypy/module/faulthandler/faulthandler.c --- a/pypy/module/faulthandler/faulthandler.c +++ b/pypy/module/faulthandler/faulthandler.c @@ -60,15 +60,53 @@ RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str) { - (void)write(fd, str, strlen(str)); + ssize_t n, count; + count = 0; + while (str[count] != 0) + count++; + + while (count > 0) { + n = write(fd, str, count); + if (n < 0) { + if (errno != EINTR) + return; /* give up */ + n = 0; + } + str += n; + count -= n; + } } RPY_EXTERN -void pypy_faulthandler_write_int(int fd, long value) +void pypy_faulthandler_write_uint(int fd, unsigned long uvalue, int min_digits) { - char buf[48]; - sprintf(buf, "%ld", value); - pypy_faulthandler_write(fd, buf); + char buf[48], *p = buf + 48; + *--p = 0; + while (uvalue || min_digits > 0) { + assert(p > buf); + *--p = '0' + (uvalue % 10UL); + uvalue /= 10UL; + min_digits--; + } + + pypy_faulthandler_write(fd, p); +} + +static void pypy_faulthandler_write_hex(int fd, unsigned long uvalue) +{ + char buf[48], *p = buf + 48; + *--p = 0; + do { + unsigned long byte = uvalue % 16UL; + assert(p > buf); + if (byte < 10) + *--p = '0' + byte; + else + *--p = 'A' + byte - 10; + uvalue /= 16UL; + } while (uvalue > 0UL); + + pypy_faulthandler_write(fd, p); } @@ -91,7 +129,6 @@ */ struct pypy_threadlocal_s *my, *p; int blankline = 0; - char buf[40]; my = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get(); p = _RPython_ThreadLocals_Head(); @@ -101,9 +138,9 @@ pypy_faulthandler_write(fd, "\n"); blankline = 1; - pypy_faulthandler_write(fd, my == p ? "Current thread" : "Thread"); - sprintf(buf, " 0x%lx", (unsigned long)p->thread_ident); - pypy_faulthandler_write(fd, buf); + pypy_faulthandler_write(fd, my == p ? "Current thread 0x" + : "Thread 0x"); + pypy_faulthandler_write_hex(fd, (unsigned long)p->thread_ident); pypy_faulthandler_write(fd, " (most recent call first):\n"); array_length = vmprof_get_traceback(p->vmprof_tl_stack, @@ -166,9 +203,9 @@ #endif RPyLockStatus st; - char buf[64]; - unsigned long hour, minutes, seconds, fraction; + unsigned long hours, minutes, seconds, fraction; long long t; + int fd; do { st = RPyThreadAcquireLockTimed(&thread_later.cancel_event, @@ -190,16 +227,21 @@ t /= 60; minutes = (unsigned long)(t % 60); t /= 60; - hour = (unsigned long)t; - if (fraction == 0) - sprintf(buf, "Timeout (%lu:%02lu:%02lu)!\n", - hour, minutes, seconds); - else - sprintf(buf, "Timeout (%lu:%02lu:%02lu.%06lu)!\n", - hour, minutes, seconds, fraction); + hours = (unsigned long)t; - pypy_faulthandler_write(thread_later.fd, buf); - pypy_faulthandler_dump_traceback(thread_later.fd, 1, NULL); + fd = thread_later.fd; + pypy_faulthandler_write(fd, "Timeout ("); + pypy_faulthandler_write_uint(fd, hours, 1); + pypy_faulthandler_write(fd, ":"); + pypy_faulthandler_write_uint(fd, minutes, 2); + pypy_faulthandler_write(fd, ":"); + pypy_faulthandler_write_uint(fd, seconds, 2); + if (fraction != 0) { + pypy_faulthandler_write(fd, "."); + pypy_faulthandler_write_uint(fd, fraction, 6); + } + pypy_faulthandler_write(fd, ")!\n"); + pypy_faulthandler_dump_traceback(fd, 1, NULL); if (thread_later.exit) _exit(1); diff --git a/pypy/module/faulthandler/faulthandler.h b/pypy/module/faulthandler/faulthandler.h --- a/pypy/module/faulthandler/faulthandler.h +++ b/pypy/module/faulthandler/faulthandler.h @@ -16,7 +16,8 @@ RPY_EXTERN int pypy_faulthandler_is_enabled(void); RPY_EXTERN void pypy_faulthandler_write(int fd, const char *str); -RPY_EXTERN void pypy_faulthandler_write_int(int fd, long value); +RPY_EXTERN void pypy_faulthandler_write_uint(int fd, unsigned long value, + int min_digits); RPY_EXTERN void pypy_faulthandler_dump_traceback(int fd, int all_threads, void *ucontext); diff --git a/pypy/module/micronumpy/concrete.py b/pypy/module/micronumpy/concrete.py --- a/pypy/module/micronumpy/concrete.py +++ b/pypy/module/micronumpy/concrete.py @@ -4,6 +4,7 @@ from rpython.rlib.listsort import make_timsort_class from rpython.rlib.buffer import Buffer from rpython.rlib.debug import make_sure_not_resized +from rpython.rlib.rstring import StringBuilder from rpython.rlib.rawstorage import alloc_raw_storage, free_raw_storage, \ raw_storage_getitem, raw_storage_setitem, RAW_STORAGE from rpython.rtyper.lltypesystem import rffi, lltype, llmemory @@ -725,7 +726,9 @@ return rffi.ptradd(self.impl.storage, self.impl.start) def getformat(self): - return self.impl.dtype.char + sb = StringBuilder() + self.impl.dtype.getformat(sb) + return sb.build() def getitemsize(self): return self.impl.dtype.elsize diff --git a/pypy/module/micronumpy/descriptor.py b/pypy/module/micronumpy/descriptor.py --- a/pypy/module/micronumpy/descriptor.py +++ b/pypy/module/micronumpy/descriptor.py @@ -24,7 +24,6 @@ return space.interp_w( W_Dtype, space.call_function(space.gettypefor(W_Dtype), w_dtype)) - @jit.unroll_safe def dtype_agreement(space, w_arr_list, shape, out=None): """ agree on dtype from a list of arrays. if out is allocated, @@ -179,6 +178,38 @@ assert dtype.is_float() return dtype + def getformat(self, stringbuilder): + # adapted from _buffer_format_string in multiarray/buffer.c + # byte-order not supported yet + if self.is_record(): + #subs = sorted(self.fields.items(), key=lambda (k,v): v[0]) + subs = [] + for name in self.fields: + offset, dtyp = self.fields[name] + i = 0 + for i in range(len(subs)): + if offset < subs[i][0]: + break + else: + i = len(subs) + subs.insert(i, (offset, dtyp, name)) + start = 0 + stringbuilder.append('T{') + for s in subs: + stringbuilder.append('x' * (s[0] - start)) + start = s[0] + s[1].elsize + s[1].getformat(stringbuilder) + stringbuilder.append(':') + stringbuilder.append(s[2]) + stringbuilder.append(':') + stringbuilder.append('}') + else: + if self.byteorder == NPY.OPPBYTE: + raise oefmt(self.itemtype.space.w_NotImplementedError, + "non-native byte order not supported yet") + # even if not, NumPy adds a '=', '@', for 'i' types + stringbuilder.append(self.char) + def get_name(self): name = self.w_box_type.getname(self.itemtype.space) if name.endswith('_'): @@ -744,6 +775,9 @@ offsets[j+1] = delta + offsets[j] if i + 1 < len(offsets) and offsets[i + 1] == 0: offsets[i + 1] = offsets[i] + max(delta, subdtype.elsize) + # sanity check + if offsets[i] % maxalign: + offsets[i] = ((offsets[i] // maxalign) + 1) * maxalign elif not use_supplied_offsets: if i + 1 < len(offsets) and offsets[i + 1] == 0: offsets[i+1] = offsets[i] + subdtype.elsize diff --git a/pypy/module/micronumpy/test/test_ndarray.py b/pypy/module/micronumpy/test/test_ndarray.py --- a/pypy/module/micronumpy/test/test_ndarray.py +++ b/pypy/module/micronumpy/test/test_ndarray.py @@ -3659,6 +3659,28 @@ assert z.dtype == 'O' assert (z == x).all() + dt1 = np.dtype( + [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], + align=True) + x = np.arange(dt1.itemsize, dtype=np.int8).view(dt1) + y = memoryview(x) + if '__pypy__' in sys.builtin_module_names: + assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}' + else: + assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx@i:c:}' + + + dt1 = np.dtype( + [('a', 'b'), ('b', 'i'), ('sub', np.dtype('b,i')), ('c', 'i')], + align=True) + x = np.arange(dt1.itemsize, dtype=np.int8).view(dt1) + y = memoryview(x) + if '__pypy__' in sys.builtin_module_names: + assert y.format == 'T{b:a:xxxi:b:T{b:f0:i:f1:}:sub:xxxi:c:}' + else: + assert y.format == 'T{b:a:xxxi:b:T{b:f0:=i:f1:}:sub:xxx@i:c:}' + + def test_fromstring(self): import sys from numpy import fromstring, dtype diff --git a/rpython/rlib/rvmprof/src/rvmprof.h b/rpython/rlib/rvmprof/src/rvmprof.h --- a/rpython/rlib/rvmprof/src/rvmprof.h +++ b/rpython/rlib/rvmprof/src/rvmprof.h @@ -1,4 +1,8 @@ -#include <stdint.h> +#ifdef _WIN32 +typedef long intptr_t; +#else +# include <stdint.h> +#endif RPY_EXTERN char *vmprof_init(int, double, char *); RPY_EXTERN void vmprof_ignore_signals(int); diff --git a/rpython/rlib/rvmprof/src/vmprof_common.h b/rpython/rlib/rvmprof/src/vmprof_common.h --- a/rpython/rlib/rvmprof/src/vmprof_common.h +++ b/rpython/rlib/rvmprof/src/vmprof_common.h @@ -134,7 +134,11 @@ intptr_t *result_p, intptr_t result_length) { int n; +#ifdef _WIN32 + intptr_t pc = 0; /* XXX implement me */ +#else intptr_t pc = ucontext ? GetPC((ucontext_t *)ucontext) : 0; +#endif if (stack == NULL) stack = get_vmprof_stack(); n = get_stack_trace(stack, result_p, result_length - 2, pc); _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit