Author: Manuel Jacob <m...@manueljacob.de> Branch: py3.6 Changeset: r93320:f04d4604c7e3 Date: 2017-12-09 03:14 +0100 http://bitbucket.org/pypy/pypy/changeset/f04d4604c7e3/
Log: hg merge py3.5 (+ fixes) I'm not 100% sure about the merge in test_dis.py, but most of the tests are failing anyway. diff too long, truncating to 2000 out of 12565 lines diff --git a/.hgignore b/.hgignore --- a/.hgignore +++ b/.hgignore @@ -59,6 +59,7 @@ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.h$ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.log$ ^rpython/rlib/rvmprof/src/shared/libbacktrace/config.status$ +^pypy/tool/dest$ ^pypy/goal/pypy-translation-snapshot$ ^pypy/goal/pypy-c ^pypy/goal/pypy3-c diff --git a/_pytest/terminal.py b/_pytest/terminal.py --- a/_pytest/terminal.py +++ b/_pytest/terminal.py @@ -366,11 +366,11 @@ EXIT_OK, EXIT_TESTSFAILED, EXIT_INTERRUPTED, EXIT_USAGEERROR, EXIT_NOTESTSCOLLECTED) if exitstatus in summary_exit_codes: - self.config.hook.pytest_terminal_summary(terminalreporter=self) self.summary_errors() self.summary_failures() self.summary_warnings() self.summary_passes() + self.config.hook.pytest_terminal_summary(terminalreporter=self) if exitstatus == EXIT_INTERRUPTED: self._report_keyboardinterrupt() del self._keyboardinterrupt_memo diff --git a/extra_tests/requirements.txt b/extra_tests/requirements.txt new file mode 100644 --- /dev/null +++ b/extra_tests/requirements.txt @@ -0,0 +1,2 @@ +pytest +hypothesis diff --git a/extra_tests/test_bytes.py b/extra_tests/test_bytes.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_bytes.py @@ -0,0 +1,84 @@ +from hypothesis import strategies as st +from hypothesis import given, example + +st_bytestring = st.binary() | st.binary().map(bytearray) + +@given(st_bytestring, st_bytestring, st_bytestring) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st_bytestring, st_bytestring, st_bytestring) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st_bytestring, st_bytestring, st_bytestring) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st_bytestring, st_bytestring, st_bytestring) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + +@given(st_bytestring, st_bytestring) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + +@example(b'x', b'', 1) +@example(b'x', b'', 2) +@given(st_bytestring, st_bytestring, st.integers()) +def test_startswith_start(u, v, start): + expected = u[start:].startswith(v) if v else (start <= len(u)) + assert u.startswith(v, start) is expected + +@example(b'x', b'', 1, 0) +@example(b'xx', b'', -1, 0) +@given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + +@given(st_bytestring, st_bytestring) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + +@example(b'x', b'', 1) +@example(b'x', b'', 2) +@given(st_bytestring, st_bytestring, st.integers()) +def test_endswith_2(u, v, start): + expected = u[start:].endswith(v) if v else (start <= len(u)) + assert u.endswith(v, start) is expected + +@example(b'x', b'', 1, 0) +@example(b'xx', b'', -1, 0) +@given(st_bytestring, st_bytestring, st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/extra_tests/test_textio.py b/extra_tests/test_textio.py new file mode 100644 --- /dev/null +++ b/extra_tests/test_textio.py @@ -0,0 +1,48 @@ +from hypothesis import given, strategies as st + +from io import BytesIO, TextIOWrapper +import os + +def translate_newlines(text): + text = text.replace('\r\n', '\n') + text = text.replace('\r', '\n') + return text.replace('\n', os.linesep) + +@st.composite +def st_readline_universal( + draw, st_nlines=st.integers(min_value=0, max_value=10)): + n_lines = draw(st_nlines) + lines = draw(st.lists( + st.text(st.characters(blacklist_characters='\r\n')), + min_size=n_lines, max_size=n_lines)) + limits = [] + for line in lines: + limit = draw(st.integers(min_value=0, max_value=len(line) + 5)) + limits.append(limit) + limits.append(-1) + endings = draw(st.lists( + st.sampled_from(['\n', '\r', '\r\n']), + min_size=n_lines, max_size=n_lines)) + return ( + ''.join(line + ending for line, ending in zip(lines, endings)), + limits) + +@given(data=st_readline_universal(), + mode=st.sampled_from(['\r', '\n', '\r\n', '', None])) +def test_readline(data, mode): + txt, limits = data + textio = TextIOWrapper( + BytesIO(txt.encode('utf-8', 'surrogatepass')), + encoding='utf-8', errors='surrogatepass', newline=mode) + lines = [] + for limit in limits: + line = textio.readline(limit) + if limit >= 0: + assert len(line) <= limit + if line: + lines.append(line) + elif limit: + break + if mode is None: + txt = translate_newlines(txt) + assert txt.startswith(u''.join(lines)) diff --git a/extra_tests/test_unicode.py b/extra_tests/test_unicode.py --- a/extra_tests/test_unicode.py +++ b/extra_tests/test_unicode.py @@ -1,3 +1,4 @@ +import sys import pytest from hypothesis import strategies as st from hypothesis import given, settings, example @@ -32,3 +33,89 @@ @given(s=st.text()) def test_composition(s, norm1, norm2, norm3): assert normalize(norm2, normalize(norm1, s)) == normalize(norm3, s) + +@given(st.text(), st.text(), st.text()) +def test_find(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.find(u) <= len(prefix) + assert s.find(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st.text(), st.text(), st.text()) +def test_index(u, prefix, suffix): + s = prefix + u + suffix + assert 0 <= s.index(u) <= len(prefix) + assert s.index(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st.text(), st.text(), st.text()) +def test_rfind(u, prefix, suffix): + s = prefix + u + suffix + assert s.rfind(u) >= len(prefix) + assert s.rfind(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +@given(st.text(), st.text(), st.text()) +def test_rindex(u, prefix, suffix): + s = prefix + u + suffix + assert s.rindex(u) >= len(prefix) + assert s.rindex(u, len(prefix), len(s) - len(suffix)) == len(prefix) + +def adjust_indices(u, start, end): + if end < 0: + end = max(end + len(u), 0) + else: + end = min(end, len(u)) + if start < 0: + start = max(start + len(u), 0) + return start, end + +@given(st.text(), st.text()) +def test_startswith_basic(u, v): + assert u.startswith(v) is (u[:len(v)] == v) + +@example(u'x', u'', 1) +@example(u'x', u'', 2) +@given(st.text(), st.text(), st.integers()) +def test_startswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].startswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.startswith(v, start) is expected + +@example(u'x', u'', 1, 0) +@example(u'xx', u'', -1, 0) +@given(st.text(), st.text(), st.integers(), st.integers()) +def test_startswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].startswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.startswith(v, start, end) is expected + +@given(st.text(), st.text()) +def test_endswith_basic(u, v): + if len(v) > len(u): + assert u.endswith(v) is False + else: + assert u.endswith(v) is (u[len(u) - len(v):] == v) + +@example(u'x', u'', 1) +@example(u'x', u'', 2) +@given(st.text(), st.text(), st.integers()) +def test_endswith_2(u, v, start): + if v or sys.version_info[0] == 2: + expected = u[start:].endswith(v) + else: # CPython leaks implementation details in this case + expected = start <= len(u) + assert u.endswith(v, start) is expected + +@example(u'x', u'', 1, 0) +@example(u'xx', u'', -1, 0) +@given(st.text(), st.text(), st.integers(), st.integers()) +def test_endswith_3(u, v, start, end): + if v or sys.version_info[0] == 2: + expected = u[start:end].endswith(v) + else: # CPython leaks implementation details in this case + start0, end0 = adjust_indices(u, start, end) + expected = start0 <= len(u) and start0 <= end0 + assert u.endswith(v, start, end) is expected diff --git a/lib-python/2.7/ctypes/__init__.py b/lib-python/2.7/ctypes/__init__.py --- a/lib-python/2.7/ctypes/__init__.py +++ b/lib-python/2.7/ctypes/__init__.py @@ -360,14 +360,15 @@ self._FuncPtr = _FuncPtr if handle is None: - if flags & _FUNCFLAG_CDECL: - pypy_dll = _ffi.CDLL(name, mode) - else: - pypy_dll = _ffi.WinDLL(name, mode) - self.__pypy_dll__ = pypy_dll - handle = int(pypy_dll) - if _sys.maxint > 2 ** 32: - handle = int(handle) # long -> int + handle = 0 + if flags & _FUNCFLAG_CDECL: + pypy_dll = _ffi.CDLL(name, mode, handle) + else: + pypy_dll = _ffi.WinDLL(name, mode, handle) + self.__pypy_dll__ = pypy_dll + handle = int(pypy_dll) + if _sys.maxint > 2 ** 32: + handle = int(handle) # long -> int self._handle = handle def __repr__(self): diff --git a/lib-python/2.7/inspect.py b/lib-python/2.7/inspect.py --- a/lib-python/2.7/inspect.py +++ b/lib-python/2.7/inspect.py @@ -40,6 +40,10 @@ import linecache from operator import attrgetter from collections import namedtuple +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # These constants are from Include/code.h. CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS = 0x1, 0x2, 0x4, 0x8 @@ -230,7 +234,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" diff --git a/lib-python/2.7/test/test_urllib2net.py b/lib-python/2.7/test/test_urllib2net.py --- a/lib-python/2.7/test/test_urllib2net.py +++ b/lib-python/2.7/test/test_urllib2net.py @@ -286,7 +286,7 @@ self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120) u.close() - FTP_HOST = 'ftp://ftp.debian.org/debian/' + FTP_HOST = 'ftp://www.pythontest.net/' def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) diff --git a/lib-python/2.7/warnings.py b/lib-python/2.7/warnings.py --- a/lib-python/2.7/warnings.py +++ b/lib-python/2.7/warnings.py @@ -43,11 +43,12 @@ unicodetype = unicode except NameError: unicodetype = () + template = "%s: %s: %s\n" try: message = str(message) except UnicodeEncodeError: - pass - s = "%s: %s: %s\n" % (lineno, category.__name__, message) + template = unicode(template) + s = template % (lineno, category.__name__, message) line = linecache.getline(filename, lineno) if line is None else line if line: line = line.strip() diff --git a/lib-python/3/doctest.py b/lib-python/3/doctest.py --- a/lib-python/3/doctest.py +++ b/lib-python/3/doctest.py @@ -948,6 +948,8 @@ elif inspect.getmodule(object) is not None: return module is inspect.getmodule(object) elif inspect.isfunction(object): + if isinstance(object.__code__, inspect._builtin_code_type): + return True # XXX: A PyPy builtin - no way to tell return module.__dict__ is object.__globals__ elif inspect.ismethoddescriptor(object): if hasattr(object, '__objclass__'): diff --git a/lib-python/3/idlelib/CallTips.py b/lib-python/3/idlelib/CallTips.py --- a/lib-python/3/idlelib/CallTips.py +++ b/lib-python/3/idlelib/CallTips.py @@ -123,6 +123,15 @@ _first_param = re.compile(r'(?<=\()\w*\,?\s*') _default_callable_argspec = "See source or doc" +def _is_user_method(ob): + """Detect user methods on PyPy""" + return (isinstance(ob, types.MethodType) and + isinstance(ob.__code__, types.CodeType)) + +def _is_user_function(ob): + """Detect user methods on PyPy""" + return (isinstance(ob, types.FunctionType) and + isinstance(ob.__code__, types.CodeType)) def get_argspec(ob): '''Return a string describing the signature of a callable object, or ''. @@ -140,21 +149,21 @@ return argspec if isinstance(ob, type): fob = ob.__init__ - elif isinstance(ob_call, types.MethodType): + elif _is_user_method(ob_call): fob = ob_call else: fob = ob if (isinstance(fob, (types.FunctionType, types.MethodType)) and hasattr(fob.__code__, 'co_code')): # PyPy: not on <builtin-code> argspec = inspect.formatargspec(*inspect.getfullargspec(fob)) - if (isinstance(ob, (type, types.MethodType)) or - isinstance(ob_call, types.MethodType)): + if (_is_user_method(ob) or _is_user_method(ob_call) or + (isinstance(ob, type) and _is_user_function(fob))): argspec = _first_param.sub("", argspec) lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT) if len(argspec) > _MAX_COLS else [argspec] if argspec else []) - if isinstance(ob_call, types.MethodType): + if _is_user_method(ob_call): doc = ob_call.__doc__ else: doc = getattr(ob, "__doc__", "") diff --git a/lib-python/3/idlelib/idle_test/test_calltips.py b/lib-python/3/idlelib/idle_test/test_calltips.py --- a/lib-python/3/idlelib/idle_test/test_calltips.py +++ b/lib-python/3/idlelib/idle_test/test_calltips.py @@ -63,7 +63,7 @@ gtest([].append, append_doc) gtest(List.append, append_doc) - gtest(types.MethodType, "method(function, instance)") + gtest(types.MethodType, "instancemethod(function, instance, class)") gtest(SB(), default_tip) def test_signature_wrap(self): diff --git a/lib-python/3/inspect.py b/lib-python/3/inspect.py --- a/lib-python/3/inspect.py +++ b/lib-python/3/inspect.py @@ -49,6 +49,10 @@ import builtins from operator import attrgetter from collections import namedtuple, OrderedDict +try: + from cpyext import is_cpyext_function as _is_cpyext_function +except ImportError: + _is_cpyext_function = lambda obj: False # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication @@ -274,7 +278,7 @@ __doc__ documentation string __name__ original name of this function or method __self__ instance to which a method is bound, or None""" - return isinstance(object, types.BuiltinFunctionType) + return isinstance(object, types.BuiltinFunctionType) or _is_cpyext_function(object) def isroutine(object): """Return true if the object is any kind of function or method.""" @@ -1823,7 +1827,7 @@ kwdefaults = getattr(obj, '__kwdefaults__', _void) # ... and not None here annotations = getattr(obj, '__annotations__', None) - return (isinstance(code, types.CodeType) and + return (isinstance(code, (types.CodeType, _builtin_code_type)) and isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and (kwdefaults is None or isinstance(kwdefaults, dict)) and @@ -2077,8 +2081,6 @@ s = getattr(func, "__text_signature__", None) if not s: - if func is object: # XXX PyPy hack until we support __text_signature__ - return '()' # in the same cases as CPython raise ValueError("no signature found for builtin {!r}".format(func)) return _signature_fromstr(cls, func, s, skip_bound_arg) diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py --- a/lib-python/3/test/test_capi.py +++ b/lib-python/3/test/test_capi.py @@ -31,8 +31,9 @@ skips = [] if support.check_impl_detail(pypy=True): skips += [ - 'test_widechar', - ] + 'test_lazy_hash_inheritance', + 'test_capsule', + ] def testfunction(self): """some doc""" @@ -55,6 +56,8 @@ self.assertEqual(testfunction.attribute, "test") self.assertRaises(AttributeError, setattr, inst.testfunction, "attribute", "test") + @unittest.skipIf(support.check_impl_detail(pypy=True), + "doesn't crash on PyPy") @unittest.skipUnless(threading, 'Threading required for this test.') def test_no_FatalError_infinite_loop(self): with support.SuppressCrashReport(): @@ -207,9 +210,9 @@ else: with self.assertRaises(SystemError) as cm: _testcapi.return_null_without_error() + # PyPy change: different message self.assertRegex(str(cm.exception), - 'return_null_without_error.* ' - 'returned NULL without setting an error') + 'Function returned a NULL result without setting an exception') def test_return_result_with_error(self): # Issue #23571: A function must not return a result with an error set @@ -239,9 +242,9 @@ else: with self.assertRaises(SystemError) as cm: _testcapi.return_result_with_error() + # PyPy change: different message self.assertRegex(str(cm.exception), - 'return_result_with_error.* ' - 'returned a result with an error set') + 'An exception was set, but function returned a value') def test_buildvalue_N(self): _testcapi.test_buildvalue_N() @@ -329,6 +332,8 @@ self.pendingcalls_wait(l, n) +@unittest.skipIf(support.check_impl_detail(pypy=True), + "subinterpreters not implemented on PyPy") class SubinterpreterTest(unittest.TestCase): def test_subinterps(self): diff --git a/lib-python/3/test/test_cmd_line_script.py b/lib-python/3/test/test_cmd_line_script.py --- a/lib-python/3/test/test_cmd_line_script.py +++ b/lib-python/3/test/test_cmd_line_script.py @@ -43,11 +43,7 @@ _loader = __loader__ if __loader__ is BuiltinImporter else type(__loader__) print('__loader__==%a' % _loader) print('__file__==%a' % __file__) -if __cached__ is not None: - # XXX: test_script_compiled on PyPy - assertEqual(__file__, __cached__) - if not __cached__.endswith(('pyc', 'pyo')): - raise AssertionError('has __cached__ but not compiled') +print('__cached__==%a' % __cached__) print('__package__==%r' % __package__) # Check PEP 451 details import os.path @@ -238,9 +234,8 @@ def test_basic_script(self): with support.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') - package = '' if support.check_impl_detail(pypy=True) else None self._check_script(script_name, script_name, script_name, - script_dir, package, + script_dir, None, importlib.machinery.SourceFileLoader) def test_script_compiled(self): @@ -249,9 +244,8 @@ py_compile.compile(script_name, doraise=True) os.remove(script_name) pyc_file = support.make_legacy_pyc(script_name) - package = '' if support.check_impl_detail(pypy=True) else None self._check_script(pyc_file, pyc_file, - pyc_file, script_dir, package, + pyc_file, script_dir, None, importlib.machinery.SourcelessFileLoader) def test_directory(self): diff --git a/lib-python/3/test/test_cprofile.py b/lib-python/3/test/test_cprofile.py --- a/lib-python/3/test/test_cprofile.py +++ b/lib-python/3/test/test_cprofile.py @@ -1,7 +1,7 @@ """Test suite for the cProfile module.""" import sys -from test.support import run_unittest, TESTFN, unlink +from test.support import run_unittest, TESTFN, unlink, cpython_only # rip off all interesting stuff from test_profile import cProfile @@ -17,6 +17,7 @@ return _ProfileOutput # Issue 3895. + @cpython_only def test_bad_counter_during_dealloc(self): import _lsprof # Must use a file as StringIO doesn't trigger the bug. diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py --- a/lib-python/3/test/test_descr.py +++ b/lib-python/3/test/test_descr.py @@ -4278,7 +4278,10 @@ c = C() c.__dict__[Evil()] = 0 - self.assertEqual(c.attr, 1) + try: + self.assertEqual(c.attr, 1) + except AttributeError: # when Evil.__eq__ is called twice + pass # this makes a crash more likely: support.gc_collect() self.assertNotHasAttr(c, 'attr') diff --git a/lib-python/3/test/test_dis.py b/lib-python/3/test/test_dis.py --- a/lib-python/3/test/test_dis.py +++ b/lib-python/3/test/test_dis.py @@ -146,24 +146,26 @@ 1) pass +# PyPy change: JUMP_IF_NOT_DEBUG dis_bug1333982 = """\ -%3d 0 LOAD_CONST 1 (0) - 2 POP_JUMP_IF_TRUE 26 - 4 LOAD_GLOBAL 0 (AssertionError) - 6 LOAD_CONST 2 (<code object <listcomp> at 0x..., file "%s", line %d>) - 8 LOAD_CONST 3 ('bug1333982.<locals>.<listcomp>') - 10 MAKE_FUNCTION 0 - 12 LOAD_FAST 0 (x) - 14 GET_ITER - 16 CALL_FUNCTION 1 +%3d 0 JUMP_IF_NOT_DEBUG 26 (to 28) + 2 LOAD_CONST 1 (0) + 4 POP_JUMP_IF_TRUE 28 + 6 LOAD_GLOBAL 0 (AssertionError) + 8 LOAD_CONST 2 (<code object <listcomp> at 0x..., file "%s", line %d>) + 10 LOAD_CONST 3 ('bug1333982.<locals>.<listcomp>') + 12 MAKE_FUNCTION 0 + 14 LOAD_FAST 0 (x) + 16 GET_ITER + 18 CALL_FUNCTION 1 -%3d 18 LOAD_CONST 4 (1) - 20 BINARY_ADD - 22 CALL_FUNCTION 1 - 24 RAISE_VARARGS 1 +%3d 20 LOAD_CONST 4 (1) + 22 BINARY_ADD + 24 CALL_FUNCTION 1 + 26 RAISE_VARARGS 1 -%3d >> 26 LOAD_CONST 0 (None) - 28 RETURN_VALUE +%3d >> 28 LOAD_CONST 0 (None) + 30 RETURN_VALUE """ % (bug1333982.__code__.co_firstlineno + 1, __file__, bug1333982.__code__.co_firstlineno + 1, diff --git a/lib-python/3/test/test_doctest.py b/lib-python/3/test/test_doctest.py --- a/lib-python/3/test/test_doctest.py +++ b/lib-python/3/test/test_doctest.py @@ -660,7 +660,7 @@ >>> import builtins >>> tests = doctest.DocTestFinder().find(builtins) - >>> lo, hi = (120, 140) if is_pypy else (790, 810) + >>> lo, hi = (420, 440) if is_pypy else (790, 810) >>> lo < len(tests) < hi # approximate number of objects with docstrings True >>> real_tests = [t for t in tests if len(t.examples) > 0] diff --git a/lib-python/3/test/test_inspect.py b/lib-python/3/test/test_inspect.py --- a/lib-python/3/test/test_inspect.py +++ b/lib-python/3/test/test_inspect.py @@ -32,6 +32,8 @@ from test import support from test.test_import import _ready_to_import +if check_impl_detail(): + import _pickle # Functions tested in this suite: @@ -372,6 +374,7 @@ self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), 'The automatic gainsaying.') + @cpython_only # XXX: _finddoc() is broken on PyPy, but getdoc() seems OK @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") def test_finddoc(self): finddoc = inspect._finddoc @@ -765,21 +768,23 @@ @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullargspec_builtin_methods(self): - import _pickle - self.assertFullArgSpecEquals(_pickle.Pickler.dump, - args_e=['self', 'obj'], formatted='(self, obj)') - - self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, - args_e=['self', 'obj'], formatted='(self, obj)') + if check_impl_detail(): + self.assertFullArgSpecEquals(_pickle.Pickler.dump, + args_e=['self', 'obj'], formatted='(self, obj)') + + self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, + args_e=['self', 'obj'], formatted='(self, obj)') + + # platform-dependent on PyPy + default_fd = os.stat.__kwdefaults__['dir_fd'] self.assertFullArgSpecEquals( os.stat, args_e=['path'], kwonlyargs_e=['dir_fd', 'follow_symlinks'], - kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}, - formatted='(path, *, dir_fd=None, follow_symlinks=True)') - - @cpython_only + kwonlydefaults_e={'dir_fd': default_fd, 'follow_symlinks': True}, + formatted='(path, *, dir_fd={}, follow_symlinks=True)'.format(default_fd)) + @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullagrspec_builtin_func(self): @@ -788,7 +793,6 @@ spec = inspect.getfullargspec(builtin) self.assertEqual(spec.defaults[0], 'avocado') - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_getfullagrspec_builtin_func_no_signature(self): @@ -826,7 +830,9 @@ attrs = attrs_wo_objs(A) - self.assertIn(('__new__', 'method', object), attrs, 'missing __new__') + # changed in PyPy + self.assertIn(('__new__', 'static method', object), attrs, 'missing __new__') + self.assertIn(('__init__', 'method', object), attrs, 'missing __init__') self.assertIn(('s', 'static method', A), attrs, 'missing static method') @@ -1969,12 +1975,10 @@ ('kwargs', ..., int, "var_keyword")), ...)) - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtins(self): import _testcapi - import _pickle def test_unbound_method(o): """Use this to test unbound methods (things that should have a self)""" @@ -2008,9 +2012,10 @@ # normal method # (PyMethodDescr_Type, "method_descriptor") - test_unbound_method(_pickle.Pickler.dump) - d = _pickle.Pickler(io.StringIO()) - test_callable(d.dump) + if check_impl_detail(): + test_unbound_method(_pickle.Pickler.dump) + d = _pickle.Pickler(io.StringIO()) + test_callable(d.dump) # static method test_callable(str.maketrans) @@ -2031,7 +2036,7 @@ # This doesn't work now. # (We don't have a valid signature for "type" in 3.4) - with self.assertRaisesRegex(ValueError, "no signature found"): + with self.assertRaisesRegex(ValueError, "signature"): class ThisWorksNow: __call__ = type test_callable(ThisWorksNow()) @@ -2043,7 +2048,6 @@ # Regression test for issue #20586 test_callable(_testcapi.docstring_with_signature_but_no_doc) - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_decorated_builtins(self): @@ -2066,7 +2070,6 @@ follow_wrapped=False), inspect.signature(wrapper_like)) - @cpython_only def test_signature_on_builtins_no_signature(self): import _testcapi with self.assertRaisesRegex(ValueError, @@ -2642,10 +2645,10 @@ with self.assertRaisesRegex(ValueError, "callable.*is not supported"): self.assertEqual(inspect.signature(D), None) + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_on_builtin_class(self): - import _pickle self.assertEqual(str(inspect.signature(_pickle.Pickler)), '(file, protocol=None, fix_imports=True)') @@ -2891,10 +2894,10 @@ foo_sig = MySignature.from_callable(foo) self.assertTrue(isinstance(foo_sig, MySignature)) + @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_signature_from_callable_builtin_obj(self): - import _pickle class MySignature(inspect.Signature): pass sig = MySignature.from_callable(_pickle.Pickler) self.assertTrue(isinstance(sig, MySignature)) @@ -3453,7 +3456,6 @@ # This test case provides a home for checking that particular APIs # have signatures available for introspection - @cpython_only @unittest.skipIf(MISSING_C_DOCSTRINGS, "Signature information for builtins requires docstrings") def test_builtins_have_signatures(self): diff --git a/lib-python/3/test/test_io.py b/lib-python/3/test/test_io.py --- a/lib-python/3/test/test_io.py +++ b/lib-python/3/test/test_io.py @@ -1202,12 +1202,7 @@ b = bytearray(2*buffer_size) self.assertEqual(bufio.peek(3), b'fgh') self.assertEqual(rawio._reads, 3) - self.assertEqual(bufio.readinto1(b), 6) # fails because of - # an apparent inconsistency in CPython: readinto1(), if the - # buffered amount is smaller, would always issue one raw read() - # call. This differs from read1(), which if the buffered amount - # if smaller (but more than zero), would just return it without - # any raw read() call. In PyPy both have the behavior of read1(). + self.assertEqual(bufio.readinto1(b), 6) self.assertEqual(b[:6], b"fghjkl") self.assertEqual(rawio._reads, 4) diff --git a/lib-python/3/test/test_pydoc.py b/lib-python/3/test/test_pydoc.py --- a/lib-python/3/test/test_pydoc.py +++ b/lib-python/3/test/test_pydoc.py @@ -143,7 +143,7 @@ <tr bgcolor="#aa55cc"> <td colspan=3 valign=bottom> <br> <font color="#ffffff" face="helvetica, arial"><big><strong>Modules</strong></big></font></td></tr> - +\x20\x20\x20\x20 <tr><td bgcolor="#aa55cc"><tt> </tt></td><td> </td> <td width="100%%"><table width="100%%" summary="list"><tr><td width="25%%" valign=top><a href="builtins.html">builtins</a><br> </td><td width="25%%" valign=top></td><td width="25%%" valign=top></td><td width="25%%" valign=top></td></tr></table></td></tr></table><p> @@ -883,7 +883,7 @@ @requires_docstrings def test_unbound_builtin_method(self): self.assertEqual(self._get_summary_line(pickle.Pickler.dump), - "dump(self, obj, /)") + "dump(self, obj)") # these no longer include "self" def test_bound_python_method(self): @@ -912,13 +912,13 @@ s = StringIO() p = pickle.Pickler(s) self.assertEqual(self._get_summary_line(p.dump), - "dump(obj, /) method of _pickle.Pickler instance") + "dump(obj) method of pickle._Pickler instance") # this should *never* include self! @requires_docstrings def test_module_level_callable(self): self.assertEqual(self._get_summary_line(os.stat), - "stat(path, *, dir_fd=None, follow_symlinks=True)") + "stat(path, *, dir_fd=-100, follow_symlinks=True)") @unittest.skipUnless(threading, 'Threading required for this test.') diff --git a/lib-python/3/test/test_tracemalloc.py b/lib-python/3/test/test_tracemalloc.py --- a/lib-python/3/test/test_tracemalloc.py +++ b/lib-python/3/test/test_tracemalloc.py @@ -1,7 +1,6 @@ import contextlib import os import sys -import tracemalloc import unittest from unittest.mock import patch from test.support.script_helper import (assert_python_ok, assert_python_failure, @@ -17,6 +16,11 @@ _testcapi = None +try: + import tracemalloc +except ImportError: + raise unittest.SkipTest("tracemalloc is required") + EMPTY_STRING_SIZE = sys.getsizeof(b'') diff --git a/lib-python/3/traceback.py b/lib-python/3/traceback.py --- a/lib-python/3/traceback.py +++ b/lib-python/3/traceback.py @@ -566,8 +566,8 @@ yield ' {}\n'.format(badline.strip()) if offset is not None: caretspace = badline.rstrip('\n') - offset = min(len(caretspace), offset) - 1 - caretspace = caretspace[:offset].lstrip() + # bug in CPython: the case offset==0 is mishandled + caretspace = caretspace[:offset].lstrip()[:-1] # non-space whitespace (likes tabs) must be kept for alignment caretspace = ((c.isspace() and c or ' ') for c in caretspace) yield ' {}^\n'.format(''.join(caretspace)) diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/cryptography.py @@ -48,6 +48,9 @@ #else #define CRYPTOGRAPHY_IS_LIBRESSL 0 #endif + +#define CRYPTOGRAPHY_LIBRESSL_251_OR_GREATER \ + (CRYPTOGRAPHY_IS_LIBRESSL && LIBRESSL_VERSION_NUMBER >= 0x20501000) """ TYPES = """ diff --git a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py --- a/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py +++ b/lib_pypy/_cffi_ssl/_cffi_src/openssl/x509_vfy.py @@ -244,6 +244,14 @@ static const long X509_V_FLAG_SUITEB_192_LOS = 0; static const long X509_V_FLAG_SUITEB_128_LOS = 0; +#if CRYPTOGRAPHY_LIBRESSL_251_OR_GREATER +int X509_VERIFY_PARAM_set1_host(X509_VERIFY_PARAM *, const char *, size_t); +int X509_VERIFY_PARAM_set1_email(X509_VERIFY_PARAM *, const char *, size_t); +int X509_VERIFY_PARAM_set1_ip(X509_VERIFY_PARAM *, const unsigned char *, + size_t); +int X509_VERIFY_PARAM_set1_ip_asc(X509_VERIFY_PARAM *, const char *); +void X509_VERIFY_PARAM_set_hostflags(X509_VERIFY_PARAM *, unsigned int); +#else int (*X509_VERIFY_PARAM_set1_host)(X509_VERIFY_PARAM *, const char *, size_t) = NULL; int (*X509_VERIFY_PARAM_set1_email)(X509_VERIFY_PARAM *, const char *, @@ -254,6 +262,7 @@ void (*X509_VERIFY_PARAM_set_hostflags)(X509_VERIFY_PARAM *, unsigned int) = NULL; #endif +#endif /* OpenSSL 1.0.2+ or Solaris's backport */ #ifdef X509_V_FLAG_PARTIAL_CHAIN diff --git a/lib_pypy/_cffi_ssl/osx-roots.diff b/lib_pypy/_cffi_ssl/osx-roots.diff new file mode 100644 --- /dev/null +++ b/lib_pypy/_cffi_ssl/osx-roots.diff @@ -0,0 +1,475 @@ +diff -Naur libressl-2.6.2.orig/crypto/Makefile.am libressl-2.6.2/crypto/Makefile.am +--- libressl-2.6.2.orig/crypto/Makefile.am 2017-09-02 01:49:55.000000000 +0200 ++++ libressl-2.6.2/crypto/Makefile.am 2017-10-07 14:05:16.000000000 +0200 +@@ -92,7 +92,7 @@ + -mv crypto_portable.sym.tmp crypto_portable.sym + endif + +-libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym ++libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym -framework Security -framework CoreFoundation + libcrypto_la_LIBADD = libcompat.la + if !HAVE_EXPLICIT_BZERO + libcrypto_la_LIBADD += libcompatnoopt.la +@@ -863,6 +863,7 @@ + libcrypto_la_SOURCES += x509/x509_txt.c + libcrypto_la_SOURCES += x509/x509_v3.c + libcrypto_la_SOURCES += x509/x509_vfy.c ++libcrypto_la_SOURCES += x509/x509_vfy_apple.c + libcrypto_la_SOURCES += x509/x509_vpm.c + libcrypto_la_SOURCES += x509/x509cset.c + libcrypto_la_SOURCES += x509/x509name.c +diff -Naur libressl-2.6.2.orig/crypto/Makefile.in libressl-2.6.2/crypto/Makefile.in +--- libressl-2.6.2.orig/crypto/Makefile.in 2017-09-26 06:07:03.000000000 +0200 ++++ libressl-2.6.2/crypto/Makefile.in 2017-10-07 14:05:24.000000000 +0200 +@@ -426,20 +426,20 @@ + x509/x509_err.c x509/x509_ext.c x509/x509_lu.c x509/x509_obj.c \ + x509/x509_r2x.c x509/x509_req.c x509/x509_set.c \ + x509/x509_trs.c x509/x509_txt.c x509/x509_v3.c x509/x509_vfy.c \ +- x509/x509_vpm.c x509/x509cset.c x509/x509name.c \ +- x509/x509rset.c x509/x509spki.c x509/x509type.c x509/x_all.c \ +- x509v3/pcy_cache.c x509v3/pcy_data.c x509v3/pcy_lib.c \ +- x509v3/pcy_map.c x509v3/pcy_node.c x509v3/pcy_tree.c \ +- x509v3/v3_akey.c x509v3/v3_akeya.c x509v3/v3_alt.c \ +- x509v3/v3_bcons.c x509v3/v3_bitst.c x509v3/v3_conf.c \ +- x509v3/v3_cpols.c x509v3/v3_crld.c x509v3/v3_enum.c \ +- x509v3/v3_extku.c x509v3/v3_genn.c x509v3/v3_ia5.c \ +- x509v3/v3_info.c x509v3/v3_int.c x509v3/v3_lib.c \ +- x509v3/v3_ncons.c x509v3/v3_ocsp.c x509v3/v3_pci.c \ +- x509v3/v3_pcia.c x509v3/v3_pcons.c x509v3/v3_pku.c \ +- x509v3/v3_pmaps.c x509v3/v3_prn.c x509v3/v3_purp.c \ +- x509v3/v3_skey.c x509v3/v3_sxnet.c x509v3/v3_utl.c \ +- x509v3/v3err.c ++ x509/x509_vfy_apple.c x509/x509_vpm.c x509/x509cset.c \ ++ x509/x509name.c x509/x509rset.c x509/x509spki.c \ ++ x509/x509type.c x509/x_all.c x509v3/pcy_cache.c \ ++ x509v3/pcy_data.c x509v3/pcy_lib.c x509v3/pcy_map.c \ ++ x509v3/pcy_node.c x509v3/pcy_tree.c x509v3/v3_akey.c \ ++ x509v3/v3_akeya.c x509v3/v3_alt.c x509v3/v3_bcons.c \ ++ x509v3/v3_bitst.c x509v3/v3_conf.c x509v3/v3_cpols.c \ ++ x509v3/v3_crld.c x509v3/v3_enum.c x509v3/v3_extku.c \ ++ x509v3/v3_genn.c x509v3/v3_ia5.c x509v3/v3_info.c \ ++ x509v3/v3_int.c x509v3/v3_lib.c x509v3/v3_ncons.c \ ++ x509v3/v3_ocsp.c x509v3/v3_pci.c x509v3/v3_pcia.c \ ++ x509v3/v3_pcons.c x509v3/v3_pku.c x509v3/v3_pmaps.c \ ++ x509v3/v3_prn.c x509v3/v3_purp.c x509v3/v3_skey.c \ ++ x509v3/v3_sxnet.c x509v3/v3_utl.c x509v3/v3err.c + am__objects_27 = aes/libcrypto_la-aes-elf-x86_64.lo \ + aes/libcrypto_la-bsaes-elf-x86_64.lo \ + aes/libcrypto_la-vpaes-elf-x86_64.lo \ +@@ -759,11 +759,12 @@ + x509/libcrypto_la-x509_r2x.lo x509/libcrypto_la-x509_req.lo \ + x509/libcrypto_la-x509_set.lo x509/libcrypto_la-x509_trs.lo \ + x509/libcrypto_la-x509_txt.lo x509/libcrypto_la-x509_v3.lo \ +- x509/libcrypto_la-x509_vfy.lo x509/libcrypto_la-x509_vpm.lo \ +- x509/libcrypto_la-x509cset.lo x509/libcrypto_la-x509name.lo \ +- x509/libcrypto_la-x509rset.lo x509/libcrypto_la-x509spki.lo \ +- x509/libcrypto_la-x509type.lo x509/libcrypto_la-x_all.lo \ +- x509v3/libcrypto_la-pcy_cache.lo \ ++ x509/libcrypto_la-x509_vfy.lo \ ++ x509/libcrypto_la-x509_vfy_apple.lo \ ++ x509/libcrypto_la-x509_vpm.lo x509/libcrypto_la-x509cset.lo \ ++ x509/libcrypto_la-x509name.lo x509/libcrypto_la-x509rset.lo \ ++ x509/libcrypto_la-x509spki.lo x509/libcrypto_la-x509type.lo \ ++ x509/libcrypto_la-x_all.lo x509v3/libcrypto_la-pcy_cache.lo \ + x509v3/libcrypto_la-pcy_data.lo x509v3/libcrypto_la-pcy_lib.lo \ + x509v3/libcrypto_la-pcy_map.lo x509v3/libcrypto_la-pcy_node.lo \ + x509v3/libcrypto_la-pcy_tree.lo x509v3/libcrypto_la-v3_akey.lo \ +@@ -1000,7 +1001,7 @@ + $(ASM_X86_64_ELF) $(ASM_X86_64_MACOSX) + BUILT_SOURCES = crypto_portable.sym + CLEANFILES = crypto_portable.sym +-libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym ++libcrypto_la_LDFLAGS = -version-info @LIBCRYPTO_VERSION@ -no-undefined -export-symbols crypto_portable.sym -framework Security -framework CoreFoundation + libcrypto_la_LIBADD = libcompat.la $(am__append_1) + libcrypto_la_CPPFLAGS = $(AM_CPPFLAGS) -DLIBRESSL_INTERNAL \ + -DOPENSSL_NO_HW_PADLOCK $(am__append_2) $(am__append_3) \ +@@ -1272,20 +1273,20 @@ + x509/x509_err.c x509/x509_ext.c x509/x509_lu.c x509/x509_obj.c \ + x509/x509_r2x.c x509/x509_req.c x509/x509_set.c \ + x509/x509_trs.c x509/x509_txt.c x509/x509_v3.c x509/x509_vfy.c \ +- x509/x509_vpm.c x509/x509cset.c x509/x509name.c \ +- x509/x509rset.c x509/x509spki.c x509/x509type.c x509/x_all.c \ +- x509v3/pcy_cache.c x509v3/pcy_data.c x509v3/pcy_lib.c \ +- x509v3/pcy_map.c x509v3/pcy_node.c x509v3/pcy_tree.c \ +- x509v3/v3_akey.c x509v3/v3_akeya.c x509v3/v3_alt.c \ +- x509v3/v3_bcons.c x509v3/v3_bitst.c x509v3/v3_conf.c \ +- x509v3/v3_cpols.c x509v3/v3_crld.c x509v3/v3_enum.c \ +- x509v3/v3_extku.c x509v3/v3_genn.c x509v3/v3_ia5.c \ +- x509v3/v3_info.c x509v3/v3_int.c x509v3/v3_lib.c \ +- x509v3/v3_ncons.c x509v3/v3_ocsp.c x509v3/v3_pci.c \ +- x509v3/v3_pcia.c x509v3/v3_pcons.c x509v3/v3_pku.c \ +- x509v3/v3_pmaps.c x509v3/v3_prn.c x509v3/v3_purp.c \ +- x509v3/v3_skey.c x509v3/v3_sxnet.c x509v3/v3_utl.c \ +- x509v3/v3err.c ++ x509/x509_vfy_apple.c x509/x509_vpm.c x509/x509cset.c \ ++ x509/x509name.c x509/x509rset.c x509/x509spki.c \ ++ x509/x509type.c x509/x_all.c x509v3/pcy_cache.c \ ++ x509v3/pcy_data.c x509v3/pcy_lib.c x509v3/pcy_map.c \ ++ x509v3/pcy_node.c x509v3/pcy_tree.c x509v3/v3_akey.c \ ++ x509v3/v3_akeya.c x509v3/v3_alt.c x509v3/v3_bcons.c \ ++ x509v3/v3_bitst.c x509v3/v3_conf.c x509v3/v3_cpols.c \ ++ x509v3/v3_crld.c x509v3/v3_enum.c x509v3/v3_extku.c \ ++ x509v3/v3_genn.c x509v3/v3_ia5.c x509v3/v3_info.c \ ++ x509v3/v3_int.c x509v3/v3_lib.c x509v3/v3_ncons.c \ ++ x509v3/v3_ocsp.c x509v3/v3_pci.c x509v3/v3_pcia.c \ ++ x509v3/v3_pcons.c x509v3/v3_pku.c x509v3/v3_pmaps.c \ ++ x509v3/v3_prn.c x509v3/v3_purp.c x509v3/v3_skey.c \ ++ x509v3/v3_sxnet.c x509v3/v3_utl.c x509v3/v3err.c + + # chacha + +@@ -2808,6 +2809,8 @@ + x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509_vfy.lo: x509/$(am__dirstamp) \ + x509/$(DEPDIR)/$(am__dirstamp) ++x509/libcrypto_la-x509_vfy_apple.lo: x509/$(am__dirstamp) \ ++ x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509_vpm.lo: x509/$(am__dirstamp) \ + x509/$(DEPDIR)/$(am__dirstamp) + x509/libcrypto_la-x509cset.lo: x509/$(am__dirstamp) \ +@@ -3583,6 +3586,7 @@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509_txt.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509_v3.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509_vfy.Plo@am__quote@ ++@AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509_vpm.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509cset.Plo@am__quote@ + @AMDEP_TRUE@@am__include@ @am__quote@x509/$(DEPDIR)/libcrypto_la-x509name.Plo@am__quote@ +@@ -7460,6 +7464,13 @@ + @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ + @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o x509/libcrypto_la-x509_vfy.lo `test -f 'x509/x509_vfy.c' || echo '$(srcdir)/'`x509/x509_vfy.c + ++x509/libcrypto_la-x509_vfy_apple.lo: x509/x509_vfy_apple.c ++@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT x509/libcrypto_la-x509_vfy_apple.lo -MD -MP -MF x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Tpo -c -o x509/libcrypto_la-x509_vfy_apple.lo `test -f 'x509/x509_vfy_apple.c' || echo '$(srcdir)/'`x509/x509_vfy_apple.c ++@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Tpo x509/$(DEPDIR)/libcrypto_la-x509_vfy_apple.Plo ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='x509/x509_vfy_apple.c' object='x509/libcrypto_la-x509_vfy_apple.lo' libtool=yes @AMDEPBACKSLASH@ ++@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ ++@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o x509/libcrypto_la-x509_vfy_apple.lo `test -f 'x509/x509_vfy_apple.c' || echo '$(srcdir)/'`x509/x509_vfy_apple.c ++ + x509/libcrypto_la-x509_vpm.lo: x509/x509_vpm.c + @am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(libcrypto_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT x509/libcrypto_la-x509_vpm.lo -MD -MP -MF x509/$(DEPDIR)/libcrypto_la-x509_vpm.Tpo -c -o x509/libcrypto_la-x509_vpm.lo `test -f 'x509/x509_vpm.c' || echo '$(srcdir)/'`x509/x509_vpm.c + @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) x509/$(DEPDIR)/libcrypto_la-x509_vpm.Tpo x509/$(DEPDIR)/libcrypto_la-x509_vpm.Plo +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy.c libressl-2.6.2/crypto/x509/x509_vfy.c +--- libressl-2.6.2.orig/crypto/x509/x509_vfy.c 2017-09-02 14:01:08.000000000 +0200 ++++ libressl-2.6.2/crypto/x509/x509_vfy.c 2017-10-07 14:05:16.000000000 +0200 +@@ -115,6 +115,13 @@ + + #define CRL_SCORE_TIME_DELTA 0x002 + ++/* ++ * If we are using Trust Evaluation Agent, rename the original function ++ */ ++#ifdef __APPLE__ ++#define X509_verify_cert X509_verify_cert_orig ++#endif ++ + static int null_callback(int ok, X509_STORE_CTX *e); + static int check_issued(X509_STORE_CTX *ctx, X509 *x, X509 *issuer); + static X509 *find_issuer(X509_STORE_CTX *ctx, STACK_OF(X509) *sk, X509 *x); +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.c libressl-2.6.2/crypto/x509/x509_vfy_apple.c +--- libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.c 1970-01-01 01:00:00.000000000 +0100 ++++ libressl-2.6.2/crypto/x509/x509_vfy_apple.c 2017-10-07 14:05:16.000000000 +0200 +@@ -0,0 +1,225 @@ ++/* ++ * Copyright (c) 2009 Apple Inc. All Rights Reserved. ++ * ++ * @APPLE_LICENSE_HEADER_START@ ++ * ++ * This file contains Original Code and/or Modifications of Original Code ++ * as defined in and that are subject to the Apple Public Source License ++ * Version 2.0 (the 'License'). You may not use this file except in ++ * compliance with the License. Please obtain a copy of the License at ++ * http://www.opensource.apple.com/apsl/ and read it before using this ++ * file. ++ * ++ * The Original Code and all software distributed under the License are ++ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER ++ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ++ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. ++ * Please see the License for the specific language governing rights and ++ * limitations under the License. ++ * ++ * @APPLE_LICENSE_HEADER_END@ ++ * ++ */ ++ ++#include <stdint.h> ++#include <inttypes.h> ++#include <syslog.h> ++ ++#include <Security/Security.h> ++ ++#include <openssl/opensslconf.h> ++ ++#include <openssl/crypto.h> ++#include <openssl/x509.h> ++#include <openssl/x509v3.h> ++ ++#include "cryptlib.h" ++#include "vpm_int.h" ++#include "x509_vfy_apple.h" ++ ++#define TEA_might_correct_error(err) (err == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY || err == X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT || err == X509_V_ERR_SELF_SIGNED_CERT_IN_CHAIN) ++ ++ ++static bool add_cert_to_array(CFMutableArrayRef array, X509 *x509) ++{ ++ unsigned char *asn1_cert_data = NULL; ++ int asn1_cert_len = i2d_X509(x509, &asn1_cert_data); ++ ++ CFDataRef data = CFDataCreate(kCFAllocatorDefault, asn1_cert_data, asn1_cert_len); ++ ++ if (data == NULL) { ++ return false; ++ } ++ ++ SecCertificateRef cert = SecCertificateCreateWithData(NULL, data); ++ ++ free(asn1_cert_data); ++ ++ if (cert == NULL) { ++ CFRelease(data); ++ return false; ++ } ++ ++ CFArrayAppendValue(array, cert); ++ CFRelease(data); ++ ++ return true; ++} ++ ++static CFStringRef to_string(const char *s) { ++ if (s == NULL) ++ return NULL; ++ return CFStringCreateWithCString(kCFAllocatorDefault, s, ++ kCFStringEncodingASCII); ++} ++ ++static SecPolicyRef get_policy(X509_VERIFY_PARAM *param) { ++ switch (param->purpose) { ++ case X509_PURPOSE_SSL_CLIENT: ++ case X509_PURPOSE_SSL_SERVER: { ++ ++ if (!param->id) { ++ fprintf(stderr, "got no ID!\n"); ++ return NULL; ++ } ++ ++ CFStringRef hostname; ++ int nhosts = sk_OPENSSL_STRING_num(param->id->hosts); ++ ++ if (nhosts != 1) { ++ hostname = NULL; ++ ++ } else { ++ hostname = to_string(sk_OPENSSL_STRING_value(param->id->hosts, 0)); ++ CFShow(hostname); ++ } ++ ++ return SecPolicyCreateSSL(param->purpose == X509_PURPOSE_SSL_SERVER, ++ hostname); ++ } ++ ++ case X509_PURPOSE_NS_SSL_SERVER: ++ case X509_PURPOSE_SMIME_SIGN: ++ case X509_PURPOSE_SMIME_ENCRYPT: ++ case X509_PURPOSE_CRL_SIGN: ++ case X509_PURPOSE_ANY: ++ case X509_PURPOSE_OCSP_HELPER: ++ case X509_PURPOSE_TIMESTAMP_SIGN: ++ default: ++ fprintf(stderr, "unsupported purpose %d", param->purpose); ++ return NULL; ++ } ++} ++ ++/* ++ * Please see comment in x509_vfy_apple.h ++ */ ++int ++X509_verify_cert(X509_STORE_CTX *ctx) ++{ ++ uint64_t certLastIndex = 0; ++ uint64_t i = 0; ++ ++ /* Try OpenSSL, if we get a local certificate issue verify against trusted roots */ ++ int ret = X509_verify_cert_orig(ctx); ++ ++ /* Verify TEA is enabled and should be used. */ ++ if (0 == X509_TEA_is_enabled() || ++ ret == 1 || !TEA_might_correct_error(ctx->error)) { ++ return ret; ++ } ++ ++ /* Verify that the certificate chain exists, otherwise make it. */ ++ if (ctx->chain == NULL && (ctx->chain = sk_X509_new_null()) == NULL) { ++ fprintf(stderr, "Could not create the certificate chain"); ++ ctx->error = X509_V_ERR_UNABLE_TO_GET_ISSUER_CERT_LOCALLY; ++ return -1; ++ } ++ ++ /* Verify chain depth */ ++ certLastIndex = sk_X509_num(ctx->untrusted); ++ if (certLastIndex > ctx->param->depth) { ++ fprintf(stderr, "Pruning certificate chain to %" PRIu64, certLastIndex); ++ certLastIndex = ctx->param->depth; ++ } ++ ++ CFMutableArrayRef certArray = CFArrayCreateMutable(NULL, certLastIndex + 1, NULL); ++ CFRetain(certArray); ++ ++ if (!add_cert_to_array(certArray, ctx->cert)) { ++ fprintf(stderr, "Failed to add certificate to array"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return -1; ++ } ++ ++ for (i = 0; i < certLastIndex; ++i) { ++ X509 *t = sk_X509_value(ctx->untrusted, i); ++ if (!add_cert_to_array(certArray, t)) { ++ fprintf(stderr, "Failed to add chain certificate %lld to array", i); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return 0; ++ } ++ } ++ ++ // We put ASN.1 encoded X509 on the CertificateChain, so we don't call TEACertificateChainSetEncodingHandler ++ SecPolicyRef policy = get_policy(ctx->param); ++ ++ if (policy == NULL) { ++ fprintf(stderr, "Failed to create policy!\n"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_UNSPECIFIED; ++ return -1; ++ } ++ ++ SecTrustRef trust = NULL; ++ ++ if (SecTrustCreateWithCertificates(certArray, policy, &trust) != errSecSuccess) { ++ fprintf(stderr, "Failed to create trust!\n"); ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_CERT_UNTRUSTED; ++ return -1; ++ } ++ ++ if (ctx->param->flags & X509_V_FLAG_USE_CHECK_TIME) { ++ fprintf(stderr, "Setting time not supported yet?\n"); ++ SecTrustSetVerifyDate(trust, CFDateCreate(NULL, ctx->param->check_time)); ++ } ++ ++ SecTrustResultType result = 0; ++ ++ if (SecTrustEvaluate(trust, &result) != errSecSuccess || result != kSecTrustResultUnspecified) { ++ CFRelease(certArray); ++ ctx->error = X509_V_ERR_CERT_UNTRUSTED; ++ return 0; ++ } ++ ++ CFRelease(certArray); ++ ctx->error = 0; ++ return 1; ++} ++ ++#pragma mark Trust Evaluation Agent ++ ++/* -1: not set ++ * 0: set to false ++ * 1: set to true ++ */ ++static int tea_enabled = -1; ++ ++void ++X509_TEA_set_state(int change) ++{ ++ tea_enabled = (change) ? 1 : 0; ++} ++ ++int ++X509_TEA_is_enabled() ++{ ++ if (tea_enabled < 0) ++ tea_enabled = (NULL == getenv(X509_TEA_ENV_DISABLE)); ++ ++ return tea_enabled != 0; ++} +diff -Naur libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.h libressl-2.6.2/crypto/x509/x509_vfy_apple.h +--- libressl-2.6.2.orig/crypto/x509/x509_vfy_apple.h 1970-01-01 01:00:00.000000000 +0100 ++++ libressl-2.6.2/crypto/x509/x509_vfy_apple.h 2017-10-07 14:05:16.000000000 +0200 +@@ -0,0 +1,74 @@ ++/* ++ * Copyright (c) 2009 Apple Inc. All Rights Reserved. ++ * ++ * @APPLE_LICENSE_HEADER_START@ ++ * ++ * This file contains Original Code and/or Modifications of Original Code ++ * as defined in and that are subject to the Apple Public Source License ++ * Version 2.0 (the 'License'). You may not use this file except in ++ * compliance with the License. Please obtain a copy of the License at ++ * http://www.opensource.apple.com/apsl/ and read it before using this ++ * file. ++ * ++ * The Original Code and all software distributed under the License are ++ * distributed on an 'AS IS' basis, WITHOUT WARRANTY OF ANY KIND, EITHER ++ * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ++ * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ++ * FITNESS FOR A PARTICULAR PURPOSE, QUIET ENJOYMENT OR NON-INFRINGEMENT. ++ * Please see the License for the specific language governing rights and ++ * limitations under the License. ++ * ++ * @APPLE_LICENSE_HEADER_END@ ++ * ++ */ ++ ++#ifndef HEADER_X509_H ++#include <openssl/x509.h> ++#endif ++ ++#ifndef HEADER_X509_VFY_APPLE_H ++#define HEADER_X509_VFY_APPLE_H ++ ++/* Environment variable name to disable TEA. */ ++#define X509_TEA_ENV_DISABLE "OPENSSL_X509_TEA_DISABLE" ++ ++/* ++ * X509_verify_cert ++ * ++ * Originally located in x509_vfy.c. ++ * ++ * Verify certificate with OpenSSL created X509_verify_cert. If and only if ++ * OpenSSL cannot get certificate issuer locally then OS X security API will ++ * verify the certificate, using Trust Evaluation Agent. ++ * ++ * Return values: ++ * -------------- ++ * -1: Null was passed for either ctx or ctx->cert. ++ * 0: Certificate is trusted. ++ * 1: Certificate is not trusted. ++ */ ++int X509_verify_cert(X509_STORE_CTX *ctx); ++ ++/* ++ * X509_TEA_is_enabled ++ * ++ * Is the Trust Evaluation Agent (TEA) used for certificate verification when ++ * the issuer cannot be verified. ++ * ++ * Returns 0 if TEA is disabled and 1 if TEA is enabled. ++ */ ++int X509_TEA_is_enabled(); ++ ++/* ++ * X509_TEA_set_state ++ * ++ * Enables/disables certificate verification with Trust Evaluation Agent (TEA) ++ * when the issuer cannot be verified. ++ * ++ * Pass 0 to disable TEA and non-zero to enable TEA. ++ */ ++void X509_TEA_set_state(int change); ++ ++int X509_verify_cert_orig(X509_STORE_CTX *ctx); ++ ++#endif /* HEADER_X509_VFY_APPLE_H */ diff --git a/lib_pypy/_ctypes_test.py b/lib_pypy/_ctypes_test.py --- a/lib_pypy/_ctypes_test.py +++ b/lib_pypy/_ctypes_test.py @@ -21,5 +21,11 @@ with fp: imp.load_module('_ctypes_test', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass print('could not find _ctypes_test in %s' % output_dir) _pypy_testcapi.compile_shared('_ctypes_test.c', '_ctypes_test', output_dir) diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -8,7 +8,8 @@ content = fid.read() # from cffi's Verifier() key = '\x00'.join([sys.version[:3], content]) - key += 'cpyext-gc-support-2' # this branch requires recompilation! + # change the key to force recompilation + key += '2017-11-21' if sys.version_info >= (3,): key = key.encode('utf-8') k1 = hex(binascii.crc32(key[0::2]) & 0xffffffff) diff --git a/lib_pypy/_testcapi.py b/lib_pypy/_testcapi.py --- a/lib_pypy/_testcapi.py +++ b/lib_pypy/_testcapi.py @@ -17,6 +17,12 @@ with fp: imp.load_module('_testcapi', fp, filename, description) except ImportError: + if os.name == 'nt': + # hack around finding compilers on win32 + try: + import setuptools + except ImportError: + pass _pypy_testcapi.compile_shared(cfile, '_testcapi', output_dir) diff --git a/lib_pypy/_testcapimodule.c b/lib_pypy/_testcapimodule.c --- a/lib_pypy/_testcapimodule.c +++ b/lib_pypy/_testcapimodule.c @@ -915,12 +915,6 @@ return -1; } Py_DECREF(res); - if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " - "arg was not decrefed in successful " - "Py_BuildValue(\"%s\")", fmt); - return -1; - } Py_INCREF(arg); res = Py_BuildValue(fmt, raise_error, NULL, arg); @@ -930,12 +924,6 @@ return -1; } PyErr_Clear(); - if (Py_REFCNT(arg) != 1) { - PyErr_Format(TestError, "test_buildvalue_N: " - "arg was not decrefed in failed " - "Py_BuildValue(\"%s\")", fmt); - return -1; - } Py_DECREF(arg); return 0; } @@ -958,10 +946,6 @@ return raiseTestError("test_buildvalue_N", "Py_BuildValue(\"N\") returned wrong result"); } - if (Py_REFCNT(arg) != 2) { - return raiseTestError("test_buildvalue_N", - "arg was not decrefed in Py_BuildValue(\"N\")"); - } Py_DECREF(res); Py_DECREF(arg); @@ -2834,8 +2818,6 @@ return PyMemoryView_FromBuffer(&info); } -#ifndef PYPY_VERSION - static PyObject * test_from_contiguous(PyObject* self, PyObject *noargs) { @@ -2885,7 +2867,6 @@ Py_RETURN_NONE; } -#endif /* PYPY_VERSION */ #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION) extern PyTypeObject _PyBytesIOBuffer_Type; @@ -3923,9 +3904,7 @@ {"test_string_to_double", (PyCFunction)test_string_to_double, METH_NOARGS}, {"test_unicode_compare_with_ascii", (PyCFunction)test_unicode_compare_with_ascii, METH_NOARGS}, {"test_capsule", (PyCFunction)test_capsule, METH_NOARGS}, -#ifndef PYPY_VERSION {"test_from_contiguous", (PyCFunction)test_from_contiguous, METH_NOARGS}, -#endif #if (defined(__linux__) || defined(__FreeBSD__)) && defined(__GNUC__) && !defined(PYPY_VERSION) {"test_pep3118_obsolete_write_locks", (PyCFunction)test_pep3118_obsolete_write_locks, METH_NOARGS}, #endif diff --git a/lib_pypy/_tkinter/app.py b/lib_pypy/_tkinter/app.py --- a/lib_pypy/_tkinter/app.py +++ b/lib_pypy/_tkinter/app.py @@ -185,6 +185,9 @@ if err == tklib.TCL_ERROR: self.raiseTclError() + def interpaddr(self): + return int(tkffi.cast('size_t', self.interp)) + def _var_invoke(self, func, *args, **kwargs): if self.threaded and self.thread_id != tklib.Tcl_GetCurrentThread(): # The current thread is not the interpreter thread. diff --git a/lib_pypy/resource.py b/lib_pypy/resource.py --- a/lib_pypy/resource.py +++ b/lib_pypy/resource.py @@ -20,6 +20,7 @@ or via the attributes ru_utime, ru_stime, ru_maxrss, and so on.""" __metaclass__ = _structseq.structseqtype + name = "resource.struct_rusage" ru_utime = _structseq.structseqfield(0, "user time used") ru_stime = _structseq.structseqfield(1, "system time used") diff --git a/pypy/doc/faq.rst b/pypy/doc/faq.rst --- a/pypy/doc/faq.rst +++ b/pypy/doc/faq.rst @@ -182,6 +182,57 @@ technical difficulties. +What about numpy, numpypy, micronumpy? +-------------------------------------- + +Way back in 2011, the PyPy team `started to reimplement`_ numpy in PyPy. It +has two pieces: + + * the builtin module :source:`pypy/module/micronumpy`: this is written in + RPython and roughly covers the content of the ``numpy.core.multiarray`` + module. Confusingly enough, this is available in PyPy under the name + ``_numpypy``. It is included by default in all the official releases of + PyPy (but it might be dropped in the future). + + * a fork_ of the official numpy repository maintained by us and informally + called ``numpypy``: even more confusing, the name of the repo on bitbucket + is ``numpy``. The main difference with the upstream numpy, is that it is + based on the micronumpy module written in RPython, instead of of + ``numpy.core.multiarray`` which is written in C. + +Moreover, it is also possible to install the upstream version of ``numpy``: +its core is written in C and it runs on PyPy under the cpyext compatibility +layer. This is what you get if you do ``pypy -m pip install numpy``. + + +Should I install numpy or numpypy? +----------------------------------- + +TL;DR version: you should use numpy. You can install it by doing ``pypy -m pip +install numpy``. You might also be interested in using the experimental `PyPy +binary wheels`_ to save compilation time. + +The upstream ``numpy`` is written in C, and runs under the cpyext +compatibility layer. Nowadays, cpyext is mature enough that you can simply +use the upstream ``numpy``, since it passes 99.9% of the test suite. At the +moment of writing (October 2017) the main drawback of ``numpy`` is that cpyext +is infamously slow, and thus it has worse performance compared to +``numpypy``. However, we are actively working on improving it, as we expect to +reach the same speed, eventually. + +On the other hand, ``numpypy`` is more JIT-friendly and very fast to call, +since it is written in RPython: but it is a reimplementation, and it's hard to +be completely compatible: over the years the project slowly matured and +eventually it was able to call out to the LAPACK and BLAS libraries to speed +matrix calculations, and reached around an 80% parity with the upstream +numpy. However, 80% is far from 100%. Since cpyext/numpy compatibility is +progressing fast, we have discontinued support for ``numpypy``. + +.. _`started to reimplement`: https://morepypy.blogspot.co.il/2011/05/numpy-in-pypy-status-and-roadmap.html +.. _fork: https://bitbucket.org/pypy/numpy +.. _`PyPy binary wheels`: https://github.com/antocuni/pypy-wheels + + Is PyPy more clever than CPython about Tail Calls? -------------------------------------------------- diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -10,3 +10,22 @@ .. branch: docs-osx-brew-openssl +.. branch: keep-debug-symbols +Add a smartstrip tool, which can optionally keep the debug symbols in a +separate file, instead of just stripping them away. Use it in packaging + +.. branch: bsd-patches +Fix failures on FreeBSD, contributed by David Naylor as patches on the issue +tracker (issues 2694, 2695, 2696, 2697) + +.. branch: run-extra-tests +Run extra_tests/ in buildbot + +.. branch: vmprof-0.4.10 +Upgrade the _vmprof backend to vmprof 0.4.10 + +.. branch: fix-vmprof-stacklet-switch +Fix a vmprof+continulets (i.e. greenelts, eventlet, gevent, ...) + +.. branch: win32-vcvars + diff --git a/pypy/doc/whatsnew-pypy3-head.rst b/pypy/doc/whatsnew-pypy3-head.rst --- a/pypy/doc/whatsnew-pypy3-head.rst +++ b/pypy/doc/whatsnew-pypy3-head.rst @@ -11,3 +11,8 @@ .. branch: py3.5-appexec Raise if space.is_true(space.appexec()) used in app level tests, fix tests that did this + +.. branch: py3.5-mac-embedding +Download and patch dependencies when building cffi-based stdlib modules + +.. branch: os_lockf diff --git a/pypy/doc/windows.rst b/pypy/doc/windows.rst --- a/pypy/doc/windows.rst +++ b/pypy/doc/windows.rst @@ -25,8 +25,10 @@ This compiler, while the standard one for Python 2.7, is deprecated. Microsoft has made it available as the `Microsoft Visual C++ Compiler for Python 2.7`_ (the link -was checked in Nov 2016). Note that the compiler suite will be installed in -``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python``. +was checked in Nov 2016). Note that the compiler suite may be installed in +``C:\Users\<user name>\AppData\Local\Programs\Common\Microsoft\Visual C++ for Python`` +or in +``C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python``. A current version of ``setuptools`` will be able to find it there. For Windows 10, you must right-click the download, and under ``Properties`` -> ``Compatibility`` mark it as ``Run run this program in comatibility mode for`` @@ -41,7 +43,6 @@ ----------------------------------- We routinely test translation using v9, also known as Visual Studio 2008. -Our buildbot is still using the Express Edition, not the compiler noted above. Other configurations may work as well. The translation scripts will set up the appropriate environment variables @@ -81,6 +82,30 @@ .. _build instructions: http://pypy.org/download.html#building-from-source +Setting Up Visual Studio for building SSL in Python3 +---------------------------------------------------- + +On Python3, the ``ssl`` module is based on ``cffi``, and requires a build step after +translation. However ``distutils`` does not support the Micorosft-provided Visual C +compiler, and ``cffi`` depends on ``distutils`` to find the compiler. The +traditional solution to this problem is to install the ``setuptools`` module +via running ``-m ensurepip`` which installs ``pip`` and ``setuptools``. However +``pip`` requires ``ssl``. So we have a chicken-and-egg problem: ``ssl`` depends on +``cffi`` which depends on ``setuptools``, which depends on ``ensurepip``, which +depends on ``ssl``. + +In order to solve this, the buildbot sets an environment varaible that helps +``distutils`` find the compiler without ``setuptools``:: + + set VS90COMNTOOLS=C:\Program Files (x86)\Common Files\Microsoft\Visual C++ for Python\9.0\VC\bin + +or whatever is appropriate for your machine. Note that this is not enough, you +must also copy the ``vcvarsall.bat`` file fron the ``...\9.0`` directory to the +``...\9.0\VC`` directory, and edit it, changing the lines that set +``VCINSTALLDIR`` and ``WindowsSdkDir``:: + set VCINSTALLDIR=%~dp0\ + set WindowsSdkDir=%~dp0\..\WinSDK\ + Preparing Windows for the large build ------------------------------------- diff --git a/pypy/goal/getnightly.py b/pypy/goal/getnightly.py --- a/pypy/goal/getnightly.py +++ b/pypy/goal/getnightly.py @@ -15,7 +15,7 @@ arch = 'linux' cmd = 'wget "%s"' TAR_OPTIONS += ' --wildcards' - binfiles = "'*/bin/pypy3' '*/bin/libpypy3-c.so'" + binfiles = "'*/bin/pypy3*' '*/bin/libpypy3-c.so*'" if os.uname()[-1].startswith('arm'): arch += '-armhf-raspbian' elif sys.platform.startswith('darwin'): diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -352,8 +352,9 @@ def hack_for_cffi_modules(self, driver): # HACKHACKHACK - # ugly hack to modify target goal from compile_* to build_cffi_imports - # this should probably get cleaned up and merged with driver.create_exe + # ugly hack to modify target goal from compile_* to build_cffi_imports, + # as done in package.py + # this is needed by the benchmark buildbot run, maybe do it as a seperate step there? from rpython.tool.runsubprocess import run_subprocess from rpython.translator.driver import taskdef import types @@ -363,11 +364,14 @@ def task_build_cffi_imports(self): ''' Use cffi to compile cffi interfaces to modules''' filename = os.path.join(pypydir, 'tool', 'build_cffi_imports.py') + if sys.platform == 'darwin': + argv = [filename, '--embed-dependencies'] + else: + argv = [filename,] status, out, err = run_subprocess(str(driver.compute_exe_name()), - [filename]) + argv) sys.stdout.write(out) sys.stderr.write(err) - # otherwise, ignore errors driver.task_build_cffi_imports = types.MethodType(task_build_cffi_imports, driver) driver.tasks['build_cffi_imports'] = driver.task_build_cffi_imports, [compile_goal] driver.default_goal = 'build_cffi_imports' diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py --- a/pypy/interpreter/app_main.py +++ b/pypy/interpreter/app_main.py @@ -579,6 +579,8 @@ __pypy__.save_module_content_for_future_reload(sys) mainmodule = type(sys)('__main__') + mainmodule.__loader__ = sys.__loader__ + mainmodule.__builtins__ = os.__builtins__ sys.modules['__main__'] = mainmodule if not no_site: @@ -727,7 +729,7 @@ SourceFileLoader, SourcelessFileLoader) if IS_WINDOWS: filename = filename.lower() - if filename.endswith('.pyc') or filename.endswith('.pyo'): + if filename.endswith('.pyc'): # We don't actually load via SourcelessFileLoader # because '__main__' must not be listed inside # 'importlib._bootstrap._module_locks' (it deadlocks diff --git a/pypy/interpreter/mixedmodule.py b/pypy/interpreter/mixedmodule.py --- a/pypy/interpreter/mixedmodule.py +++ b/pypy/interpreter/mixedmodule.py @@ -254,7 +254,7 @@ assert typ == imp.PY_SOURCE source = file.read() file.close() - if fn.endswith('.pyc') or fn.endswith('.pyo'): + if fn.endswith('.pyc'): fn = fn[:-1] app = gateway.applevel(source, filename=fn, modname=appname) applevelcache[impbase] = app diff --git a/pypy/interpreter/pyparser/error.py b/pypy/interpreter/pyparser/error.py --- a/pypy/interpreter/pyparser/error.py +++ b/pypy/interpreter/pyparser/error.py @@ -14,7 +14,20 @@ def wrap_info(self, space): w_text = w_filename = space.w_None offset = self.offset - if self.text is not None: + w_lineno = space.newint(self.lineno) + if self.filename is not None: + w_filename = space.newfilename(self.filename) + if self.text is None and self.filename is not None: + w_text = space.appexec([w_filename, w_lineno], + """(filename, lineno): + try: + with open(filename) as f: + for _ in range(lineno - 1): + f.readline() + return f.readline() + except: # we can't allow any exceptions here! + return None""") + elif self.text is not None: from rpython.rlib.runicode import str_decode_utf_8 # self.text may not be UTF-8 in case of decoding errors. # adjust the encoded text offset to a decoded offset @@ -29,20 +42,15 @@ text, _ = str_decode_utf_8(self.text, len(self.text), 'replace') w_text = space.newunicode(text) - if self.filename is not None: - w_filename = space.newfilename(self.filename) - return space.newtuple([space.newtext(self.msg), - space.newtuple([w_filename, - space.newint(self.lineno), - space.newint(offset), - w_text, - space.newint(self.lastlineno)])]) + return space.newtuple([ + space.newtext(self.msg), + space.newtuple([ + w_filename, w_lineno, space.newint(offset), + w_text, space.newint(self.lastlineno)])]) def __str__(self): - return "%s at pos (%d, %d) in %r" % (self.__class__.__name__, - self.lineno, - self.offset, - self.text) + return "%s at pos (%d, %d) in %r" % ( + self.__class__.__name__, self.lineno, self.offset, self.text) class IndentationError(SyntaxError): pass @@ -51,10 +59,11 @@ def __init__(self, lineno=0, offset=0, text=None, filename=None, lastlineno=0): msg = "inconsistent use of tabs and spaces in indentation" - IndentationError.__init__(self, msg, lineno, offset, text, filename, lastlineno) + IndentationError.__init__( + self, msg, lineno, offset, text, filename, lastlineno) class ASTError(Exception): - def __init__(self, msg, ast_node ): + def __init__(self, msg, ast_node): self.msg = msg self.ast_node = ast_node diff --git a/pypy/interpreter/test/test_gateway.py b/pypy/interpreter/test/test_gateway.py --- a/pypy/interpreter/test/test_gateway.py +++ b/pypy/interpreter/test/test_gateway.py @@ -18,7 +18,7 @@ class TestBuiltinCode: - def test_signature(self): + def test_signature(self, space): def c(space, w_x, w_y, hello_w): pass code = gateway.BuiltinCode(c, unwrap_spec=[gateway.ObjSpace, @@ -53,6 +53,8 @@ code = gateway.BuiltinCode(f, unwrap_spec=[gateway.ObjSpace, "kwonly", W_Root]) assert code.signature() == Signature([], kwonlyargnames=['x']) + assert space.int_w(space.getattr( + code, space.newtext('co_kwonlyargcount'))) == 1 def test_call(self): diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -16,7 +16,7 @@ @not_rpython def __init__(self, __name, __base=None, __total_ordering__=None, __buffer=None, __confirm_applevel_del__=False, - variable_sized=False, **rawdict): + _text_signature_=None, variable_sized=False, **rawdict): "initialization-time only" self.name = __name if __base is None: @@ -36,6 +36,7 @@ assert '__del__' not in rawdict self.weakrefable = '__weakref__' in rawdict self.doc = rawdict.get('__doc__', None) + self.text_signature = _text_signature_ for base in bases: self.hasdict |= base.hasdict self.weakrefable |= base.weakrefable @@ -539,6 +540,9 @@ def fget_co_argcount(space, code): # unwrapping through unwrap_spec return space.newint(code.signature().num_argnames()) +def fget_co_kwonlyargcount(space, code): # unwrapping through unwrap_spec + return space.newint(code.signature().num_kwonlyargnames()) + def fget_zero(space, code): return space.newint(0) @@ -598,7 +602,7 @@ co_name = interp_attrproperty('co_name', cls=BuiltinCode, wrapfn="newtext_or_none"), co_varnames = GetSetProperty(fget_co_varnames, cls=BuiltinCode), co_argcount = GetSetProperty(fget_co_argcount, cls=BuiltinCode), - co_kwonlyargcount = GetSetProperty(fget_zero, cls=BuiltinCode), + co_kwonlyargcount = GetSetProperty(fget_co_kwonlyargcount, cls=BuiltinCode), co_flags = GetSetProperty(fget_co_flags, cls=BuiltinCode), co_consts = GetSetProperty(fget_co_consts, cls=BuiltinCode), ) diff --git a/pypy/module/_continuation/test/conftest.py b/pypy/module/_continuation/test/conftest.py new file mode 100644 --- /dev/null +++ b/pypy/module/_continuation/test/conftest.py @@ -0,0 +1,7 @@ +import pytest +import sys + +def pytest_configure(config): + if sys.platform.startswith('linux'): + from rpython.rlib.rvmprof.cintf import configure_libbacktrace_linux + configure_libbacktrace_linux() diff --git a/pypy/module/_continuation/test/test_stacklet.py b/pypy/module/_continuation/test/test_stacklet.py --- a/pypy/module/_continuation/test/test_stacklet.py +++ b/pypy/module/_continuation/test/test_stacklet.py @@ -8,6 +8,35 @@ cls.w_translated = cls.space.wrap( os.path.join(os.path.dirname(__file__), 'test_translated.py')) + cls.w_stack = cls.space.appexec([], """(): + import sys + def stack(f=None): + ''' + get the call-stack of the caller or the specified frame + ''' + if f is None: + f = sys._getframe(1) + res = [] + seen = set() + while f: + if f in seen: + # frame cycle + res.append('...') + break + if f.f_code.co_name == 'runtest': + # if we are running with -A, cut all the stack above + # the test function + break + seen.add(f) + res.append(f.f_code.co_name) + f = f.f_back + #print res + return res + return stack + """) + if cls.runappdirect: + # make sure that "self.stack" does not pass the self + cls.w_stack = staticmethod(cls.w_stack.im_func) def test_new_empty(self): from _continuation import continulet @@ -339,17 +368,24 @@ def test_f_back(self): import sys from _continuation import continulet + stack = self.stack # def bar(c): + assert stack() == ['bar', 'foo', 'test_f_back'] c.switch(sys._getframe(0)) c.switch(sys._getframe(0).f_back) c.switch(sys._getframe(1)) + # + assert stack() == ['bar', 'foo', 'main', 'test_f_back'] c.switch(sys._getframe(1).f_back) + # + assert stack() == ['bar', 'foo', 'main2', 'test_f_back'] assert sys._getframe(2) is f3_foo.f_back c.switch(sys._getframe(2)) def foo(c): bar(c) # + assert stack() == ['test_f_back'] c = continulet(foo) f1_bar = c.switch() assert f1_bar.f_code.co_name == 'bar' @@ -358,14 +394,20 @@ f3_foo = c.switch() assert f3_foo is f2_foo assert f1_bar.f_back is f3_foo + # def main(): f4_main = c.switch() assert f4_main.f_code.co_name == 'main' assert f3_foo.f_back is f1_bar # not running, so a loop + assert stack() == ['main', 'test_f_back'] + assert stack(f1_bar) == ['bar', 'foo', '...'] + # def main2(): _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit