Author: Philip Jenvey <pjen...@underboss.org> Branch: py3k-qualname Changeset: r72715:525ebcd3c24a Date: 2014-08-05 17:36 -0700 http://bitbucket.org/pypy/pypy/changeset/525ebcd3c24a/
Log: merge py3.3 diff too long, truncating to 2000 out of 4104 lines diff --git a/lib-python/3/distutils/command/build_ext.py b/lib-python/3/distutils/command/build_ext.py --- a/lib-python/3/distutils/command/build_ext.py +++ b/lib-python/3/distutils/command/build_ext.py @@ -4,7 +4,8 @@ modules (currently limited to C extensions, should accommodate C++ extensions ASAP).""" -import sys, os, re, imp +import sys, os, re +import importlib.machinery from distutils.core import Command from distutils.errors import * from distutils.sysconfig import customize_compiler, get_python_version @@ -36,9 +37,8 @@ show_compilers() def _get_c_extension_suffix(): - for ext, mod, typ in imp.get_suffixes(): - if typ == imp.C_EXTENSION: - return ext + suffixes = importlib.machinery.EXTENSION_SUFFIXES + return suffixes[0] if suffixes else None class build_ext(Command): diff --git a/lib-python/3/test/test_audioop.py b/lib-python/3/test/test_audioop.py --- a/lib-python/3/test/test_audioop.py +++ b/lib-python/3/test/test_audioop.py @@ -1,6 +1,7 @@ import audioop import sys import unittest +from test.support import run_unittest, impl_detail def pack(width, data): return b''.join(v.to_bytes(width, sys.byteorder, signed=True) for v in data) @@ -170,6 +171,7 @@ self.assertEqual(audioop.lin2lin(datas[4], 4, 2), packs[2](0, 0x1234, 0x4567, -0x4568, 0x7fff, -0x8000, -1)) + @impl_detail(pypy=False) def test_adpcm2lin(self): self.assertEqual(audioop.adpcm2lin(b'\x07\x7f\x7f', 1, None), (b'\x00\x00\x00\xff\x00\xff', (-179, 40))) @@ -184,6 +186,7 @@ self.assertEqual(audioop.adpcm2lin(b'\0' * 5, w, None), (b'\0' * w * 10, (0, 0))) + @impl_detail(pypy=False) def test_lin2adpcm(self): self.assertEqual(audioop.lin2adpcm(datas[1], 1, None), (b'\x07\x7f\x7f', (-221, 39))) @@ -197,6 +200,7 @@ self.assertEqual(audioop.lin2adpcm(b'\0' * w * 10, w, None), (b'\0' * 5, (0, 0))) + @impl_detail(pypy=False) def test_lin2alaw(self): self.assertEqual(audioop.lin2alaw(datas[1], 1), b'\xd5\x87\xa4\x24\xaa\x2a\x5a') @@ -205,6 +209,7 @@ self.assertEqual(audioop.lin2alaw(datas[4], 4), b'\xd5\x87\xa4\x24\xaa\x2a\x55') + @impl_detail(pypy=False) def test_alaw2lin(self): encoded = b'\x00\x03\x24\x2a\x51\x54\x55\x58\x6b\x71\x7f'\ b'\x80\x83\xa4\xaa\xd1\xd4\xd5\xd8\xeb\xf1\xff' @@ -219,6 +224,7 @@ decoded = audioop.alaw2lin(encoded, w) self.assertEqual(audioop.lin2alaw(decoded, w), encoded) + @impl_detail(pypy=False) def test_lin2ulaw(self): self.assertEqual(audioop.lin2ulaw(datas[1], 1), b'\xff\xad\x8e\x0e\x80\x00\x67') @@ -227,6 +233,7 @@ self.assertEqual(audioop.lin2ulaw(datas[4], 4), b'\xff\xad\x8e\x0e\x80\x00\x7e') + @impl_detail(pypy=False) def test_ulaw2lin(self): encoded = b'\x00\x0e\x28\x3f\x57\x6a\x76\x7c\x7e\x7f'\ b'\x80\x8e\xa8\xbf\xd7\xea\xf6\xfc\xfe\xff' @@ -341,6 +348,7 @@ self.assertRaises(audioop.error, audioop.findmax, bytes(range(256)), -2392392) + @impl_detail(pypy=False) def test_issue7673(self): state = None for data, size in INVALID_DATA: @@ -365,6 +373,7 @@ self.assertRaises(audioop.error, audioop.lin2alaw, data, size) self.assertRaises(audioop.error, audioop.lin2adpcm, data, size, state) + @impl_detail(pypy=False) def test_wrongsize(self): data = b'abcdefgh' state = None diff --git a/lib-python/3/test/test_builtin.py b/lib-python/3/test/test_builtin.py --- a/lib-python/3/test/test_builtin.py +++ b/lib-python/3/test/test_builtin.py @@ -15,7 +15,8 @@ import unittest import warnings from operator import neg -from test.support import TESTFN, unlink, run_unittest, check_warnings +from test.support import ( + TESTFN, unlink, run_unittest, check_warnings, check_impl_detail) try: import pty, signal except ImportError: @@ -558,18 +559,21 @@ self.assertEqual((g, l), ({'a': 1}, {'b': 2})) def test_exec_globals(self): - code = compile("print('Hello World!')", "", "exec") - # no builtin function - self.assertRaisesRegex(NameError, "name 'print' is not defined", - exec, code, {'__builtins__': {}}) - # __builtins__ must be a mapping type - self.assertRaises(TypeError, - exec, code, {'__builtins__': 123}) + if check_impl_detail(): + # strict __builtins__ compliance (CPython) + code = compile("print('Hello World!')", "", "exec") + # no builtin function + self.assertRaisesRegex(NameError, "name 'print' is not defined", + exec, code, {'__builtins__': {}}) + # __builtins__ must be a mapping type + self.assertRaises(TypeError, + exec, code, {'__builtins__': 123}) - # no __build_class__ function - code = compile("class A: pass", "", "exec") - self.assertRaisesRegex(NameError, "__build_class__ not found", - exec, code, {'__builtins__': {}}) + # no __build_class__ function + code = compile("class A: pass", "", "exec") + if True: + self.assertRaisesRegex(NameError, "__build_class__ not found", + exec, code, {'__builtins__': {}}) class frozendict_error(Exception): pass @@ -579,7 +583,7 @@ raise frozendict_error("frozendict is readonly") # read-only builtins - frozen_builtins = frozendict(__builtins__) + frozen_builtins = frozendict(builtins.__dict__) code = compile("__builtins__['superglobal']=2; print(superglobal)", "test", "exec") self.assertRaises(frozendict_error, exec, code, {'__builtins__': frozen_builtins}) diff --git a/lib-python/3/test/test_capi.py b/lib-python/3/test/test_capi.py --- a/lib-python/3/test/test_capi.py +++ b/lib-python/3/test/test_capi.py @@ -110,6 +110,8 @@ self.assertRaises(TypeError, _posixsubprocess.fork_exec, Z(),[b'1'],3,[1, 2],5,6,7,8,9,10,11,12,13,14,15,16,17) +@unittest.skipIf(support.check_impl_detail(pypy=True), + 'Py_AddPendingCall not currently supported.') @unittest.skipUnless(threading, 'Threading required for this test.') class TestPendingCalls(unittest.TestCase): @@ -327,6 +329,8 @@ self.assertRaises(ValueError, _testcapi.parse_tuple_and_keywords, (), {}, b'', [42]) +@unittest.skipIf(support.check_impl_detail(pypy=True), + 'Not currently supported under PyPy') @unittest.skipUnless(threading, 'Threading required for this test.') class TestThreadState(unittest.TestCase): diff --git a/lib-python/3/test/test_concurrent_futures.py b/lib-python/3/test/test_concurrent_futures.py --- a/lib-python/3/test/test_concurrent_futures.py +++ b/lib-python/3/test/test_concurrent_futures.py @@ -295,14 +295,19 @@ event = threading.Event() def future_func(): event.wait() - oldswitchinterval = sys.getswitchinterval() - sys.setswitchinterval(1e-6) + newgil = hasattr(sys, 'getswitchinterval') + if newgil: + geti, seti = sys.getswitchinterval, sys.setswitchinterval + else: + geti, seti = sys.getcheckinterval, sys.setcheckinterval + oldinterval = geti() + seti(1e-6 if newgil else 1) try: fs = {self.executor.submit(future_func) for i in range(100)} event.set() futures.wait(fs, return_when=futures.ALL_COMPLETED) finally: - sys.setswitchinterval(oldswitchinterval) + seti(oldinterval) class ProcessPoolWaitTests(ProcessPoolMixin, WaitTests, unittest.TestCase): diff --git a/lib-python/3/test/test_descr.py b/lib-python/3/test/test_descr.py --- a/lib-python/3/test/test_descr.py +++ b/lib-python/3/test/test_descr.py @@ -1196,7 +1196,7 @@ self.assertEqual(Counted.counter, 0) # Test lookup leaks [SF bug 572567] - if hasattr(gc, 'get_objects'): + if hasattr(gc, 'get_objects') and support.check_impl_detail(pypy=False): class G(object): def __eq__(self, other): return False @@ -3035,15 +3035,24 @@ class R(J): __slots__ = ["__dict__", "__weakref__"] - for cls, cls2 in ((G, H), (G, I), (I, H), (Q, R), (R, Q)): + if support.check_impl_detail(pypy=False): + lst = ((G, H), (G, I), (I, H), (Q, R), (R, Q)) + else: + # Not supported in pypy: changing the __class__ of an object + # to another __class__ that just happens to have the same slots. + # If needed, we can add the feature, but what we'll likely do + # then is to allow mostly any __class__ assignment, even if the + # classes have different __slots__, because we it's easier. + lst = ((Q, R), (R, Q)) + for cls, cls2 in lst: x = cls() x.a = 1 x.__class__ = cls2 - self.assertIs(x.__class__, cls2, + self.assertTrue(x.__class__ is cls2, "assigning %r as __class__ for %r silently failed" % (cls2, x)) self.assertEqual(x.a, 1) x.__class__ = cls - self.assertIs(x.__class__, cls, + self.assertTrue(x.__class__ is cls, "assigning %r as __class__ for %r silently failed" % (cls, x)) self.assertEqual(x.a, 1) for cls in G, J, K, L, M, N, P, R, list, Int: @@ -3055,7 +3064,8 @@ # Issue5283: when __class__ changes in __del__, the wrong # type gets DECREF'd. class O(object): - pass + def __del__(self): + pass class A(object): def __del__(self): self.__class__ = O @@ -3118,7 +3128,8 @@ except TypeError: pass else: - self.fail("%r's __dict__ can be modified" % cls) + if support.check_impl_detail(pypy=False): + self.fail("%r's __dict__ can be modified" % cls) # Modules also disallow __dict__ assignment class Module1(types.ModuleType, Base): diff --git a/lib-python/3/test/test_exceptions.py b/lib-python/3/test/test_exceptions.py --- a/lib-python/3/test/test_exceptions.py +++ b/lib-python/3/test/test_exceptions.py @@ -512,6 +512,7 @@ except MyException as e: pass obj = None + gc_collect() obj = wr() self.assertTrue(obj is None, "%s" % obj) @@ -523,6 +524,7 @@ except MyException: pass obj = None + gc_collect() obj = wr() self.assertTrue(obj is None, "%s" % obj) @@ -534,6 +536,7 @@ except: pass obj = None + gc_collect() obj = wr() self.assertTrue(obj is None, "%s" % obj) @@ -546,6 +549,7 @@ except: break obj = None + gc_collect() # XXX it seems it's not enough obj = wr() self.assertTrue(obj is None, "%s" % obj) @@ -564,6 +568,7 @@ # must clear the latter manually for our test to succeed. e.__context__ = None obj = None + gc_collect() obj = wr() # guarantee no ref cycles on CPython (don't gc_collect) if check_impl_detail(cpython=False): @@ -708,6 +713,7 @@ next(g) testfunc(g) g = obj = None + gc_collect() obj = wr() self.assertIs(obj, None) @@ -761,6 +767,7 @@ raise Exception(MyObject()) except: pass + gc_collect() self.assertEqual(e, (None, None, None)) def testUnicodeChangeAttributes(self): @@ -911,6 +918,7 @@ self.assertNotEqual(wr(), None) else: self.fail("MemoryError not raised") + gc_collect() self.assertEqual(wr(), None) @no_tracing @@ -931,6 +939,7 @@ self.assertNotEqual(wr(), None) else: self.fail("RuntimeError not raised") + gc_collect() self.assertEqual(wr(), None) def test_errno_ENOTDIR(self): diff --git a/lib-python/3/test/test_fileio.py b/lib-python/3/test/test_fileio.py --- a/lib-python/3/test/test_fileio.py +++ b/lib-python/3/test/test_fileio.py @@ -10,6 +10,7 @@ from functools import wraps from test.support import TESTFN, check_warnings, run_unittest, make_bad_fd, cpython_only +from test.support import gc_collect from collections import UserList from _io import FileIO as _FileIO @@ -32,6 +33,7 @@ self.assertEqual(self.f.tell(), p.tell()) self.f.close() self.f = None + gc_collect() self.assertRaises(ReferenceError, getattr, p, 'tell') def testSeekTell(self): diff --git a/lib-python/3/test/test_functools.py b/lib-python/3/test/test_functools.py --- a/lib-python/3/test/test_functools.py +++ b/lib-python/3/test/test_functools.py @@ -45,6 +45,8 @@ self.assertEqual(p.args, (1, 2)) self.assertEqual(p.keywords, dict(a=10, b=20)) # attributes should not be writable + if not support.check_impl_detail(): + return self.assertRaises(AttributeError, setattr, p, 'func', map) self.assertRaises(AttributeError, setattr, p, 'args', (1, 2)) self.assertRaises(AttributeError, setattr, p, 'keywords', dict(a=1, b=2)) @@ -136,6 +138,7 @@ p = proxy(f) self.assertEqual(f.func, p.func) f = None + support.gc_collect() self.assertRaises(ReferenceError, getattr, p, 'func') def test_with_bound_and_unbound_methods(self): @@ -192,9 +195,13 @@ raise IndexError f = self.thetype(object) - self.assertRaisesRegex(SystemError, - "new style getargs format but argument is not a tuple", - f.__setstate__, BadSequence()) + if support.check_impl_detail(pypy=True): + # CPython fails, pypy does not :-) + f.__setstate__(BadSequence()) + else: + self.assertRaisesRegex(SystemError, + "new style getargs format but argument is not a tuple", + f.__setstate__, BadSequence()) class PartialSubclass(functools.partial): pass @@ -223,7 +230,7 @@ updated=functools.WRAPPER_UPDATES): # Check attributes were assigned for name in assigned: - self.assertTrue(getattr(wrapper, name) is getattr(wrapped, name)) + self.assertTrue(getattr(wrapper, name) == getattr(wrapped, name)) # Check attributes were updated for name in updated: wrapper_attr = getattr(wrapper, name) diff --git a/lib-python/3/test/test_int.py b/lib-python/3/test/test_int.py --- a/lib-python/3/test/test_int.py +++ b/lib-python/3/test/test_int.py @@ -307,9 +307,10 @@ try: int(TruncReturnsNonIntegral()) except TypeError as e: - self.assertEqual(str(e), - "__trunc__ returned non-Integral" - " (type NonIntegral)") + if support.check_impl_detail(pypy=False): + self.assertEqual(str(e), + "__trunc__ returned non-Integral" + " (type NonIntegral)") else: self.fail("Failed to raise TypeError with %s" % ((base, trunc_result_base),)) diff --git a/lib-python/3/test/test_marshal.py b/lib-python/3/test/test_marshal.py --- a/lib-python/3/test/test_marshal.py +++ b/lib-python/3/test/test_marshal.py @@ -203,6 +203,7 @@ s = b'c' + (b'X' * 4*4) + b'{' * 2**20 self.assertRaises(ValueError, marshal.loads, s) + @support.impl_detail('specific recursion check') def test_recursion_limit(self): # Create a deeply nested structure. head = last = [] @@ -291,6 +292,10 @@ LARGE_SIZE = 2**31 pointer_size = 8 if sys.maxsize > 0xFFFFFFFF else 4 +if support.check_impl_detail(pypy=False): + sizeof_large_size = sys.getsizeof(LARGE_SIZE-1) +else: + sizeof_large_size = 32 # Some value for PyPy class NullWriter: def write(self, s): @@ -318,13 +323,13 @@ self.check_unmarshallable([None] * size) @support.bigmemtest(size=LARGE_SIZE, - memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), + memuse=pointer_size*12 + sizeof_large_size, dry_run=False) def test_set(self, size): self.check_unmarshallable(set(range(size))) @support.bigmemtest(size=LARGE_SIZE, - memuse=pointer_size*12 + sys.getsizeof(LARGE_SIZE-1), + memuse=pointer_size*12 + sizeof_large_size, dry_run=False) def test_frozenset(self, size): self.check_unmarshallable(frozenset(range(size))) diff --git a/lib-python/3/test/test_peepholer.py b/lib-python/3/test/test_peepholer.py --- a/lib-python/3/test/test_peepholer.py +++ b/lib-python/3/test/test_peepholer.py @@ -81,10 +81,13 @@ self.assertIn(elem, asm) def test_pack_unpack(self): + # On PyPy, "a, b = ..." is even more optimized, by removing + # the ROT_TWO. But the ROT_TWO is not removed if assigning + # to more complex expressions, so check that. for line, elem in ( ('a, = a,', 'LOAD_CONST',), - ('a, b = a, b', 'ROT_TWO',), - ('a, b, c = a, b, c', 'ROT_THREE',), + ('a[1], b = a, b', 'ROT_TWO',), + ('a, b[2], c = a, b, c', 'ROT_THREE',), ): asm = dis_single(line) self.assertIn(elem, asm) @@ -92,6 +95,8 @@ self.assertNotIn('UNPACK_TUPLE', asm) def test_folding_of_tuples_of_constants(self): + # On CPython, "a,b,c=1,2,3" turns into "a,b,c=<constant (1,2,3)>" + # but on PyPy, it turns into "a=1;b=2;c=3". for line, elem in ( ('a = 1,2,3', '((1, 2, 3))'), ('("a","b","c")', "(('a', 'b', 'c'))"), @@ -100,7 +105,8 @@ ('((1, 2), 3, 4)', '(((1, 2), 3, 4))'), ): asm = dis_single(line) - self.assertIn(elem, asm) + self.assert_(elem in asm or ( + line == 'a,b,c = 1,2,3' and 'UNPACK_TUPLE' not in asm)) self.assertNotIn('BUILD_TUPLE', asm) # Long tuples should be folded too. diff --git a/lib-python/3/test/test_subprocess.py b/lib-python/3/test/test_subprocess.py --- a/lib-python/3/test/test_subprocess.py +++ b/lib-python/3/test/test_subprocess.py @@ -1314,6 +1314,7 @@ stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=raise_it) + @support.impl_detail("PyPy's _posixsubprocess doesn't have to disable gc") def test_preexec_gc_module_failure(self): # This tests the code that disables garbage collection if the child # process will execute any Python. @@ -1964,6 +1965,7 @@ ident = id(p) pid = p.pid del p + support.gc_collect() # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) @@ -1983,6 +1985,7 @@ ident = id(p) pid = p.pid del p + support.gc_collect() os.kill(pid, signal.SIGKILL) # check that p is in the active processes list self.assertIn(ident, [id(o) for o in subprocess._active]) diff --git a/lib-python/3/test/test_sys.py b/lib-python/3/test/test_sys.py --- a/lib-python/3/test/test_sys.py +++ b/lib-python/3/test/test_sys.py @@ -405,8 +405,10 @@ self.assertEqual(len(sys.float_info), 11) self.assertEqual(sys.float_info.radix, 2) self.assertEqual(len(sys.int_info), 2) - self.assertTrue(sys.int_info.bits_per_digit % 5 == 0) - self.assertTrue(sys.int_info.sizeof_digit >= 1) + if test.support.check_impl_detail(cpython=True): + self.assertTrue(sys.int_info.bits_per_digit % 5 == 0) + else: + self.assertTrue(sys.int_info.sizeof_digit >= 1) self.assertEqual(type(sys.int_info.bits_per_digit), int) self.assertEqual(type(sys.int_info.sizeof_digit), int) self.assertIsInstance(sys.hexversion, int) @@ -503,6 +505,7 @@ self.assertTrue(repr(sys.flags)) self.assertEqual(len(sys.flags), len(attrs)) + @test.support.impl_detail("sys._clear_type_cache", pypy=False) def test_clear_type_cache(self): sys._clear_type_cache() diff --git a/lib-python/3/test/test_weakref.py b/lib-python/3/test/test_weakref.py --- a/lib-python/3/test/test_weakref.py +++ b/lib-python/3/test/test_weakref.py @@ -8,6 +8,7 @@ import copy from test import support +from test.support import gc_collect # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -88,6 +89,7 @@ ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del o + gc_collect() self.assertIsNone(ref1(), "expected reference to be invalidated") self.assertIsNone(ref2(), "expected reference to be invalidated") self.assertEqual(self.cbcalled, 2, @@ -117,13 +119,16 @@ ref1 = weakref.proxy(o, self.callback) ref2 = weakref.proxy(o, self.callback) del o + gc_collect() def check(proxy): proxy.bar self.assertRaises(ReferenceError, check, ref1) self.assertRaises(ReferenceError, check, ref2) - self.assertRaises(ReferenceError, bool, weakref.proxy(C())) + ref3 = weakref.proxy(C()) + gc_collect() + self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) def check_basic_ref(self, factory): @@ -140,6 +145,7 @@ o = factory() ref = weakref.ref(o, self.callback) del o + gc_collect() self.assertEqual(self.cbcalled, 1, "callback did not properly set 'cbcalled'") self.assertIsNone(ref(), @@ -164,6 +170,7 @@ self.assertEqual(weakref.getweakrefcount(o), 2, "wrong weak ref count for object") del proxy + gc_collect() self.assertEqual(weakref.getweakrefcount(o), 1, "wrong weak ref count for object after deleting proxy") @@ -338,6 +345,7 @@ "got wrong number of weak reference objects") del ref1, ref2, proxy1, proxy2 + gc_collect() self.assertEqual(weakref.getweakrefcount(o), 0, "weak reference objects not unlinked from" " referent when discarded.") @@ -351,6 +359,7 @@ ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del ref1 + gc_collect() self.assertEqual(weakref.getweakrefs(o), [ref2], "list of refs does not match") @@ -358,10 +367,12 @@ ref1 = weakref.ref(o, self.callback) ref2 = weakref.ref(o, self.callback) del ref2 + gc_collect() self.assertEqual(weakref.getweakrefs(o), [ref1], "list of refs does not match") del ref1 + gc_collect() self.assertEqual(weakref.getweakrefs(o), [], "list of refs not cleared") @@ -647,9 +658,11 @@ gc.collect() self.assertEqual(alist, []) + @support.impl_detail(pypy=False) def test_gc_during_ref_creation(self): self.check_gc_during_creation(weakref.ref) + @support.impl_detail(pypy=False) def test_gc_during_proxy_creation(self): self.check_gc_during_creation(weakref.proxy) @@ -811,6 +824,7 @@ self.assertTrue(mr.called) self.assertEqual(mr.value, 24) del o + gc_collect() self.assertIsNone(mr()) self.assertTrue(mr.called) @@ -917,6 +931,7 @@ n1 = len(dct) del it gc.collect() + gc.collect() n2 = len(dct) # one item may be kept alive inside the iterator self.assertIn(n1, (0, 1)) @@ -928,6 +943,7 @@ def test_weak_valued_len_cycles(self): self.check_len_cycles(weakref.WeakValueDictionary, lambda k: (1, k)) + @support.impl_detail(pypy=False) def check_len_race(self, dict_type, cons): # Extended sanity checks for len() in the face of cyclic collection self.addCleanup(gc.set_threshold, *gc.get_threshold()) @@ -976,15 +992,18 @@ del items1, items2 self.assertEqual(len(dict), self.COUNT) del objects[0] + gc_collect() self.assertEqual(len(dict), self.COUNT - 1, "deleting object did not cause dictionary update") del objects, o + gc_collect() self.assertEqual(len(dict), 0, "deleting the values did not clear the dictionary") # regression on SF bug #447152: dict = weakref.WeakValueDictionary() self.assertRaises(KeyError, dict.__getitem__, 1) dict[2] = C() + gc_collect() self.assertRaises(KeyError, dict.__getitem__, 2) def test_weak_keys(self): @@ -1005,9 +1024,11 @@ del items1, items2 self.assertEqual(len(dict), self.COUNT) del objects[0] + gc_collect() self.assertEqual(len(dict), (self.COUNT - 1), "deleting object did not cause dictionary update") del objects, o + gc_collect() self.assertEqual(len(dict), 0, "deleting the keys did not clear the dictionary") o = Object(42) @@ -1368,6 +1389,7 @@ for o in objs: count += 1 del d[o] + gc_collect() self.assertEqual(len(d), 0) self.assertEqual(count, 2) @@ -1389,6 +1411,7 @@ libreftest = """ Doctest for examples in the library reference: weakref.rst +>>> from test.support import gc_collect >>> import weakref >>> class Dict(dict): ... pass @@ -1408,6 +1431,7 @@ >>> o is o2 True >>> del o, o2 +>>> gc_collect() >>> print(r()) None @@ -1460,6 +1484,7 @@ >>> id2obj(a_id) is a True >>> del a +>>> gc_collect() >>> try: ... id2obj(a_id) ... except KeyError: diff --git a/lib-python/3/test/test_weakset.py b/lib-python/3/test/test_weakset.py --- a/lib-python/3/test/test_weakset.py +++ b/lib-python/3/test/test_weakset.py @@ -416,11 +416,13 @@ n1 = len(s) del it gc.collect() + gc.collect() n2 = len(s) # one item may be kept alive inside the iterator self.assertIn(n1, (0, 1)) self.assertEqual(n2, 0) + @support.impl_detail("PyPy has no cyclic collection", pypy=False) def test_len_race(self): # Extended sanity checks for len() in the face of cyclic collection self.addCleanup(gc.set_threshold, *gc.get_threshold()) diff --git a/lib_pypy/_curses.py b/lib_pypy/_curses.py --- a/lib_pypy/_curses.py +++ b/lib_pypy/_curses.py @@ -309,11 +309,9 @@ #endif int _m_ispad(WINDOW *win) { -#if defined WINDOW_HAS_FLAGS + // <curses.h> may not have _flags (and possibly _ISPAD), + // but for now let's assume that <ncurses.h> always has it return (win->_flags & _ISPAD); -#else - return 0; -#endif } void _m_getsyx(int *yx) { diff --git a/lib_pypy/_pypy_testcapi.py b/lib_pypy/_pypy_testcapi.py --- a/lib_pypy/_pypy_testcapi.py +++ b/lib_pypy/_pypy_testcapi.py @@ -1,5 +1,6 @@ import os, sys, imp import tempfile, binascii +import importlib.machinery def get_hashed_dir(cfile): @@ -28,9 +29,8 @@ def _get_c_extension_suffix(): - for ext, mod, typ in imp.get_suffixes(): - if typ == imp.C_EXTENSION: - return ext + suffixes = importlib.machinery.EXTENSION_SUFFIXES + return suffixes[0] if suffixes else None def compile_shared(csource, modulename, output_dir=None): diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,7 +1,17 @@ -import sys, os, binascii, imp, shutil +import sys, os, binascii, shutil from . import __version__ from . import ffiplatform +if sys.version_info >= (3, 3): + import importlib.machinery + def extension_suffixes(): + return importlib.machinery.EXTENSION_SUFFIXES[:] +else: + import imp + def extension_suffixes(): + return [suffix for suffix, _, type in imp.get_suffixes() + if type == imp.C_EXTENSION] + class Verifier(object): @@ -222,11 +232,7 @@ pass def _get_so_suffixes(): - suffixes = [] - for suffix, mode, type in imp.get_suffixes(): - if type == imp.C_EXTENSION: - suffixes.append(suffix) - + suffixes = extension_suffixes() if not suffixes: # bah, no C_EXTENSION available. Occurs on pypy without cpyext if sys.platform == 'win32': diff --git a/lib_pypy/pyrepl/module_lister.py b/lib_pypy/pyrepl/module_lister.py --- a/lib_pypy/pyrepl/module_lister.py +++ b/lib_pypy/pyrepl/module_lister.py @@ -40,8 +40,8 @@ return sorted(set(l)) def _make_module_list(): - import imp - suffs = [x[0] for x in imp.get_suffixes() if x[0] != '.pyc'] + import importlib.machinery + suffs = [x for x in importlib.machinery.all_suffixes() if x != '.pyc'] suffs.sort(reverse=True) _packages[''] = list(sys.builtin_module_names) for dir in sys.path: diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -37,7 +37,7 @@ "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "_continuation", "_cffi_backend", "_csv", "_pypyjson", "_posixsubprocess", # "cppyy", "micronumpy" - "faulthandler", + "faulthandler", "_lzma", ]) translation_modules = default_modules.copy() @@ -106,6 +106,7 @@ "_hashlib" : ["pypy.module._ssl.interp_ssl"], "_minimal_curses": ["pypy.module._minimal_curses.fficurses"], "_continuation": ["rpython.rlib.rstacklet"], + "_lzma" : ["pypy.module._lzma.interp_lzma"], } def get_module_validator(modname): diff --git a/pypy/doc/coding-guide.rst b/pypy/doc/coding-guide.rst --- a/pypy/doc/coding-guide.rst +++ b/pypy/doc/coding-guide.rst @@ -740,7 +740,7 @@ Adding an entry under pypy/module (e.g. mymodule) entails automatic creation of a new config option (such as --withmod-mymodule and ---withoutmod-mymodule (the later being the default)) for py.py and +--withoutmod-mymodule (the latter being the default)) for py.py and translate.py. Testing modules in ``lib_pypy/`` @@ -931,7 +931,7 @@ assert self.result == 2 ** 6 which executes the code string function with the given arguments at app level. -Note the use of ``w_result`` in ``setup_class`` but self.result in the test +Note the use of ``w_result`` in ``setup_class`` but self.result in the test. Here is how to define an app level class in ``setup_class`` that can be used in subsequent tests:: diff --git a/pypy/doc/config/objspace.usemodules._lzma.txt b/pypy/doc/config/objspace.usemodules._lzma.txt new file mode 100644 --- /dev/null +++ b/pypy/doc/config/objspace.usemodules._lzma.txt @@ -0,0 +1,2 @@ +Use the '_lzma' module. +This module is expected to be working and is included by default. diff --git a/pypy/doc/cpython_differences.rst b/pypy/doc/cpython_differences.rst --- a/pypy/doc/cpython_differences.rst +++ b/pypy/doc/cpython_differences.rst @@ -328,7 +328,7 @@ * directly calling the internal magic methods of a few built-in types with invalid arguments may have a slightly different result. For example, ``[].__add__(None)`` and ``(2).__add__(None)`` both return - ``NotImplemented`` on PyPy; on CPython, only the later does, and the + ``NotImplemented`` on PyPy; on CPython, only the latter does, and the former raises ``TypeError``. (Of course, ``[]+None`` and ``2+None`` both raise ``TypeError`` everywhere.) This difference is an implementation detail that shows up because of internal C-level slots diff --git a/pypy/interpreter/astcompiler/assemble.py b/pypy/interpreter/astcompiler/assemble.py --- a/pypy/interpreter/astcompiler/assemble.py +++ b/pypy/interpreter/astcompiler/assemble.py @@ -7,7 +7,7 @@ from pypy.tool import stdlib_opcode as ops from pypy.interpreter.error import OperationError -from rpython.rlib.objectmodel import we_are_translated +from rpython.rlib.objectmodel import specialize, we_are_translated from rpython.rlib import rfloat @@ -141,11 +141,12 @@ i += 1 return result -def _list_to_dict(l, offset=0): +@specialize.argtype(0) +def _iter_to_dict(iterable, offset=0): result = {} index = offset - for i in range(len(l)): - result[l[i]] = index + for item in iterable: + result[item] = index index += 1 return result @@ -161,10 +162,10 @@ self.first_block = self.new_block() self.use_block(self.first_block) self.names = {} - self.var_names = _list_to_dict(scope.varnames) + self.var_names = _iter_to_dict(scope.varnames) self.cell_vars = _make_index_dict_filter(scope.symbols, symtable.SCOPE_CELL) - self.free_vars = _list_to_dict(scope.free_vars, len(self.cell_vars)) + self.free_vars = _iter_to_dict(scope.free_vars, len(self.cell_vars)) self.w_consts = space.newdict() self.argcount = 0 self.kwonlyargcount = 0 diff --git a/pypy/interpreter/astcompiler/codegen.py b/pypy/interpreter/astcompiler/codegen.py --- a/pypy/interpreter/astcompiler/codegen.py +++ b/pypy/interpreter/astcompiler/codegen.py @@ -244,7 +244,7 @@ self.emit_op_arg(op, self.add_name(container, identifier)) def possible_docstring(self, node): - if isinstance(node, ast.Expr): + if isinstance(node, ast.Expr) and self.compile_info.optimize < 2: expr_value = node.value if isinstance(expr_value, ast.Str): return expr_value diff --git a/pypy/interpreter/astcompiler/symtable.py b/pypy/interpreter/astcompiler/symtable.py --- a/pypy/interpreter/astcompiler/symtable.py +++ b/pypy/interpreter/astcompiler/symtable.py @@ -37,7 +37,7 @@ self.roles = {} self.varnames = [] self.children = [] - self.free_vars = [] + self.free_vars = {} self.temp_name_counter = 1 self.has_free = False self.child_has_free = False @@ -136,7 +136,9 @@ err = "no binding for nonlocal '%s' found" % (name,) raise SyntaxError(err, self.lineno, self.col_offset) self.symbols[name] = SCOPE_FREE + self.free_vars[name] = None free[name] = None + self.has_free = True elif flags & SYM_BOUND: self.symbols[name] = SCOPE_LOCAL local[name] = None @@ -146,7 +148,7 @@ pass elif bound and name in bound: self.symbols[name] = SCOPE_FREE - self.free_vars.append(name) + self.free_vars[name] = None free[name] = None self.has_free = True elif name in globs: @@ -203,7 +205,7 @@ except KeyError: if name in bound: self.symbols[name] = SCOPE_FREE - self.free_vars.append(name) + self.free_vars[name] = None else: if role_here & (SYM_BOUND | SYM_GLOBAL) and \ self._hide_bound_from_nested_scopes: @@ -212,7 +214,7 @@ # scope. We add the name to the class scope's list of free # vars, so it will be passed through by the interpreter, but # we leave the scope alone, so it can be local on its own. - self.free_vars.append(name) + self.free_vars[name] = None self._check_optimization() free.update(new_free) @@ -244,18 +246,12 @@ return Scope.note_symbol(self, identifier, role) def note_yield(self, yield_node): - if self.return_with_value: - raise SyntaxError("'return' with argument inside generator", - self.ret.lineno, self.ret.col_offset) self.is_generator = True if self._in_try_body_depth > 0: self.has_yield_inside_try = True def note_return(self, ret): if ret.value: - if self.is_generator: - raise SyntaxError("'return' with argument inside generator", - ret.lineno, ret.col_offset) self.return_with_value = True self.ret = ret diff --git a/pypy/interpreter/astcompiler/test/test_compiler.py b/pypy/interpreter/astcompiler/test/test_compiler.py --- a/pypy/interpreter/astcompiler/test/test_compiler.py +++ b/pypy/interpreter/astcompiler/test/test_compiler.py @@ -808,6 +808,28 @@ return y""" yield self.st, test, "f()", 4 + def test_nonlocal_from_arg(self): + test = """if 1: + def test1(x): + def test2(): + nonlocal x + def test3(): + return x + return test3() + return test2()""" + yield self.st, test, "test1(2)", 2 + + def test_class_nonlocal_from_arg(self): + test = """if 1: + def f(x): + class c: + nonlocal x + x += 1 + def get(self): + return x + return c().get()""" + yield self.st, test, "f(3)", 4 + def test_lots_of_loops(self): source = "for x in y: pass\n" * 1000 compile_with_astcompiler(source, 'exec', self.space) diff --git a/pypy/interpreter/astcompiler/test/test_symtable.py b/pypy/interpreter/astcompiler/test/test_symtable.py --- a/pypy/interpreter/astcompiler/test/test_symtable.py +++ b/pypy/interpreter/astcompiler/test/test_symtable.py @@ -361,8 +361,7 @@ assert exc.msg == "'yield' outside function" for input in ("yield\n return x", "return x\n yield"): input = "def f():\n " + input - exc = py.test.raises(SyntaxError, self.func_scope, input).value - assert exc.msg == "'return' with argument inside generator" + scp = self.func_scope(input) scp = self.func_scope("def f():\n return\n yield x") def test_yield_inside_try(self): diff --git a/pypy/interpreter/astcompiler/validate.py b/pypy/interpreter/astcompiler/validate.py --- a/pypy/interpreter/astcompiler/validate.py +++ b/pypy/interpreter/astcompiler/validate.py @@ -14,6 +14,9 @@ def __init__(self, message): self.message = message + def __str__(self): + return self.message + def expr_context_name(ctx): if not 1 <= ctx <= len(ast.expr_context_to_class): diff --git a/pypy/interpreter/generator.py b/pypy/interpreter/generator.py --- a/pypy/interpreter/generator.py +++ b/pypy/interpreter/generator.py @@ -99,7 +99,7 @@ # if the frame is now marked as finished, it was RETURNed from if frame.frame_finished_execution: self.frame = None - raise OperationError(space.w_StopIteration, space.w_None) + raise OperationError(space.w_StopIteration, w_result) else: return w_result # YIELDed finally: diff --git a/pypy/interpreter/module.py b/pypy/interpreter/module.py --- a/pypy/interpreter/module.py +++ b/pypy/interpreter/module.py @@ -3,7 +3,7 @@ """ from pypy.interpreter.baseobjspace import W_Root -from pypy.interpreter.error import OperationError +from pypy.interpreter.error import OperationError, oefmt from rpython.rlib.objectmodel import we_are_translated @@ -123,3 +123,10 @@ except OperationError: __file__ = u'?' return space.wrap(u"<module %s from %s>" % (name, __file__)) + + def descr_module__dir__(self, space): + w_dict = space.getattr(self, space.wrap('__dict__')) + if not space.isinstance_w(w_dict, space.w_dict): + raise oefmt(space.w_TypeError, "%N.__dict__ is not a dictionary", + self) + return space.call_function(space.w_list, w_dict) diff --git a/pypy/interpreter/nestedscope.py b/pypy/interpreter/nestedscope.py --- a/pypy/interpreter/nestedscope.py +++ b/pypy/interpreter/nestedscope.py @@ -75,8 +75,9 @@ if self.w_value is None: content = "empty" else: - content = "%s object at 0x%x" % (space.type(self.w_value).name, uid(self.w_value)) - s = "<cell at 0x%x: %s>" % (uid(self), content) + content = "%s object at 0x%s" % (space.type(self.w_value).name, + self.w_value.getaddrstring(space)) + s = "<cell at 0x%s: %s>" % (self.getaddrstring(space), content) return space.wrap(s.decode('utf-8')) def descr__cell_contents(self, space): diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -4,7 +4,7 @@ The bytecode interpreter itself is implemented by the PyFrame class. """ -import dis, imp, struct, types, new, sys +import imp, struct, types, new, sys from pypy.interpreter import eval from pypy.interpreter.signature import Signature @@ -13,6 +13,7 @@ from pypy.interpreter.astcompiler.consts import ( CO_OPTIMIZED, CO_NEWLOCALS, CO_VARARGS, CO_VARKEYWORDS, CO_NESTED, CO_GENERATOR, CO_KILL_DOCSTRING, CO_YIELD_INSIDE_TRY) +from pypy.tool import dis3 from pypy.tool.stdlib_opcode import opcodedesc, HAVE_ARGUMENT from rpython.rlib.rarithmetic import intmask from rpython.rlib.objectmodel import compute_hash, we_are_translated @@ -60,7 +61,7 @@ else: varargname = None if code.co_flags & CO_VARKEYWORDS: - kwargname = code.co_varnames[argcount] + kwargname = code.co_varnames[argcount+kwonlyargcount] argcount += 1 else: kwargname = None @@ -245,33 +246,6 @@ if isinstance(w_co, PyCode): w_co.remove_docstrings(space) - def _to_code(self): - """For debugging only.""" - consts = [None] * len(self.co_consts_w) - num = 0 - for w in self.co_consts_w: - if isinstance(w, PyCode): - consts[num] = w._to_code() - else: - consts[num] = self.space.unwrap(w) - num += 1 - assert self.co_kwonlyargcount == 0, 'kwonlyargcount is py3k only, cannot turn this code object into a Python2 one' - return new.code(self.co_argcount, - #self.co_kwonlyargcount, # this does not exists in python2 - self.co_nlocals, - self.co_stacksize, - self.co_flags, - self.co_code, - tuple(consts), - tuple(self.co_names), - tuple(self.co_varnames), - self.co_filename, - self.co_name, - self.co_firstlineno, - self.co_lnotab, - tuple(self.co_freevars), - tuple(self.co_cellvars)) - def exec_host_bytecode(self, w_globals, w_locals): if sys.version_info < (2, 7): raise Exception("PyPy no longer supports Python 2.6 or lower") @@ -280,11 +254,11 @@ return frame.run() def dump(self): - """A dis.dis() dump of the code object.""" - print 'WARNING: dumping a py3k bytecode using python2 opmap, the result might be inaccurate or wrong' - print - co = self._to_code() - dis.dis(co) + """NOT_RPYTHON: A dis.dis() dump of the code object.""" + if not hasattr(self, 'co_consts'): + self.co_consts = [w if isinstance(w, PyCode) else self.space.unwrap(w) + for w in self.co_consts_w] + dis3.dis(self) def fget_co_consts(self, space): return space.newtuple(self.co_consts_w) diff --git a/pypy/interpreter/pycompiler.py b/pypy/interpreter/pycompiler.py --- a/pypy/interpreter/pycompiler.py +++ b/pypy/interpreter/pycompiler.py @@ -96,7 +96,7 @@ XXX: This class should override the baseclass implementation of compile_command() in order to optimize it, especially in case - of incomplete inputs (e.g. we shouldn't re-compile from sracth + of incomplete inputs (e.g. we shouldn't re-compile from scratch the whole source after having only added a new '\n') """ def __init__(self, space, override_version=None): diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -1006,13 +1006,14 @@ else: w_retval = space.call_method(w_gen, "send", w_value) except OperationError as e: - if not e.match(self.space, self.space.w_StopIteration): + if not e.match(space, space.w_StopIteration): raise self.popvalue() # Remove iter from stack + e.normalize_exception(space) try: w_value = space.getattr(e.get_w_value(space), space.wrap("value")) except OperationError as e: - if not e.match(self.space, self.space.w_AttributeError): + if not e.match(space, space.w_AttributeError): raise w_value = space.w_None self.pushvalue(w_value) diff --git a/pypy/interpreter/pytraceback.py b/pypy/interpreter/pytraceback.py --- a/pypy/interpreter/pytraceback.py +++ b/pypy/interpreter/pytraceback.py @@ -50,6 +50,10 @@ self.lasti = space.int_w(w_lasti) self.next = space.interp_w(PyTraceback, w_next, can_be_None=True) + def descr__dir__(self, space): + return space.newlist([space.wrap(n) for n in + ['tb_frame', 'tb_next', 'tb_lasti', 'tb_lineno']]) + def record_application_traceback(space, operror, frame, last_instruction): if frame.pycode.hidden_applevel: diff --git a/pypy/interpreter/test/test_compiler.py b/pypy/interpreter/test/test_compiler.py --- a/pypy/interpreter/test/test_compiler.py +++ b/pypy/interpreter/test/test_compiler.py @@ -268,10 +268,7 @@ def test_return_in_generator(self): code = 'def f():\n return None\n yield 19\n' - e = py.test.raises(OperationError, self.compiler.compile, code, '', 'single', 0) - ex = e.value - ex.normalize_exception(self.space) - assert ex.match(self.space, self.space.w_SyntaxError) + self.compiler.compile(code, '', 'single', 0) def test_yield_in_finally(self): code ='def f():\n try:\n yield 19\n finally:\n pass\n' @@ -717,6 +714,27 @@ else: py.test.fail("Did not raise") + def test_signature_kwargname(self): + from pypy.interpreter.pycode import cpython_code_signature + from pypy.interpreter.signature import Signature + + def find_func(code): + for w_const in code.co_consts_w: + if isinstance(w_const, PyCode): + return w_const + + snippet = 'def f(a, b, m=1, n=2, **kwargs): pass' + containing_co = self.compiler.compile(snippet, '<string>', 'single', 0) + co = find_func(containing_co) + sig = cpython_code_signature(co) + assert sig == Signature(['a', 'b', 'm', 'n'], None, 'kwargs', []) + + snippet = 'def f(a, b, *, m=1, n=2, **kwargs): pass' + containing_co = self.compiler.compile(snippet, '<string>', 'single', 0) + co = find_func(containing_co) + sig = cpython_code_signature(co) + assert sig == Signature(['a', 'b'], None, 'kwargs', ['m', 'n']) + class AppTestCompiler: diff --git a/pypy/interpreter/test/test_generator.py b/pypy/interpreter/test/test_generator.py --- a/pypy/interpreter/test/test_generator.py +++ b/pypy/interpreter/test/test_generator.py @@ -301,6 +301,39 @@ raise StopIteration assert tuple(f()) == (1,) + def test_yield_return(self): + """ + def f(): + yield 1 + return 2 + g = f() + assert next(g) == 1 + try: + next(g) + except StopIteration as e: + assert e.value == 2 + else: + assert False, 'Expected StopIteration' + """ + + def test_yield_from_return(self): + """ + def f1(): + result = yield from f2() + return result + def f2(): + yield 1 + return 2 + g = f1() + assert next(g) == 1 + try: + next(g) + except StopIteration as e: + assert e.value == 2 + else: + assert False, 'Expected StopIteration' + """ + def test_should_not_inline(space): from pypy.interpreter.generator import should_not_inline diff --git a/pypy/interpreter/test/test_module.py b/pypy/interpreter/test/test_module.py --- a/pypy/interpreter/test/test_module.py +++ b/pypy/interpreter/test/test_module.py @@ -68,6 +68,11 @@ m = type(_pypy_interact).__new__(type(_pypy_interact)) assert repr(m).startswith("<module '?'") + def test_dir(self): + import sys + items = sys.__dir__() + assert sorted(items) == dir(sys) + def test_package(self): import sys import os diff --git a/pypy/interpreter/test/test_pycode.py b/pypy/interpreter/test/test_pycode.py new file mode 100644 --- /dev/null +++ b/pypy/interpreter/test/test_pycode.py @@ -0,0 +1,19 @@ +import sys, StringIO + +def test_dump(space): + """test that pycode.dump kind of works with py3 opcodes""" + compiler = space.createcompiler() + code = compiler.compile('lambda *, y=7: None', 'filename', 'exec', 0) + output = None + stdout = sys.stdout + try: + sys.stdout = StringIO.StringIO() + code.dump() + output = sys.stdout.getvalue() + sys.stdout.close() + finally: + sys.stdout = stdout + print '>>>\n' + output + '\n<<<' + assert ' 1 (7)' in output + assert ' 3 (None)' in output + assert ' 16 RETURN_VALUE ' in output diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -779,6 +779,7 @@ __new__ = interp2app(Module.descr_module__new__.im_func), __init__ = interp2app(Module.descr_module__init__), __repr__ = interp2app(Module.descr_module__repr__), + __dir__ = interp2app(Module.descr_module__dir__), __reduce__ = interp2app(Module.descr__reduce__), __dict__ = GetSetProperty(descr_get_dict, cls=Module), # module dictionaries are readonly attributes __doc__ = 'module(name[, doc])\n\nCreate a module object.\nThe name must be a string; the optional doc argument can have any type.' @@ -912,6 +913,7 @@ PyTraceback.typedef = TypeDef("traceback", __reduce__ = interp2app(PyTraceback.descr__reduce__), __setstate__ = interp2app(PyTraceback.descr__setstate__), + __dir__ = interp2app(PyTraceback.descr__dir__), tb_frame = interp_attrproperty('frame', cls=PyTraceback), tb_lasti = interp_attrproperty('lasti', cls=PyTraceback), tb_lineno = GetSetProperty(PyTraceback.descr_tb_lineno), diff --git a/pypy/module/__builtin__/app_inspect.py b/pypy/module/__builtin__/app_inspect.py --- a/pypy/module/__builtin__/app_inspect.py +++ b/pypy/module/__builtin__/app_inspect.py @@ -7,8 +7,8 @@ from __pypy__ import lookup_special -def _caller_locals(): - return sys._getframe(0).f_locals +def _caller_locals(): + return sys._getframe(0).f_locals def vars(*obj): """Return a dictionary of all the attributes currently bound in obj. If @@ -18,11 +18,10 @@ return _caller_locals() elif len(obj) != 1: raise TypeError("vars() takes at most 1 argument.") - else: - try: - return obj[0].__dict__ - except AttributeError: - raise TypeError("vars() argument must have __dict__ attribute") + try: + return obj[0].__dict__ + except AttributeError: + raise TypeError("vars() argument must have __dict__ attribute") def dir(*args): """dir([object]) -> list of strings @@ -38,76 +37,16 @@ attributes of its class's base classes. """ if len(args) > 1: - raise TypeError("dir expected at most 1 arguments, got %d" - % len(args)) + raise TypeError("dir expected at most 1 arguments, got %d" % len(args)) if len(args) == 0: - local_names = list(_caller_locals().keys()) # 2 stackframes away - local_names.sort() - return local_names - - import types + return sorted(_caller_locals().keys()) # 2 stackframes away obj = args[0] - - dir_meth = lookup_special(obj, "__dir__") + dir_meth = lookup_special(obj, '__dir__') if dir_meth is not None: - result = dir_meth() - if not isinstance(result, list): - result = list(result) # Will throw TypeError if not iterable - result.sort() - return result - elif isinstance(obj, types.ModuleType): - try: - result = list(obj.__dict__) - result.sort() - return result - except AttributeError: - return [] - - elif isinstance(obj, type): - #Don't look at __class__, as metaclass methods would be confusing. - result = list(_classdir(obj).keys()) - result.sort() - return result - - else: #(regular item) - Dict = {} - try: - if isinstance(obj.__dict__, dict): - Dict.update(obj.__dict__) - except AttributeError: - pass - try: - Dict.update(_classdir(obj.__class__)) - except AttributeError: - pass - result = list(Dict.keys()) - result.sort() - return result - -def _classdir(klass): - """Return a dict of the accessible attributes of class/type klass. - - This includes all attributes of klass and all of the - base classes recursively. - - The values of this dict have no meaning - only the keys have - meaning. - """ - Dict = {} - try: - Dict.update(klass.__dict__) - except AttributeError: pass - try: - # XXX - Use of .__mro__ would be suggested, if the existance - # of that attribute could be guarranted. - bases = klass.__bases__ - except AttributeError: pass - else: - try: - #Note that since we are only interested in the keys, - # the order we merge classes is unimportant - for base in bases: - Dict.update(_classdir(base)) - except TypeError: pass - return Dict + # obscure: lookup_special won't bind None.__dir__! + result = dir_meth(obj) if obj is None else dir_meth() + # Will throw TypeError if not iterable + return sorted(result) + # we should never reach here since object.__dir__ exists + return [] diff --git a/pypy/module/__builtin__/functional.py b/pypy/module/__builtin__/functional.py --- a/pypy/module/__builtin__/functional.py +++ b/pypy/module/__builtin__/functional.py @@ -670,8 +670,8 @@ def descr_reduce(self, space): w_map = space.getattr(space.getbuiltinmodule('builtins'), space.wrap('map')) - args = [self.w_fun] + self.iterators_w - return space.newtuple([w_map, space.newtuple(args)]) + args_w = [self.w_fun] + self.iterators_w + return space.newtuple([w_map, space.newtuple(args_w)]) def W_Map___new__(space, w_subtype, w_fun, args_w): @@ -721,9 +721,9 @@ def descr_reduce(self, space): w_filter = space.getattr(space.getbuiltinmodule('builtins'), space.wrap('filter')) - args = [space.w_None if self.no_predicate else self.w_predicate, - self.iterable] - return space.newtuple([w_filter, space.newtuple(args)]) + args_w = [space.w_None if self.no_predicate else self.w_predicate, + self.iterable] + return space.newtuple([w_filter, space.newtuple(args_w)]) def W_Filter___new__(space, w_subtype, w_predicate, w_iterable): diff --git a/pypy/module/__builtin__/test/test_compile.py b/pypy/module/__builtin__/test/test_compile.py --- a/pypy/module/__builtin__/test/test_compile.py +++ b/pypy/module/__builtin__/test/test_compile.py @@ -1,8 +1,7 @@ class AppTestCompile: - # TODO: This test still fails for now because the docstrings are not - # removed with optimize=2. - def untest_compile(self): + def test_compile(self): + """Clone of the part of the original test that was failing.""" import ast codestr = '''def f(): @@ -37,7 +36,7 @@ assert rv == (debugval, docstring) def test_assert_remove(self): - """Test just removal of the asserts with optimize=1.""" + """Test removal of the asserts with optimize=1.""" import ast code = """def f(): @@ -50,9 +49,73 @@ exec(compiled, ns) ns['f']() + def test_docstring_remove(self): + """Test removal of docstrings with optimize=2.""" + import ast + import marshal -# TODO: Remove docstrings with optimize=2. + code = """ +'module_doc' + +def f(): + 'func_doc' + +class C: + 'class_doc' +""" + tree = ast.parse(code) + for to_compile in [code, tree]: + compiled = compile(to_compile, "<test>", "exec", optimize=2) + + ns = {} + exec(compiled, ns) + assert '__doc__' not in ns + assert ns['f'].__doc__ is None + assert ns['C'].__doc__ is None + + # Check that the docstrings are gone from the bytecode and not just + # inaccessible. + marshalled = str(marshal.dumps(compiled)) + assert 'module_doc' not in marshalled + assert 'func_doc' not in marshalled + assert 'class_doc' not in marshalled + + +class TestOptimizeO: + """Test interaction of -O flag and optimize parameter of compile.""" + + def setup_method(self, method): + space = self.space + self._sys_debug = space.sys.debug + # imitate -O + space.sys.debug = False + + def teardown_method(self, method): + self.space.sys.debug = self._sys_debug + + def test_O_optmize_0(self): + """Test that assert is not ignored if -O flag is set but optimize=0.""" + space = self.space + w_res = space.appexec([], """(): + assert False # check that our -O imitation hack works + try: + exec(compile('assert False', '', 'exec', optimize=0)) + except AssertionError: + return True + else: + return False + """) + assert space.unwrap(w_res) + + def test_O_optimize__1(self): + """Test that assert is ignored with -O and optimize=-1.""" + space = self.space + space.appexec([], """(): + exec(compile('assert False', '', 'exec', optimize=-1)) + """) + + # TODO: Check the value of __debug__ inside of the compiled block! # According to the documentation, it should follow the optimize flag. -# TODO: It would also be good to test that with the assert is not removed and -# is executed when -O flag is set but optimize=0. +# However, cpython3.5.0a0 behaves the same way as PyPy (__debug__ follows +# -O, -OO flags of the interpreter). diff --git a/pypy/module/__builtin__/test/test_dir.py b/pypy/module/__builtin__/test/test_dir.py --- a/pypy/module/__builtin__/test/test_dir.py +++ b/pypy/module/__builtin__/test/test_dir.py @@ -24,3 +24,82 @@ def __dir__(self): return 42 raises(TypeError, dir, Foo()) + + def test_dir_traceback(self): + """Test dir() of traceback.""" + try: + raise IndexError + except Exception as e: + tb_dir = dir(e.__traceback__) + assert tb_dir == ['tb_frame', 'tb_lasti', 'tb_lineno', 'tb_next'] + + def test_dir_object_inheritance(self): + """Dir should behave the same regardless of inheriting from object.""" + class A: + pass + + class B(object): + pass + assert dir(A) == dir(B) + + def test_dir_sanity(self): + """Test that dir returns reasonable items.""" + class A(object): + a = 1 + + class B(A): + y = 2 + + b = B() + b.z = 1 + + names = dir(b) + for name in 'ayz': + assert name in names + + assert '__doc__' in names + assert '__module__' in names + assert '__dict__' in names + assert '__dir__' in names + assert '__weakref__' in names + assert '__class__' in names + assert '__format__' in names + # Not an exhaustive list, but will be enough if dir is very broken. + + def test_dir_module(self): + import sys + assert dir(sys) == list(sorted(sys.__dict__)) + + def test_dir_list(self): + """Check that dir([]) has methods from list and from object.""" + names = dir([]) + + dct = {} + dct.update(list.__dict__) + dct.update(object.__dict__) + + assert names == sorted(dct) + + def test_dir_builtins(self): + """Test that builtin objects have sane __dir__().""" + import sys + + for builtin in [sys, object(), [], {}, {1}, "", 1, (), sys, + map(ord, "abc"), filter(None, "abc"), zip([1, 2], [3, 4]), + compile('1', '', 'exec')]: + assert sorted(builtin.__dir__()) == dir(builtin) + + def test_dir_type(self): + """Test .__dir__() and dir(...) behavior on types. + + * t.__dir__() throws a TypeError, + * dir(t) == sorted(t().__dir__()) + + This is the behavior that I observe with cpython3.3.2. + """ + for t in [int, list, tuple, set, str]: + raises(TypeError, t.__dir__) + assert dir(t) == sorted(t().__dir__()) + + def test_dir_none(self): + assert dir(None) == sorted(None.__dir__()) diff --git a/pypy/module/__builtin__/test/test_format.py b/pypy/module/__builtin__/test/test_format.py new file mode 100644 --- /dev/null +++ b/pypy/module/__builtin__/test/test_format.py @@ -0,0 +1,38 @@ +class AppTestFormat: + + def test_format(self): + """Test deprecation warnings from format(object(), 'nonempty')""" + + import warnings + + def test_deprecated(obj, fmt_str, should_raise_warning): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always", DeprecationWarning) + format(obj, fmt_str) + if should_raise_warning: + assert len(w) == 1 + assert isinstance(w[0].message, DeprecationWarning) + assert 'object.__format__ with a non-empty format string '\ + in str(w[0].message) + else: + assert len(w) == 0 + + fmt_strs = ['', 's'] + + class A: + def __format__(self, fmt_str): + return format('', fmt_str) + + for fmt_str in fmt_strs: + test_deprecated(A(), fmt_str, False) + + class B: + pass + + class C(object): + pass + + for cls in [object, B, C]: + for fmt_str in fmt_strs: + print(cls, fmt_str) + test_deprecated(cls(), fmt_str, len(fmt_str) != 0) diff --git a/pypy/module/_io/interp_textio.py b/pypy/module/_io/interp_textio.py --- a/pypy/module/_io/interp_textio.py +++ b/pypy/module/_io/interp_textio.py @@ -287,7 +287,8 @@ try: w_locale = space.call_method(space.builtin, '__import__', space.wrap('locale')) - w_encoding = space.call_method(w_locale, 'getpreferredencoding') + w_encoding = space.call_method(w_locale, 'getpreferredencoding', + space.w_False) except OperationError as e: # getpreferredencoding() may also raise ImportError if not e.match(space, space.w_ImportError): diff --git a/pypy/module/_lzma/__init__.py b/pypy/module/_lzma/__init__.py new file mode 100644 --- /dev/null +++ b/pypy/module/_lzma/__init__.py @@ -0,0 +1,20 @@ +from pypy.interpreter.mixedmodule import MixedModule + +class Module(MixedModule): + # The private part of the lzma module. + + applevel_name = '_lzma' + + interpleveldefs = { + 'LZMACompressor': 'interp_lzma.W_LZMACompressor', + 'LZMADecompressor': 'interp_lzma.W_LZMADecompressor', + '_encode_filter_properties': 'interp_lzma.encode_filter_properties', + '_decode_filter_properties': 'interp_lzma.decode_filter_properties', + 'FORMAT_AUTO': 'space.wrap(interp_lzma.FORMAT_AUTO)', + 'FORMAT_XZ': 'space.wrap(interp_lzma.FORMAT_XZ)', + 'FORMAT_ALONE': 'space.wrap(interp_lzma.FORMAT_ALONE)', + 'FORMAT_RAW': 'space.wrap(interp_lzma.FORMAT_RAW)', + } + + appleveldefs = { + } diff --git a/pypy/module/_lzma/interp_lzma.py b/pypy/module/_lzma/interp_lzma.py new file mode 100644 --- /dev/null +++ b/pypy/module/_lzma/interp_lzma.py @@ -0,0 +1,360 @@ +from pypy.interpreter.baseobjspace import W_Root +from pypy.interpreter.typedef import ( + TypeDef, interp_attrproperty_bytes, interp_attrproperty) +from pypy.interpreter.error import oefmt +from pypy.interpreter.gateway import interp2app, unwrap_spec, WrappedDefault +from pypy.module.thread.os_lock import Lock +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import LONGLONG_MASK, r_ulonglong +from rpython.rtyper.tool import rffi_platform as platform +from rpython.rtyper.lltypesystem import rffi +from rpython.rtyper.lltypesystem import lltype +from rpython.translator.tool.cbuild import ExternalCompilationInfo + + +FORMAT_AUTO, FORMAT_XZ, FORMAT_ALONE, FORMAT_RAW = range(4) +R_LONGLONG_MASK = r_ulonglong(LONGLONG_MASK) + + +eci = ExternalCompilationInfo( + includes = ['lzma.h'], + libraries = ['lzma'], + ) +eci = platform.configure_external_library( + 'lzma', eci, + [dict(prefix='lzma-')]) +if not eci: + raise ImportError("Could not find lzma library") + + +class CConfig: + _compilation_info_ = eci + calling_conv = 'c' + + BUFSIZ = platform.ConstantInteger("BUFSIZ") + + lzma_stream = platform.Struct( + 'lzma_stream', + [('next_in', rffi.CCHARP), + ('avail_in', rffi.UINT), + ('total_in', rffi.UINT), + ('next_out', rffi.CCHARP), + ('avail_out', rffi.UINT), + ('total_out', rffi.UINT), + ]) + + lzma_options_lzma = platform.Struct( + 'lzma_options_lzma', + []) + +constant_names = ''' + LZMA_RUN LZMA_FINISH + LZMA_OK LZMA_GET_CHECK LZMA_NO_CHECK LZMA_STREAM_END + LZMA_PRESET_DEFAULT + LZMA_CHECK_ID_MAX + LZMA_TELL_ANY_CHECK LZMA_TELL_NO_CHECK + '''.split() +for name in constant_names: + setattr(CConfig, name, platform.ConstantInteger(name)) + +class cConfig(object): + pass +for k, v in platform.configure(CConfig).items(): + setattr(cConfig, k, v) + +for name in constant_names: + globals()[name] = getattr(cConfig, name) +lzma_stream = lltype.Ptr(cConfig.lzma_stream) +lzma_options_lzma = lltype.Ptr(cConfig.lzma_options_lzma) +BUFSIZ = cConfig.BUFSIZ +LZMA_CHECK_UNKNOWN = LZMA_CHECK_ID_MAX + 1 + +def external(name, args, result, **kwds): + return rffi.llexternal(name, args, result, compilation_info= + CConfig._compilation_info_, **kwds) + +lzma_ret = rffi.INT +lzma_action = rffi.INT +lzma_bool = rffi.INT + +lzma_lzma_preset = external('lzma_lzma_preset', [lzma_options_lzma, rffi.UINT], lzma_bool) +lzma_alone_encoder = external('lzma_alone_encoder', [lzma_stream, lzma_options_lzma], lzma_ret) +lzma_end = external('lzma_end', [lzma_stream], lltype.Void, releasegil=False) + +lzma_auto_decoder = external('lzma_auto_decoder', [lzma_stream, rffi.LONG, rffi.INT], lzma_ret) +lzma_get_check = external('lzma_get_check', [lzma_stream], rffi.INT) + +lzma_code = external('lzma_code', [lzma_stream, lzma_action], rffi.INT) + + +@specialize.arg(1) +def raise_error(space, fmt, *args): + raise oefmt(space.w_RuntimeError, fmt, *args) + + +def _catch_lzma_error(space, lzret): + if (lzret == LZMA_OK or lzret == LZMA_GET_CHECK or + lzret == LZMA_NO_CHECK or lzret == LZMA_STREAM_END): + return + raise raise_error(space, "Unrecognized error from liblzma: %d", lzret) + + +if BUFSIZ < 8192: + SMALLCHUNK = 8192 +else: + SMALLCHUNK = BUFSIZ +if rffi.sizeof(rffi.INT) > 4: + BIGCHUNK = 512 * 32 +else: + BIGCHUNK = 512 * 1024 + + +def _new_buffer_size(current_size): + # keep doubling until we reach BIGCHUNK; then the buffer size is no + # longer increased + if current_size < BIGCHUNK: + return current_size + current_size + return current_size + + +class OutBuffer(object): + """Handler for the output buffer. A bit custom code trying to + encapsulate the logic of setting up the fields of 'lzs' and + allocating raw memory as needed. + """ + def __init__(self, lzs, initial_size=SMALLCHUNK): + # when the constructor is called, allocate a piece of memory + # of length 'piece_size' and make lzs ready to dump there. + self.temp = [] + self.lzs = lzs + self._allocate_chunk(initial_size) + + def _allocate_chunk(self, size): + self.raw_buf, self.gc_buf = rffi.alloc_buffer(size) + self.current_size = size + self.lzs.c_next_out = self.raw_buf + rffi.setintfield(self.lzs, 'c_avail_out', size) + + def _get_chunk(self, chunksize): + assert 0 <= chunksize <= self.current_size + raw_buf = self.raw_buf + gc_buf = self.gc_buf + s = rffi.str_from_buffer(raw_buf, gc_buf, self.current_size, chunksize) + rffi.keep_buffer_alive_until_here(raw_buf, gc_buf) + self.current_size = 0 + return s + + def prepare_next_chunk(self): + size = self.current_size + self.temp.append(self._get_chunk(size)) + self._allocate_chunk(_new_buffer_size(size)) + + def make_result_string(self): + count_unoccupied = rffi.getintfield(self.lzs, 'c_avail_out') + s = self._get_chunk(self.current_size - count_unoccupied) + if self.temp: + self.temp.append(s) + return ''.join(self.temp) + else: + return s + + def free(self): + if self.current_size > 0: + rffi.keep_buffer_alive_until_here(self.raw_buf, self.gc_buf) + + def __enter__(self): + return self + def __exit__(self, *args): + self.free() + + +class W_LZMACompressor(W_Root): + def __init__(self, space, format): + self.format = format + self.lock = Lock(space) + self.flushed = False + self.lzs = lltype.malloc(lzma_stream.TO, flavor='raw', zero=True) + + def __del__(self): + lzma_end(self.lzs) + lltype.free(self.lzs, flavor='raw') + + def _init_alone(self, space, preset, w_filters): + if space.is_none(w_filters): + with lltype.scoped_alloc(lzma_options_lzma.TO) as options: + if lzma_lzma_preset(options, preset): + raise_error(space, "Invalid compression preset: %d", preset) + lzret = lzma_alone_encoder(self.lzs, options) + else: + raise NotImplementedError + _catch_lzma_error(space, lzret) + + @staticmethod + @unwrap_spec(format=int, + w_check=WrappedDefault(None), _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit