Author: Ronan Lamy <ronan.l...@gmail.com> Branch: exctrans Changeset: r81867:2f4d7c650765 Date: 2016-01-19 19:37 +0000 http://bitbucket.org/pypy/pypy/changeset/2f4d7c650765/
Log: hg merge default diff too long, truncating to 2000 out of 6108 lines diff --git a/Makefile b/Makefile --- a/Makefile +++ b/Makefile @@ -39,5 +39,5 @@ # runs. We cannot get their original value either: # http://lists.gnu.org/archive/html/help-make/2010-08/msg00106.html -cffi_imports: +cffi_imports: pypy-c PYTHONPATH=. ./pypy-c pypy/tool/build_cffi_imports.py diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -685,13 +685,17 @@ # the previous version of this code did. This should work for # CPython too. The point is that on PyPy with cpyext, the # config var 'SO' is just ".so" but we want to return - # ".pypy-VERSION.so" instead. - so_ext = _get_c_extension_suffix() + # ".pypy-VERSION.so" instead. Note a further tweak for cffi's + # embedding mode: if EXT_SUFFIX is also defined, use that + # directly. + so_ext = get_config_var('EXT_SUFFIX') if so_ext is None: - so_ext = get_config_var('SO') # fall-back - # extensions in debug_mode are named 'module_d.pyd' under windows - if os.name == 'nt' and self.debug: - so_ext = '_d.pyd' + so_ext = _get_c_extension_suffix() + if so_ext is None: + so_ext = get_config_var('SO') # fall-back + # extensions in debug_mode are named 'module_d.pyd' under windows + if os.name == 'nt' and self.debug: + so_ext = '_d.pyd' return os.path.join(*ext_path) + so_ext def get_export_symbols (self, ext): diff --git a/lib-python/2.7/sysconfig.py b/lib-python/2.7/sysconfig.py --- a/lib-python/2.7/sysconfig.py +++ b/lib-python/2.7/sysconfig.py @@ -524,6 +524,13 @@ import _osx_support _osx_support.customize_config_vars(_CONFIG_VARS) + # PyPy: + import imp + for suffix, mode, type_ in imp.get_suffixes(): + if type_ == imp.C_EXTENSION: + _CONFIG_VARS['SOABI'] = suffix.split('.')[1] + break + if args: vals = [] for name in args: diff --git a/lib-python/2.7/test/capath/0e4015b9.0 b/lib-python/2.7/test/capath/0e4015b9.0 new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/capath/0e4015b9.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/lib-python/2.7/test/capath/ce7b8643.0 b/lib-python/2.7/test/capath/ce7b8643.0 new file mode 100644 --- /dev/null +++ b/lib-python/2.7/test/capath/ce7b8643.0 @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE----- +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv +bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG +A1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0 +aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ +Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm +Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= +-----END CERTIFICATE----- diff --git a/lib-python/2.7/test/https_svn_python_org_root.pem b/lib-python/2.7/test/https_svn_python_org_root.pem deleted file mode 100644 --- a/lib-python/2.7/test/https_svn_python_org_root.pem +++ /dev/null @@ -1,41 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 -IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB -IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA -Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO -BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi -MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ -ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC -CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ -8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 -zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y -fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 -w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc -G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k -epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q -laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ -QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU -fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 -YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w -ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY -gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe -MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 -IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy -dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw -czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 -dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl -aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC -AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg -b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB -ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc -nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg -18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c -gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl -Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY -sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T -SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF -CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum -GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk -zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW -omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD ------END CERTIFICATE----- diff --git a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem --- a/lib-python/2.7/test/selfsigned_pythontestdotnet.pem +++ b/lib-python/2.7/test/selfsigned_pythontestdotnet.pem @@ -1,5 +1,5 @@ -----BEGIN CERTIFICATE----- -MIIChzCCAfCgAwIBAgIJAKGU95wKR8pSMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV +MIIClTCCAf6gAwIBAgIJAKGU95wKR8pTMA0GCSqGSIb3DQEBBQUAMHAxCzAJBgNV BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u IFNvZnR3YXJlIEZvdW5kYXRpb24xIzAhBgNVBAMMGnNlbGYtc2lnbmVkLnB5dGhv bnRlc3QubmV0MB4XDTE0MTEwMjE4MDkyOVoXDTI0MTAzMDE4MDkyOVowcDELMAkG @@ -8,9 +8,9 @@ aG9udGVzdC5uZXQwgZ8wDQYJKoZIhvcNAQEBBQADgY0AMIGJAoGBANDXQXW9tjyZ Xt0Iv2tLL1+jinr4wGg36ioLDLFkMf+2Y1GL0v0BnKYG4N1OKlAU15LXGeGer8vm Sv/yIvmdrELvhAbbo3w4a9TMYQA4XkIVLdvu3mvNOAet+8PMJxn26dbDhG809ALv -EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjKTAnMCUGA1UdEQQeMByCGnNl -bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MA0GCSqGSIb3DQEBBQUAA4GBAIOXmdtM -eG9qzP9TiXW/Gc/zI4cBfdCpC+Y4gOfC9bQUC7hefix4iO3+iZjgy3X/FaRxUUoV -HKiXcXIaWqTSUWp45cSh0MbwZXudp6JIAptzdAhvvCrPKeC9i9GvxsPD4LtDAL97 -vSaxQBezA7hdxZd90/EeyMgVZgAnTCnvAWX9 +EHY57lQsBS3G59RZyBPVqAqmImWNJnVzAgMBAAGjNzA1MCUGA1UdEQQeMByCGnNl +bGYtc2lnbmVkLnB5dGhvbnRlc3QubmV0MAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcN +AQEFBQADgYEAIuzAhgMouJpNdf3URCHIineyoSt6WK/9+eyUcjlKOrDoXNZaD72h +TXMeKYoWvJyVcSLKL8ckPtDobgP2OTt0UkyAaj0n+ZHaqq1lH2yVfGUA1ILJv515 +C8BqbvVZuqm3i7ygmw3bqE/lYMgOrYtXXnqOrz6nvsE6Yc9V9rFflOM= -----END CERTIFICATE----- diff --git a/lib-python/2.7/test/test_ssl.py b/lib-python/2.7/test/test_ssl.py --- a/lib-python/2.7/test/test_ssl.py +++ b/lib-python/2.7/test/test_ssl.py @@ -57,7 +57,8 @@ SIGNED_CERTFILE2 = data_file("keycert4.pem") SIGNING_CA = data_file("pycacert.pem") -SVN_PYTHON_ORG_ROOT_CERT = data_file("https_svn_python_org_root.pem") +REMOTE_HOST = "self-signed.pythontest.net" +REMOTE_ROOT_CERT = data_file("selfsigned_pythontestdotnet.pem") EMPTYCERT = data_file("nullcert.pem") BADCERT = data_file("badcert.pem") @@ -244,7 +245,7 @@ self.assertEqual(p['subjectAltName'], san) def test_DER_to_PEM(self): - with open(SVN_PYTHON_ORG_ROOT_CERT, 'r') as f: + with open(CAFILE_CACERT, 'r') as f: pem = f.read() d1 = ssl.PEM_cert_to_DER_cert(pem) p2 = ssl.DER_cert_to_PEM_cert(d1) @@ -792,7 +793,7 @@ # Mismatching key and cert ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) with self.assertRaisesRegexp(ssl.SSLError, "key values mismatch"): - ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY) + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) # Password protected key and cert ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) @@ -1013,7 +1014,7 @@ ctx.load_verify_locations(CERTFILE) self.assertEqual(ctx.cert_store_stats(), {'x509_ca': 0, 'crl': 0, 'x509': 1}) - ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + ctx.load_verify_locations(CAFILE_CACERT) self.assertEqual(ctx.cert_store_stats(), {'x509_ca': 1, 'crl': 0, 'x509': 2}) @@ -1023,8 +1024,8 @@ # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE ctx.load_verify_locations(CERTFILE) self.assertEqual(ctx.get_ca_certs(), []) - # but SVN_PYTHON_ORG_ROOT_CERT is a CA cert - ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) self.assertEqual(ctx.get_ca_certs(), [{'issuer': ((('organizationName', 'Root CA'),), (('organizationalUnitName', 'http://www.cacert.org'),), @@ -1040,7 +1041,7 @@ (('emailAddress', 'supp...@cacert.org'),)), 'version': 3}]) - with open(SVN_PYTHON_ORG_ROOT_CERT) as f: + with open(CAFILE_CACERT) as f: pem = f.read() der = ssl.PEM_cert_to_DER_cert(pem) self.assertEqual(ctx.get_ca_certs(True), [der]) @@ -1215,11 +1216,11 @@ class NetworkedTests(unittest.TestCase): def test_connect(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_NONE) try: - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) self.assertEqual({}, s.getpeercert()) finally: s.close() @@ -1228,27 +1229,27 @@ s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED) self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", - s.connect, ("svn.python.org", 443)) + s.connect, (REMOTE_HOST, 443)) s.close() # this should succeed because we specify the root cert s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, - ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + ca_certs=REMOTE_ROOT_CERT) try: - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) self.assertTrue(s.getpeercert()) finally: s.close() def test_connect_ex(self): # Issue #11326: check connect_ex() implementation - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, - ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + ca_certs=REMOTE_ROOT_CERT) try: - self.assertEqual(0, s.connect_ex(("svn.python.org", 443))) + self.assertEqual(0, s.connect_ex((REMOTE_HOST, 443))) self.assertTrue(s.getpeercert()) finally: s.close() @@ -1256,14 +1257,14 @@ def test_non_blocking_connect_ex(self): # Issue #11326: non-blocking connect_ex() should allow handshake # to proceed after the socket gets ready. - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, - ca_certs=SVN_PYTHON_ORG_ROOT_CERT, + ca_certs=REMOTE_ROOT_CERT, do_handshake_on_connect=False) try: s.setblocking(False) - rc = s.connect_ex(('svn.python.org', 443)) + rc = s.connect_ex((REMOTE_HOST, 443)) # EWOULDBLOCK under Windows, EINPROGRESS elsewhere self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) # Wait for connect to finish @@ -1285,58 +1286,62 @@ def test_timeout_connect_ex(self): # Issue #12065: on a timeout, connect_ex() should return the original # errno (mimicking the behaviour of non-SSL sockets). - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, - ca_certs=SVN_PYTHON_ORG_ROOT_CERT, + ca_certs=REMOTE_ROOT_CERT, do_handshake_on_connect=False) try: s.settimeout(0.0000001) - rc = s.connect_ex(('svn.python.org', 443)) + rc = s.connect_ex((REMOTE_HOST, 443)) if rc == 0: - self.skipTest("svn.python.org responded too quickly") + self.skipTest("REMOTE_HOST responded too quickly") self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) finally: s.close() def test_connect_ex_error(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_REQUIRED, - ca_certs=SVN_PYTHON_ORG_ROOT_CERT) + ca_certs=REMOTE_ROOT_CERT) try: - rc = s.connect_ex(("svn.python.org", 444)) + rc = s.connect_ex((REMOTE_HOST, 444)) # Issue #19919: Windows machines or VMs hosted on Windows # machines sometimes return EWOULDBLOCK. - self.assertIn(rc, (errno.ECONNREFUSED, errno.EWOULDBLOCK)) + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) finally: s.close() def test_connect_with_context(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): # Same as test_connect, but with a separately created context ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) try: self.assertEqual({}, s.getpeercert()) finally: s.close() # Same with a server hostname s = ctx.wrap_socket(socket.socket(socket.AF_INET), - server_hostname="svn.python.org") - s.connect(("svn.python.org", 443)) + server_hostname=REMOTE_HOST) + s.connect((REMOTE_HOST, 443)) s.close() # This should fail because we have no verification certs ctx.verify_mode = ssl.CERT_REQUIRED s = ctx.wrap_socket(socket.socket(socket.AF_INET)) self.assertRaisesRegexp(ssl.SSLError, "certificate verify failed", - s.connect, ("svn.python.org", 443)) + s.connect, (REMOTE_HOST, 443)) s.close() # This should succeed because we specify the root cert - ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + ctx.load_verify_locations(REMOTE_ROOT_CERT) s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) try: cert = s.getpeercert() self.assertTrue(cert) @@ -1349,12 +1354,12 @@ # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must # contain both versions of each certificate (same content, different # filename) for this test to be portable across OpenSSL releases. - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.verify_mode = ssl.CERT_REQUIRED ctx.load_verify_locations(capath=CAPATH) s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) try: cert = s.getpeercert() self.assertTrue(cert) @@ -1365,7 +1370,7 @@ ctx.verify_mode = ssl.CERT_REQUIRED ctx.load_verify_locations(capath=BYTES_CAPATH) s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) try: cert = s.getpeercert() self.assertTrue(cert) @@ -1373,15 +1378,15 @@ s.close() def test_connect_cadata(self): - with open(CAFILE_CACERT) as f: + with open(REMOTE_ROOT_CERT) as f: pem = f.read().decode('ascii') der = ssl.PEM_cert_to_DER_cert(pem) - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.verify_mode = ssl.CERT_REQUIRED ctx.load_verify_locations(cadata=pem) with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) cert = s.getpeercert() self.assertTrue(cert) @@ -1390,7 +1395,7 @@ ctx.verify_mode = ssl.CERT_REQUIRED ctx.load_verify_locations(cadata=der) with closing(ctx.wrap_socket(socket.socket(socket.AF_INET))) as s: - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) cert = s.getpeercert() self.assertTrue(cert) @@ -1399,9 +1404,9 @@ # Issue #5238: creating a file-like object with makefile() shouldn't # delay closing the underlying "real socket" (here tested with its # file descriptor, hence skipping the test under Windows). - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): ss = ssl.wrap_socket(socket.socket(socket.AF_INET)) - ss.connect(("svn.python.org", 443)) + ss.connect((REMOTE_HOST, 443)) fd = ss.fileno() f = ss.makefile() f.close() @@ -1415,9 +1420,9 @@ self.assertEqual(e.exception.errno, errno.EBADF) def test_non_blocking_handshake(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): s = socket.socket(socket.AF_INET) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) s.setblocking(False) s = ssl.wrap_socket(s, cert_reqs=ssl.CERT_NONE, @@ -1460,12 +1465,12 @@ if support.verbose: sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT) + _test_get_server_certificate(REMOTE_HOST, 443, REMOTE_ROOT_CERT) if support.IPV6_ENABLED: _test_get_server_certificate('ipv6.google.com', 443) def test_ciphers(self): - remote = ("svn.python.org", 443) + remote = (REMOTE_HOST, 443) with support.transient_internet(remote[0]): with closing(ssl.wrap_socket(socket.socket(socket.AF_INET), cert_reqs=ssl.CERT_NONE, ciphers="ALL")) as s: @@ -1510,13 +1515,13 @@ def test_get_ca_certs_capath(self): # capath certs are loaded on request - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.verify_mode = ssl.CERT_REQUIRED ctx.load_verify_locations(capath=CAPATH) self.assertEqual(ctx.get_ca_certs(), []) s = ctx.wrap_socket(socket.socket(socket.AF_INET)) - s.connect(("svn.python.org", 443)) + s.connect((REMOTE_HOST, 443)) try: cert = s.getpeercert() self.assertTrue(cert) @@ -1527,12 +1532,12 @@ @needs_sni def test_context_setget(self): # Check that the context of a connected socket can be replaced. - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLSv1) ctx2 = ssl.SSLContext(ssl.PROTOCOL_SSLv23) s = socket.socket(socket.AF_INET) with closing(ctx1.wrap_socket(s)) as ss: - ss.connect(("svn.python.org", 443)) + ss.connect((REMOTE_HOST, 443)) self.assertIs(ss.context, ctx1) self.assertIs(ss._sslobj.context, ctx1) ss.context = ctx2 @@ -3026,7 +3031,7 @@ pass for filename in [ - CERTFILE, SVN_PYTHON_ORG_ROOT_CERT, BYTES_CERTFILE, + CERTFILE, REMOTE_ROOT_CERT, BYTES_CERTFILE, ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, BADCERT, BADKEY, EMPTYCERT]: diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py --- a/lib_pypy/cPickle.py +++ b/lib_pypy/cPickle.py @@ -167,7 +167,11 @@ try: key = ord(self.read(1)) while key != STOP: - self.dispatch[key](self) + try: + meth = self.dispatch[key] + except KeyError: + raise UnpicklingError("invalid load key, %r." % chr(key)) + meth(self) key = ord(self.read(1)) except TypeError: if self.read(1) == '': diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.4.2 +Version: 1.5.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.4.2" -__version_info__ = (1, 4, 2) +__version__ = "1.5.0" +__version_info__ = (1, 5, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -146,8 +146,9 @@ ((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23]) #define _cffi_convert_array_from_object \ ((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24]) +#define _CFFI_CPIDX 25 #define _cffi_call_python \ - ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[25]) + ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX]) #define _CFFI_NUM_EXPORTS 26 typedef struct _ctypedescr CTypeDescrObject; @@ -206,7 +207,8 @@ /********** end CPython-specific section **********/ #else _CFFI_UNUSED_FN -static void (*_cffi_call_python)(struct _cffi_externpy_s *, char *); +static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *); +# define _cffi_call_python _cffi_call_python_org #endif diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -74,6 +74,7 @@ self._windows_unicode = None self._init_once_cache = {} self._cdef_version = None + self._embedding = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -101,13 +102,21 @@ If 'packed' is specified as True, all structs declared inside this cdef are packed, i.e. laid out without any field alignment at all. """ + self._cdef(csource, override=override, packed=packed) + + def embedding_api(self, csource, packed=False): + self._cdef(csource, packed=packed, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): if not isinstance(csource, str): # unicode, on Python 2 if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') with self._lock: self._cdef_version = object() - self._parser.parse(csource, override=override, packed=packed) + self._parser.parse(csource, override=override, **options) self._cdefsources.append(csource) if override: for cache in self._function_caches: @@ -533,6 +542,31 @@ ('_UNICODE', '1')] kwds['define_macros'] = defmacros + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + if '__pypy__' in sys.builtin_module_names: + if hasattr(sys, 'prefix'): + import os + libdir = os.path.join(sys.prefix, 'bin') + dirs = kwds.setdefault('library_dirs', []) + if libdir not in dirs: + dirs.append(libdir) + pythonlib = "pypy-c" + else: + if sys.platform == "win32": + template = "python%d%d" + if sys.flags.debug: + template = template + '_d' + else: + template = "python%d.%d" + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + libraries = kwds.setdefault('libraries', []) + if pythonlib not in libraries: + libraries.append(pythonlib) + def set_source(self, module_name, source, source_extension='.c', **kwds): if hasattr(self, '_assigned_source'): raise ValueError("set_source() cannot be called several times " @@ -592,14 +626,23 @@ recompile(self, module_name, source, c_file=filename, call_c_compiler=False, **kwds) - def compile(self, tmpdir='.', verbose=0): + def compile(self, tmpdir='.', verbose=0, target=None): + """The 'target' argument gives the final file name of the + compiled DLL. Use '*' to force distutils' choice, suitable for + regular CPython C API modules. Use a file name ending in '.*' + to ask for the system's default extension for dynamic libraries + (.so/.dll). + + The default is '*' when building a non-embedded C API extension, + and (module_name + '.*') when building an embedded library. + """ from .recompiler import recompile # if not hasattr(self, '_assigned_source'): raise ValueError("set_source() must be called before compile()") module_name, source, source_extension, kwds = self._assigned_source return recompile(self, module_name, source, tmpdir=tmpdir, - source_extension=source_extension, + target=target, source_extension=source_extension, compiler_verbose=verbose, **kwds) def init_once(self, func, tag): @@ -626,6 +669,32 @@ self._init_once_cache[tag] = (True, result) return result + def embedding_init_code(self, pysource): + if self._embedding: + raise ValueError("embedding_init_code() can only be called once") + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -220,8 +220,7 @@ self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() - self._override = False - self._packed = False + self._options = None self._int_constants = {} self._recomplete = [] self._uses_new_feature = None @@ -281,16 +280,15 @@ msg = 'parse error\n%s' % (msg,) raise api.CDefError(msg) - def parse(self, csource, override=False, packed=False): - prev_override = self._override - prev_packed = self._packed + def parse(self, csource, override=False, packed=False, dllexport=False): + prev_options = self._options try: - self._override = override - self._packed = packed + self._options = {'override': override, + 'packed': packed, + 'dllexport': dllexport} self._internal_parse(csource) finally: - self._override = prev_override - self._packed = prev_packed + self._options = prev_options def _internal_parse(self, csource): ast, macros, csource = self._parse(csource) @@ -376,10 +374,13 @@ def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) - if self._inside_extern_python: - self._declare('extern_python ' + decl.name, tp) + if self._options['dllexport']: + tag = 'dllexport_python ' + elif self._inside_extern_python: + tag = 'extern_python ' else: - self._declare('function ' + decl.name, tp) + tag = 'function ' + self._declare(tag + decl.name, tp) def _parse_decl(self, decl): node = decl.type @@ -449,7 +450,7 @@ prevobj, prevquals = self._declarations[name] if prevobj is obj and prevquals == quals: return - if not self._override: + if not self._options['override']: raise api.FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) @@ -728,7 +729,7 @@ if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" % (tp,)) - tp.packed = self._packed + tp.packed = self._options['packed'] if tp.completed: # must be re-completed: it is not opaque any more tp.completed = 0 self._recomplete.append(tp) diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -21,12 +21,14 @@ allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0): +def compile(tmpdir, ext, compiler_verbose=0, target_extension=None, + embedding=False): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose) + outputfilename = _build(tmpdir, ext, compiler_verbose, + target_extension, embedding) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -36,7 +38,32 @@ os.environ[key] = value return outputfilename -def _build(tmpdir, ext, compiler_verbose=0): +def _save_val(name): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + return config_vars.get(name, Ellipsis) + +def _restore_val(name, value): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + config_vars[name] = value + if value is Ellipsis: + del config_vars[name] + +def _win32_hack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'): + MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \ + MSVCCompiler._remove_visual_c_ref + MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file: manifest_file + +def _win32_unhack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + MSVCCompiler._remove_visual_c_ref = \ + MSVCCompiler._remove_visual_c_ref_CFFI_BAK + +def _build(tmpdir, ext, compiler_verbose=0, target_extension=None, + embedding=False): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -49,18 +76,29 @@ options['build_temp'] = ('ffiplatform', tmpdir) # try: + if sys.platform == 'win32' and embedding: + _win32_hack_for_embedding() old_level = distutils.log.set_threshold(0) or 0 + old_SO = _save_val('SO') + old_EXT_SUFFIX = _save_val('EXT_SUFFIX') try: + if target_extension is not None: + _restore_val('SO', target_extension) + _restore_val('EXT_SUFFIX', target_extension) distutils.log.set_verbosity(compiler_verbose) dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() finally: distutils.log.set_threshold(old_level) + _restore_val('SO', old_SO) + _restore_val('EXT_SUFFIX', old_EXT_SUFFIX) + if sys.platform == 'win32' and embedding: + _win32_unhack_for_embedding() except (distutils.errors.CompileError, distutils.errors.LinkError) as e: raise VerificationError('%s: %s' % (e.__class__.__name__, e)) # - cmd_obj = dist.get_command_obj('build_ext') - [soname] = cmd_obj.get_outputs() return soname try: diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -3,6 +3,7 @@ from .cffi_opcode import * VERSION = "0x2601" +VERSION_EMBEDDED = "0x2701" class GlobalExpr: @@ -281,6 +282,29 @@ lines[i:i+1] = self._rel_readlines('parse_c_type.h') prnt(''.join(lines)) # + # if we have ffi._embedding != None, we give it here as a macro + # and include an extra file + base_module_name = self.module_name.split('.')[-1] + if self.ffi._embedding is not None: + prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) + prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % + (self._string_literal(self.ffi._embedding),)) + prnt('#ifdef PYPY_VERSION') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( + base_module_name,)) + prnt('#elif PY_MAJOR_VERSION >= 3') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % ( + base_module_name,)) + prnt('#else') + prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % ( + base_module_name,)) + prnt('#endif') + lines = self._rel_readlines('_embedding.h') + prnt(''.join(lines)) + version = VERSION_EMBEDDED + else: + version = VERSION + # # then paste the C source given by the user, verbatim. prnt('/************************************************************/') prnt() @@ -365,17 +389,16 @@ prnt() # # the init function - base_module_name = self.module_name.split('.')[-1] prnt('#ifdef PYPY_VERSION') prnt('PyMODINIT_FUNC') prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,)) prnt('{') if self._num_externpy: prnt(' if (((intptr_t)p[0]) >= 0x0A03) {') - prnt(' _cffi_call_python = ' + prnt(' _cffi_call_python_org = ' '(void(*)(struct _cffi_externpy_s *, char *))p[1];') prnt(' }') - prnt(' p[0] = (const void *)%s;' % VERSION) + prnt(' p[0] = (const void *)%s;' % version) prnt(' p[1] = &_cffi_type_context;') prnt('}') # on Windows, distutils insists on putting init_cffi_xyz in @@ -394,14 +417,14 @@ prnt('PyInit_%s(void)' % (base_module_name,)) prnt('{') prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % ( - self.module_name, VERSION)) + self.module_name, version)) prnt('}') prnt('#else') prnt('PyMODINIT_FUNC') prnt('init%s(void)' % (base_module_name,)) prnt('{') prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % ( - self.module_name, VERSION)) + self.module_name, version)) prnt('}') prnt('#endif') @@ -1123,7 +1146,10 @@ assert isinstance(tp, model.FunctionPtrType) self._do_collect_type(tp) - def _generate_cpy_extern_python_decl(self, tp, name): + def _generate_cpy_dllexport_python_collecttype(self, tp, name): + self._generate_cpy_extern_python_collecttype(tp, name) + + def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False): prnt = self._prnt if isinstance(tp.result, model.VoidType): size_of_result = '0' @@ -1156,7 +1182,11 @@ size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % ( tp.result.get_c_name(''), size_of_a, tp.result.get_c_name(''), size_of_a) - prnt('static %s' % tp.result.get_c_name(name_and_arguments)) + if dllexport: + tag = 'CFFI_DLLEXPORT' + else: + tag = 'static' + prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments))) prnt('{') prnt(' char a[%s];' % size_of_a) prnt(' char *p = a;') @@ -1174,6 +1204,9 @@ prnt() self._num_externpy += 1 + def _generate_cpy_dllexport_python_decl(self, tp, name): + self._generate_cpy_extern_python_decl(tp, name, dllexport=True) + def _generate_cpy_extern_python_ctx(self, tp, name): if self.target_is_python: raise ffiplatform.VerificationError( @@ -1185,6 +1218,21 @@ self._lsts["global"].append( GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name)) + def _generate_cpy_dllexport_python_ctx(self, tp, name): + self._generate_cpy_extern_python_ctx(tp, name) + + def _string_literal(self, s): + def _char_repr(c): + # escape with a '\' the characters '\', '"' or (for trigraphs) '?' + if c in '\\"?': return '\\' + c + if ' ' <= c < '\x7F': return c + if c == '\n': return '\\n' + return '\\%03o' % ord(c) + lines = [] + for line in s.splitlines(True): + lines.append('"%s"' % ''.join([_char_repr(c) for c in line])) + return ' \\\n'.join(lines) + # ---------- # emitting the opcodes for individual types @@ -1311,12 +1359,15 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, **kwds): + compiler_verbose=1, target=None, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: ffi._apply_windows_unicode(kwds) if preamble is not None: + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) if c_file is None: c_file, parts = _modname_to_file(tmpdir, module_name, source_extension) @@ -1325,13 +1376,40 @@ ext_c_file = os.path.join(*parts) else: ext_c_file = c_file - ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + # + if target is None: + if embedding: + target = '%s.*' % module_name + else: + target = '*' + if target == '*': + target_module_name = module_name + target_extension = None # use default + else: + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + else: + target += '.so' + # split along the first '.' (not the last one, otherwise the + # preceeding dots are interpreted as splitting package names) + index = target.find('.') + if index < 0: + raise ValueError("target argument %r should be a file name " + "containing a '.'" % (target,)) + target_module_name = target[:index] + target_extension = target[index:] + # + ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds) updated = make_c_source(ffi, module_name, preamble, c_file) if call_c_compiler: cwd = os.getcwd() try: os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose) + outputfilename = ffiplatform.compile('.', ext, compiler_verbose, + target_extension, + embedding=embedding) finally: os.chdir(cwd) return outputfilename diff --git a/pypy/doc/embedding.rst b/pypy/doc/embedding.rst --- a/pypy/doc/embedding.rst +++ b/pypy/doc/embedding.rst @@ -10,6 +10,15 @@ with a ``libpypy-c.so`` or ``pypy-c.dll`` file. This is the default in recent versions of PyPy. +.. note:: + + The interface described in this page is kept for backward compatibility. + From PyPy 4.1, it is recommended to use instead CFFI's `native embedding + support,`__ which gives a simpler approach that works on CPython as well + as PyPy. + +.. __: http://cffi.readthedocs.org/en/latest/embedding.html + The resulting shared library exports very few functions, however they are enough to accomplish everything you need, provided you follow a few principles. The API is: diff --git a/pypy/doc/getting-started-dev.rst b/pypy/doc/getting-started-dev.rst --- a/pypy/doc/getting-started-dev.rst +++ b/pypy/doc/getting-started-dev.rst @@ -19,7 +19,9 @@ * Clone this new repo (i.e. the fork) to your local machine with the command ``hg clone ssh://h...@bitbucket.org/yourname/pypy``. It is a very slow - operation but only ever needs to be done once. If you already cloned + operation but only ever needs to be done once. See also + http://pypy.org/download.html#building-from-source . + If you already cloned ``https://bitbucket.org/pypy/pypy`` before, even if some time ago, then you can reuse the same clone by editing the file ``.hg/hgrc`` in your clone to contain the line ``default = diff --git a/pypy/doc/how-to-contribute.rst b/pypy/doc/how-to-contribute.rst --- a/pypy/doc/how-to-contribute.rst +++ b/pypy/doc/how-to-contribute.rst @@ -67,8 +67,8 @@ **module** directory contains extension modules written in RPython * **rpython compiler** that resides in ``rpython/annotator`` and - ``rpython/rtyper`` directories. Consult :doc:`introduction to RPython <getting-started-dev>` for - further reading + ``rpython/rtyper`` directories. Consult `Getting Started with RPython`_ + for further reading * **JIT generator** lives in ``rpython/jit`` directory. optimizations live in ``rpython/jit/metainterp/optimizeopt``, the main JIT in @@ -80,3 +80,14 @@ The rest of directories serve specific niche goal and are unlikely a good entry point. + + +More documentation +------------------ + +* `Getting Started Developing With PyPy`_ + +* `Getting Started with RPython`_ + +.. _`Getting Started Developing With PyPy`: getting-started-dev.html +.. _`Getting started with RPython`: http://rpython.readthedocs.org/en/latest/getting-started.html diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -110,3 +110,16 @@ short-running Python callbacks. (CFFI on CPython has a hack to achieve the same result.) This can also be seen as a bug fix: previously, thread-local objects would be reset between two such calls. + +.. branch: globals-quasiimmut + +Optimize global lookups. + +.. branch: cffi-static-callback-embedding + +Updated to CFFI 1.5, which supports a new way to do embedding. +Deprecates http://pypy.readthedocs.org/en/latest/embedding.html. + +.. branch: fix-cpython-ssl-tests-2.7 + +Fix SSL tests by importing cpython's patch diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -84,13 +84,6 @@ from rpython.rlib.entrypoint import entrypoint_highlevel from rpython.rtyper.lltypesystem import rffi, lltype - w_pathsetter = space.appexec([], """(): - def f(path): - import sys - sys.path[:] = path - return f - """) - @entrypoint_highlevel('main', [rffi.CCHARP, rffi.INT], c_name='pypy_setup_home') def pypy_setup_home(ll_home, verbose): @@ -109,7 +102,10 @@ " not found in '%s' or in any parent directory" % home1) return rffi.cast(rffi.INT, 1) space.startup() - space.call_function(w_pathsetter, w_path) + space.appexec([w_path], """(path): + import sys + sys.path[:] = path + """) # import site try: space.setattr(space.getbuiltinmodule('sys'), @@ -149,6 +145,9 @@ return os_thread.setup_threads(space) os_thread.bootstrapper.acquire(space, None, None) + # XXX this doesn't really work. Don't use os.fork(), and + # if your embedder program uses fork(), don't use any PyPy + # code in the fork rthread.gc_thread_start() os_thread.bootstrapper.nbthreads += 1 os_thread.bootstrapper.release() diff --git a/pypy/interpreter/eval.py b/pypy/interpreter/eval.py --- a/pypy/interpreter/eval.py +++ b/pypy/interpreter/eval.py @@ -9,8 +9,8 @@ class Code(W_Root): """A code is a compiled version of some source code. Abstract base class.""" - _immutable_ = True hidden_applevel = False + _immutable_fields_ = ['co_name', 'fast_natural_arity', 'hidden_applevel'] # n >= 0 : arity # FLATPYCALL = 0x100 diff --git a/pypy/interpreter/pycode.py b/pypy/interpreter/pycode.py --- a/pypy/interpreter/pycode.py +++ b/pypy/interpreter/pycode.py @@ -56,11 +56,13 @@ class PyCode(eval.Code): "CPython-style code objects." - _immutable_ = True - _immutable_fields_ = ["co_consts_w[*]", "co_names_w[*]", "co_varnames[*]", - "co_freevars[*]", "co_cellvars[*]", - "_args_as_cellvars[*]"] - + _immutable_fields_ = ["_signature", "co_argcount", "co_cellvars[*]", + "co_code", "co_consts_w[*]", "co_filename", + "co_firstlineno", "co_flags", "co_freevars[*]", + "co_lnotab", "co_names_w[*]", "co_nlocals", + "co_stacksize", "co_varnames[*]", + "_args_as_cellvars[*]", "w_globals?"] + def __init__(self, space, argcount, nlocals, stacksize, flags, code, consts, names, varnames, filename, name, firstlineno, lnotab, freevars, cellvars, @@ -84,6 +86,10 @@ self.co_name = name self.co_firstlineno = firstlineno self.co_lnotab = lnotab + # store the first globals object that the code object is run in in + # here. if a frame is run in that globals object, it does not need to + # store it at all + self.w_globals = None self.hidden_applevel = hidden_applevel self.magic = magic self._signature = cpython_code_signature(self) @@ -91,6 +97,14 @@ self._init_ready() self.new_code_hook() + def frame_stores_global(self, w_globals): + if self.w_globals is None: + self.w_globals = w_globals + return False + if self.w_globals is w_globals: + return False + return True + def new_code_hook(self): code_hook = self.space.fromcache(CodeHookCache)._code_hook if code_hook is not None: diff --git a/pypy/interpreter/pyframe.py b/pypy/interpreter/pyframe.py --- a/pypy/interpreter/pyframe.py +++ b/pypy/interpreter/pyframe.py @@ -36,6 +36,7 @@ def __init__(self, pycode): self.f_lineno = pycode.co_firstlineno + self.w_globals = pycode.w_globals class PyFrame(W_Root): """Represents a frame for a regular Python function @@ -67,7 +68,6 @@ escaped = False # see mark_as_escaped() debugdata = None - w_globals = None pycode = None # code object executed by that frame locals_cells_stack_w = None # the list of all locals, cells and the valuestack valuestackdepth = 0 # number of items on valuestack @@ -90,8 +90,9 @@ self = hint(self, access_directly=True, fresh_virtualizable=True) assert isinstance(code, pycode.PyCode) self.space = space - self.w_globals = w_globals self.pycode = code + if code.frame_stores_global(w_globals): + self.getorcreatedebug().w_globals = w_globals ncellvars = len(code.co_cellvars) nfreevars = len(code.co_freevars) size = code.co_nlocals + ncellvars + nfreevars + code.co_stacksize @@ -116,6 +117,12 @@ self.debugdata = FrameDebugData(self.pycode) return self.debugdata + def get_w_globals(self): + debugdata = self.getdebug() + if debugdata is not None: + return debugdata.w_globals + return jit.promote(self.pycode).w_globals + def get_w_f_trace(self): d = self.getdebug() if d is None: @@ -201,8 +208,9 @@ if flags & pycode.CO_NEWLOCALS: self.getorcreatedebug().w_locals = self.space.newdict(module=True) else: - assert self.w_globals is not None - self.getorcreatedebug().w_locals = self.w_globals + w_globals = self.get_w_globals() + assert w_globals is not None + self.getorcreatedebug().w_locals = w_globals ncellvars = len(code.co_cellvars) nfreevars = len(code.co_freevars) @@ -449,7 +457,7 @@ w_blockstack, w_exc_value, # last_exception w_tb, # - self.w_globals, + self.get_w_globals(), w(self.last_instr), w(self.frame_finished_execution), w(f_lineno), @@ -658,6 +666,11 @@ def fget_getdictscope(self, space): return self.getdictscope() + def fget_w_globals(self, space): + # bit silly, but GetSetProperty passes a space + return self.get_w_globals() + + ### line numbers ### def fget_f_lineno(self, space): diff --git a/pypy/interpreter/pyopcode.py b/pypy/interpreter/pyopcode.py --- a/pypy/interpreter/pyopcode.py +++ b/pypy/interpreter/pyopcode.py @@ -837,7 +837,7 @@ w_bases = self.popvalue() w_name = self.popvalue() w_metaclass = find_metaclass(self.space, w_bases, - w_methodsdict, self.w_globals, + w_methodsdict, self.get_w_globals(), self.space.wrap(self.get_builtin())) w_newclass = self.space.call_function(w_metaclass, w_name, w_bases, w_methodsdict) @@ -881,14 +881,14 @@ def STORE_GLOBAL(self, nameindex, next_instr): varname = self.getname_u(nameindex) w_newvalue = self.popvalue() - self.space.setitem_str(self.w_globals, varname, w_newvalue) + self.space.setitem_str(self.get_w_globals(), varname, w_newvalue) def DELETE_GLOBAL(self, nameindex, next_instr): w_varname = self.getname_w(nameindex) - self.space.delitem(self.w_globals, w_varname) + self.space.delitem(self.get_w_globals(), w_varname) def LOAD_NAME(self, nameindex, next_instr): - if self.getorcreatedebug().w_locals is not self.w_globals: + if self.getorcreatedebug().w_locals is not self.get_w_globals(): varname = self.getname_u(nameindex) w_value = self.space.finditem_str(self.getorcreatedebug().w_locals, varname) @@ -898,7 +898,7 @@ self.LOAD_GLOBAL(nameindex, next_instr) # fall-back def _load_global(self, varname): - w_value = self.space.finditem_str(self.w_globals, varname) + w_value = self.space.finditem_str(self.get_w_globals(), varname) if w_value is None: # not in the globals, now look in the built-ins w_value = self.get_builtin().getdictvalue(self.space, varname) @@ -1029,7 +1029,7 @@ if w_locals is None: # CPython does this w_locals = space.w_None w_modulename = space.wrap(modulename) - w_globals = self.w_globals + w_globals = self.get_w_globals() if w_flag is None: w_obj = space.call_function(w_import, w_modulename, w_globals, w_locals, w_fromlist) @@ -1237,7 +1237,7 @@ w_codeobj = self.popvalue() codeobj = self.space.interp_w(PyCode, w_codeobj) defaultarguments = self.popvalues(numdefaults) - fn = function.Function(self.space, codeobj, self.w_globals, + fn = function.Function(self.space, codeobj, self.get_w_globals(), defaultarguments) self.pushvalue(self.space.wrap(fn)) @@ -1249,7 +1249,7 @@ freevars = [self.space.interp_w(Cell, cell) for cell in self.space.fixedview(w_freevarstuple)] defaultarguments = self.popvalues(numdefaults) - fn = function.Function(self.space, codeobj, self.w_globals, + fn = function.Function(self.space, codeobj, self.get_w_globals(), defaultarguments, freevars) self.pushvalue(self.space.wrap(fn)) diff --git a/pypy/interpreter/test/test_pyframe.py b/pypy/interpreter/test/test_pyframe.py --- a/pypy/interpreter/test/test_pyframe.py +++ b/pypy/interpreter/test/test_pyframe.py @@ -34,6 +34,7 @@ import sys f = sys._getframe() assert f.f_globals is globals() + raises(TypeError, "f.f_globals = globals()") def test_f_builtins(self): import sys, __builtin__ diff --git a/pypy/interpreter/typedef.py b/pypy/interpreter/typedef.py --- a/pypy/interpreter/typedef.py +++ b/pypy/interpreter/typedef.py @@ -772,7 +772,7 @@ f_restricted = GetSetProperty(PyFrame.fget_f_restricted), f_code = GetSetProperty(PyFrame.fget_code), f_locals = GetSetProperty(PyFrame.fget_getdictscope), - f_globals = interp_attrproperty_w('w_globals', cls=PyFrame), + f_globals = GetSetProperty(PyFrame.fget_w_globals), ) assert not PyFrame.typedef.acceptable_as_base_class # no __new__ diff --git a/pypy/module/__builtin__/compiling.py b/pypy/module/__builtin__/compiling.py --- a/pypy/module/__builtin__/compiling.py +++ b/pypy/module/__builtin__/compiling.py @@ -93,7 +93,7 @@ if space.is_none(w_locals): w_locals = w_globals else: - w_globals = caller.w_globals + w_globals = caller.get_w_globals() if space.is_none(w_locals): w_locals = caller.getdictscope() elif space.is_none(w_locals): diff --git a/pypy/module/__builtin__/interp_inspect.py b/pypy/module/__builtin__/interp_inspect.py --- a/pypy/module/__builtin__/interp_inspect.py +++ b/pypy/module/__builtin__/interp_inspect.py @@ -2,7 +2,7 @@ def globals(space): "Return the dictionary containing the current scope's global variables." ec = space.getexecutioncontext() - return ec.gettopframe_nohidden().w_globals + return ec.gettopframe_nohidden().get_w_globals() def locals(space): """Return a dictionary containing the current scope's local variables. diff --git a/pypy/module/__builtin__/test/test_classobj.py b/pypy/module/__builtin__/test/test_classobj.py --- a/pypy/module/__builtin__/test/test_classobj.py +++ b/pypy/module/__builtin__/test/test_classobj.py @@ -1084,7 +1084,7 @@ def is_strdict(space, w_class): from pypy.objspace.std.dictmultiobject import BytesDictStrategy w_d = w_class.getdict(space) - return space.wrap(isinstance(w_d.strategy, BytesDictStrategy)) + return space.wrap(isinstance(w_d.get_strategy(), BytesDictStrategy)) cls.w_is_strdict = cls.space.wrap(gateway.interp2app(is_strdict)) diff --git a/pypy/module/__pypy__/interp_magic.py b/pypy/module/__pypy__/interp_magic.py --- a/pypy/module/__pypy__/interp_magic.py +++ b/pypy/module/__pypy__/interp_magic.py @@ -93,7 +93,7 @@ Return the underlying strategy currently used by a dict, list or set object """ if isinstance(w_obj, W_DictMultiObject): - name = w_obj.strategy.__class__.__name__ + name = w_obj.get_strategy().__class__.__name__ elif isinstance(w_obj, W_ListObject): name = w_obj.strategy.__class__.__name__ elif isinstance(w_obj, W_BaseSetObject): diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -1,8 +1,9 @@ import sys from pypy.interpreter.mixedmodule import MixedModule -from rpython.rlib import rdynload, clibffi +from rpython.rlib import rdynload, clibffi, entrypoint +from rpython.rtyper.lltypesystem import rffi -VERSION = "1.4.2" +VERSION = "1.5.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: @@ -65,6 +66,10 @@ if has_stdcall: interpleveldefs['FFI_STDCALL'] = 'space.wrap(%d)' % FFI_STDCALL + def startup(self, space): + from pypy.module._cffi_backend import embedding + embedding.glob.space = space + def get_dict_rtld_constants(): found = {} @@ -78,3 +83,11 @@ for _name, _value in get_dict_rtld_constants().items(): Module.interpleveldefs[_name] = 'space.wrap(%d)' % _value + + +# write this entrypoint() here, to make sure it is registered early enough +@entrypoint.entrypoint_highlevel('main', [rffi.INT, rffi.VOIDP], + c_name='pypy_init_embedded_cffi_module') +def pypy_init_embedded_cffi_module(version, init_struct): + from pypy.module._cffi_backend import embedding + return embedding.pypy_init_embedded_cffi_module(version, init_struct) diff --git a/pypy/module/_cffi_backend/cffi1_module.py b/pypy/module/_cffi_backend/cffi1_module.py --- a/pypy/module/_cffi_backend/cffi1_module.py +++ b/pypy/module/_cffi_backend/cffi1_module.py @@ -2,24 +2,25 @@ from pypy.interpreter.error import oefmt from pypy.interpreter.module import Module +from pypy.module import _cffi_backend from pypy.module._cffi_backend import parse_c_type from pypy.module._cffi_backend.ffi_obj import W_FFIObject from pypy.module._cffi_backend.lib_obj import W_LibObject VERSION_MIN = 0x2601 -VERSION_MAX = 0x26FF +VERSION_MAX = 0x27FF VERSION_EXPORT = 0x0A03 -initfunctype = lltype.Ptr(lltype.FuncType([rffi.VOIDPP], lltype.Void)) +INITFUNCPTR = lltype.Ptr(lltype.FuncType([rffi.VOIDPP], lltype.Void)) def load_cffi1_module(space, name, path, initptr): # This is called from pypy.module.cpyext.api.load_extension_module() from pypy.module._cffi_backend.call_python import get_ll_cffi_call_python - initfunc = rffi.cast(initfunctype, initptr) + initfunc = rffi.cast(INITFUNCPTR, initptr) with lltype.scoped_alloc(rffi.VOIDPP.TO, 16, zero=True) as p: p[0] = rffi.cast(rffi.VOIDP, VERSION_EXPORT) p[1] = rffi.cast(rffi.VOIDP, get_ll_cffi_call_python()) @@ -27,8 +28,10 @@ version = rffi.cast(lltype.Signed, p[0]) if not (VERSION_MIN <= version <= VERSION_MAX): raise oefmt(space.w_ImportError, - "cffi extension module '%s' has unknown version %s", - name, hex(version)) + "cffi extension module '%s' uses an unknown version tag %s. " + "This module might need a more recent version of PyPy. " + "The current PyPy provides CFFI %s.", + name, hex(version), _cffi_backend.VERSION) src_ctx = rffi.cast(parse_c_type.PCTX, p[1]) ffi = W_FFIObject(space, src_ctx) @@ -38,7 +41,8 @@ w_name = space.wrap(name) module = Module(space, w_name) - module.setdictvalue(space, '__file__', space.wrap(path)) + if path is not None: + module.setdictvalue(space, '__file__', space.wrap(path)) module.setdictvalue(space, 'ffi', space.wrap(ffi)) module.setdictvalue(space, 'lib', space.wrap(lib)) w_modules_dict = space.sys.get('modules') diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py new file mode 100644 --- /dev/null +++ b/pypy/module/_cffi_backend/embedding.py @@ -0,0 +1,146 @@ +import os +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.translator.tool.cbuild import ExternalCompilationInfo + +from pypy.interpreter.error import OperationError, oefmt + +# ____________________________________________________________ + + +EMBED_VERSION_MIN = 0xB011 +EMBED_VERSION_MAX = 0xB0FF + +STDERR = 2 +INITSTRUCTPTR = lltype.Ptr(lltype.Struct('CFFI_INIT', + ('name', rffi.CCHARP), + ('func', rffi.VOIDP), + ('code', rffi.CCHARP))) + +def load_embedded_cffi_module(space, version, init_struct): + from pypy.module._cffi_backend.cffi1_module import load_cffi1_module + declare_c_function() # translation-time hint only: + # declare _cffi_carefully_make_gil() + # + version = rffi.cast(lltype.Signed, version) + if not (EMBED_VERSION_MIN <= version <= EMBED_VERSION_MAX): + raise oefmt(space.w_ImportError, + "cffi embedded module has got unknown version tag %s", + hex(version)) + # + if space.config.objspace.usemodules.thread: + from pypy.module.thread import os_thread + os_thread.setup_threads(space) + # + name = rffi.charp2str(init_struct.name) + load_cffi1_module(space, name, None, init_struct.func) + code = rffi.charp2str(init_struct.code) + compiler = space.createcompiler() + pycode = compiler.compile(code, "<init code for '%s'>" % name, 'exec', 0) + w_globals = space.newdict(module=True) + space.setitem_str(w_globals, "__builtins__", space.wrap(space.builtin)) + pycode.exec_code(space, w_globals, w_globals) + + +class Global: + pass +glob = Global() + +def pypy_init_embedded_cffi_module(version, init_struct): + # called from __init__.py + name = "?" + try: + init_struct = rffi.cast(INITSTRUCTPTR, init_struct) + name = rffi.charp2str(init_struct.name) + # + space = glob.space + must_leave = False + try: + must_leave = space.threadlocals.try_enter_thread(space) + load_embedded_cffi_module(space, version, init_struct) + res = 0 + except OperationError, operr: + operr.write_unraisable(space, "initialization of '%s'" % name, + with_traceback=True) + space.appexec([], r"""(): + import sys + sys.stderr.write('pypy version: %s.%s.%s\n' % + sys.pypy_version_info[:3]) + sys.stderr.write('sys.path: %r\n' % (sys.path,)) + """) + res = -1 + if must_leave: + space.threadlocals.leave_thread(space) + except Exception, e: + # oups! last-level attempt to recover. + try: + os.write(STDERR, "From initialization of '") + os.write(STDERR, name) + os.write(STDERR, "':\n") + os.write(STDERR, str(e)) + os.write(STDERR, "\n") + except: + pass + res = -1 + return rffi.cast(rffi.INT, res) + +# ____________________________________________________________ + + +eci = ExternalCompilationInfo(separate_module_sources=[ +r""" +/* XXX Windows missing */ +#include <stdio.h> +#include <dlfcn.h> +#include <pthread.h> + +RPY_EXPORTED void rpython_startup_code(void); +RPY_EXPORTED int pypy_setup_home(char *, int); + +static unsigned char _cffi_ready = 0; +static const char *volatile _cffi_module_name; + +static void _cffi_init_error(const char *msg, const char *extra) +{ + fprintf(stderr, + "\nPyPy initialization failure when loading module '%s':\n%s%s\n", + _cffi_module_name, msg, extra); +} + +static void _cffi_init(void) +{ + Dl_info info; + char *home; + + rpython_startup_code(); + RPyGilAllocate(); + + if (dladdr(&_cffi_init, &info) == 0) { + _cffi_init_error("dladdr() failed: ", dlerror()); + return; + } + home = realpath(info.dli_fname, NULL); + if (pypy_setup_home(home, 1) != 0) { + _cffi_init_error("pypy_setup_home() failed", ""); + return; + } + _cffi_ready = 1; +} + +RPY_EXPORTED +int pypy_carefully_make_gil(const char *name) +{ + /* For CFFI: this initializes the GIL and loads the home path. + It can be called completely concurrently from unrelated threads. + It assumes that we don't hold the GIL before (if it exists), and we + don't hold it afterwards. + */ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + + _cffi_module_name = name; /* not really thread-safe, but better than + nothing */ + pthread_once(&once_control, _cffi_init); + return (int)_cffi_ready - 1; +} +"""]) + +declare_c_function = rffi.llexternal_use_eci(eci) diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.4.2", ("This test_c.py file is for testing a version" +assert __version__ == "1.5.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_continuation/interp_continuation.py b/pypy/module/_continuation/interp_continuation.py --- a/pypy/module/_continuation/interp_continuation.py +++ b/pypy/module/_continuation/interp_continuation.py @@ -195,7 +195,7 @@ class SThread(StackletThread): def __init__(self, space, ec): - StackletThread.__init__(self, space.config) + StackletThread.__init__(self) self.space = space self.ec = ec # for unpickling diff --git a/pypy/module/_warnings/interp_warnings.py b/pypy/module/_warnings/interp_warnings.py --- a/pypy/module/_warnings/interp_warnings.py +++ b/pypy/module/_warnings/interp_warnings.py @@ -75,7 +75,7 @@ frame = ec.getnextframe_nohidden(frame) stacklevel -= 1 if frame: - w_globals = frame.w_globals + w_globals = frame.get_w_globals() lineno = frame.get_last_lineno() else: w_globals = space.sys.w_dict diff --git a/pypy/module/cpyext/Doc_stubgen_enable.patch b/pypy/module/cpyext/Doc_stubgen_enable.patch deleted file mode 100644 --- a/pypy/module/cpyext/Doc_stubgen_enable.patch +++ /dev/null @@ -1,27 +0,0 @@ -Index: Doc/tools/sphinx/ext/refcounting.py -=================================================================== ---- Doc/tools/sphinx/ext/refcounting.py (Revision 79453) -+++ Doc/tools/sphinx/ext/refcounting.py (Arbeitskopie) -@@ -91,6 +91,7 @@ - if app.config.refcount_file: - refcounts = Refcounts.fromfile( - path.join(app.srcdir, app.config.refcount_file)) -+ app._refcounts = refcounts - app.connect('doctree-read', refcounts.add_refcount_annotations) - - -Index: Doc/conf.py -=================================================================== ---- Doc/conf.py (Revision 79421) -+++ Doc/conf.py (Arbeitskopie) -@@ -13,8 +13,8 @@ - # General configuration - # --------------------- - --extensions = ['sphinx.ext.refcounting', 'sphinx.ext.coverage', -- 'sphinx.ext.doctest', 'pyspecific'] -+extensions = ['pypy.module.cpyext.stubgen', 'sphinx.ext.refcounting', 'sphinx.ext.coverage', -+ 'sphinx.ext.doctest', 'pyspecific', ] - templates_path = ['tools/sphinxext'] - - # General substitutions. diff --git a/pypy/module/cpyext/eval.py b/pypy/module/cpyext/eval.py --- a/pypy/module/cpyext/eval.py +++ b/pypy/module/cpyext/eval.py @@ -30,7 +30,7 @@ currently executing.""" caller = space.getexecutioncontext().gettopframe_nohidden() if caller is not None: - w_globals = caller.w_globals + w_globals = caller.get_w_globals() w_builtins = space.getitem(w_globals, space.wrap('__builtins__')) if not space.isinstance_w(w_builtins, space.w_dict): w_builtins = w_builtins.getdict(space) @@ -54,7 +54,7 @@ caller = space.getexecutioncontext().gettopframe_nohidden() if caller is None: return None - return borrow_from(None, caller.w_globals) + return borrow_from(None, caller.get_w_globals()) @cpython_api([PyCodeObject, PyObject, PyObject], PyObject) def PyEval_EvalCode(space, w_code, w_globals, w_locals): diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -34,7 +34,7 @@ frame = space.interp_w(PyFrame, w_obj) py_frame = rffi.cast(PyFrameObject, py_obj) py_frame.c_f_code = rffi.cast(PyCodeObject, make_ref(space, frame.pycode)) - py_frame.c_f_globals = make_ref(space, frame.w_globals) + py_frame.c_f_globals = make_ref(space, frame.get_w_globals()) rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno) @cpython_api([PyObject], lltype.Void, external=False) diff --git a/pypy/module/cpyext/import_.py b/pypy/module/cpyext/import_.py --- a/pypy/module/cpyext/import_.py +++ b/pypy/module/cpyext/import_.py @@ -20,7 +20,7 @@ caller = space.getexecutioncontext().gettopframe_nohidden() # Get the builtins from current globals if caller is not None: - w_globals = caller.w_globals + w_globals = caller.get_w_globals() w_builtin = space.getitem(w_globals, space.wrap('__builtins__')) else: # No globals -- use standard builtins, and fake globals diff --git a/pypy/module/cpyext/patches/Doc_stubgen_enable.patch b/pypy/module/cpyext/patches/Doc_stubgen_enable.patch new file mode 100644 --- /dev/null +++ b/pypy/module/cpyext/patches/Doc_stubgen_enable.patch @@ -0,0 +1,27 @@ +Index: Doc/tools/sphinx/ext/refcounting.py +=================================================================== +--- Doc/tools/sphinx/ext/refcounting.py (Revision 79453) ++++ Doc/tools/sphinx/ext/refcounting.py (Arbeitskopie) +@@ -91,6 +91,7 @@ + if app.config.refcount_file: + refcounts = Refcounts.fromfile( + path.join(app.srcdir, app.config.refcount_file)) ++ app._refcounts = refcounts + app.connect('doctree-read', refcounts.add_refcount_annotations) + + +Index: Doc/conf.py +=================================================================== +--- Doc/conf.py (Revision 79421) ++++ Doc/conf.py (Arbeitskopie) +@@ -13,8 +13,8 @@ + # General configuration + # --------------------- + +-extensions = ['sphinx.ext.refcounting', 'sphinx.ext.coverage', +- 'sphinx.ext.doctest', 'pyspecific'] ++extensions = ['pypy.module.cpyext.stubgen', 'sphinx.ext.refcounting', 'sphinx.ext.coverage', ++ 'sphinx.ext.doctest', 'pyspecific', ] + templates_path = ['tools/sphinxext'] + + # General substitutions. diff --git a/pypy/module/itertools/interp_itertools.py b/pypy/module/itertools/interp_itertools.py --- a/pypy/module/itertools/interp_itertools.py +++ b/pypy/module/itertools/interp_itertools.py @@ -372,7 +372,7 @@ def arg_int_w(self, w_obj, minimum, errormsg): space = self.space try: - result = space.int_w(w_obj) + result = space.int_w(space.int(w_obj)) # CPython allows floats as parameters except OperationError, e: if e.async(space): raise diff --git a/pypy/module/itertools/test/test_itertools.py b/pypy/module/itertools/test/test_itertools.py --- a/pypy/module/itertools/test/test_itertools.py +++ b/pypy/module/itertools/test/test_itertools.py @@ -225,6 +225,12 @@ assert it.next() == x raises(StopIteration, it.next) + # CPython implementation allows floats + it = itertools.islice([1, 2, 3, 4, 5], 0.0, 3.0, 2.0) + for x in [1, 3]: + assert it.next() == x + raises(StopIteration, it.next) + it = itertools.islice([1, 2, 3], 0, None) for x in [1, 2, 3]: assert it.next() == x diff --git a/pypy/module/pypyjit/test_pypy_c/test_00_model.py b/pypy/module/pypyjit/test_pypy_c/test_00_model.py --- a/pypy/module/pypyjit/test_pypy_c/test_00_model.py +++ b/pypy/module/pypyjit/test_pypy_c/test_00_model.py @@ -526,7 +526,7 @@ log = self.run(f) loop, = log.loops_by_filename(self.filepath) call_ops = log.opnames(loop.ops_by_id('call')) - assert call_ops == ['force_token'] # it does not follow inlining + assert call_ops == ['guard_not_invalidated', 'force_token'] # it does not follow inlining # add_ops = log.opnames(loop.ops_by_id('add')) assert add_ops == ['int_add'] @@ -534,9 +534,10 @@ ops = log.opnames(loop.allops()) assert ops == [ # this is the actual loop - 'int_lt', 'guard_true', 'force_token', 'int_add', + 'int_lt', 'guard_true', + 'guard_not_invalidated', 'force_token', 'int_add', # this is the signal checking stuff - 'guard_not_invalidated', 'getfield_raw_i', 'int_lt', 'guard_false', + 'getfield_raw_i', 'int_lt', 'guard_false', 'jump' ] diff --git a/pypy/module/pypyjit/test_pypy_c/test_call.py b/pypy/module/pypyjit/test_pypy_c/test_call.py --- a/pypy/module/pypyjit/test_pypy_c/test_call.py +++ b/pypy/module/pypyjit/test_pypy_c/test_call.py @@ -72,8 +72,6 @@ # LOAD_GLOBAL of OFFSET ops = entry_bridge.ops_by_id('cond', opcode='LOAD_GLOBAL') assert log.opnames(ops) == ["guard_value", - "guard_value", - "getfield_gc_r", "guard_value", "guard_not_invalidated"] ops = entry_bridge.ops_by_id('add', opcode='LOAD_GLOBAL') assert log.opnames(ops) == [] @@ -200,6 +198,7 @@ assert log.result == 1000 loop, = log.loops_by_id('call') assert loop.match_by_id('call', """ + guard_not_invalidated? i14 = force_token() i16 = force_token() """) @@ -222,7 +221,7 @@ loop, = log.loops_by_id('call') ops = log.opnames(loop.ops_by_id('call')) guards = [ops for ops in ops if ops.startswith('guard')] - assert guards == ["guard_no_overflow"] + assert guards == ["guard_not_invalidated", "guard_no_overflow"] def test_kwargs(self): # this is not a very precise test, could be improved @@ -281,6 +280,7 @@ assert log.result == 13000 loop0, = log.loops_by_id('g1') assert loop0.match_by_id('g1', """ + guard_not_invalidated? i20 = force_token() i22 = int_add_ovf(i8, 3) guard_no_overflow(descr=...) @@ -438,9 +438,6 @@ i22 = getfield_gc_pure_i(p12, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>) i24 = int_lt(i22, 5000) guard_true(i24, descr=...) - guard_value(p7, ConstPtr(ptr25), descr=...) - p26 = getfield_gc_r(p7, descr=<FieldP pypy.objspace.std.dictmultiobject.W_DictMultiObject.inst_strategy .*>) - guard_value(p26, ConstPtr(ptr27), descr=...) guard_not_invalidated(descr=...) p29 = call_r(ConstClass(_ll_1_threadlocalref_get__Ptr_GcStruct_objectLlT_Signed), #, descr=<Callr . i EF=1 OS=5>) p30 = getfield_gc_r(p29, descr=<FieldP pypy.interpreter.executioncontext.ExecutionContext.inst_topframeref .*>) @@ -472,6 +469,7 @@ i8 = getfield_gc_pure_i(p6, descr=<FieldS pypy.objspace.std.intobject.W_IntObject.inst_intval .*>) i10 = int_lt(i8, 5000) guard_true(i10, descr=...) + guard_not_invalidated? i11 = force_token() i13 = int_add(i8, 1) --TICK-- diff --git a/pypy/module/pypyjit/test_pypy_c/test_globals.py b/pypy/module/pypyjit/test_pypy_c/test_globals.py --- a/pypy/module/pypyjit/test_pypy_c/test_globals.py +++ b/pypy/module/pypyjit/test_pypy_c/test_globals.py @@ -16,9 +16,5 @@ assert log.result == 500 loop, = log.loops_by_filename(self.filepath) assert loop.match_by_id("loadglobal", """ - p12 = getfield_gc_r(p10, descr=<FieldP .*W_DictMultiObject.inst_strategy .*>) - guard_value(p12, ConstPtr(ptr13), descr=...) guard_not_invalidated(descr=...) - p19 = getfield_gc_r(ConstPtr(p17), descr=<FieldP .*W_DictMultiObject.inst_strategy .*>) - guard_value(p19, ConstPtr(ptr20), descr=...) """) diff --git a/pypy/module/pypyjit/test_pypy_c/test_instance.py b/pypy/module/pypyjit/test_pypy_c/test_instance.py --- a/pypy/module/pypyjit/test_pypy_c/test_instance.py +++ b/pypy/module/pypyjit/test_pypy_c/test_instance.py @@ -124,7 +124,7 @@ setfield_gc(ConstPtr(ptr39), i59, descr=...) i62 = int_lt(i61, 0) guard_false(i62, descr=...) - jump(p0, p1, p3, p6, p7, p12, i59, p18, i31, i59, p100, descr=...) + jump(..., descr=...) """) def test_mutate_class(self): @@ -183,7 +183,7 @@ setfield_gc(p77, ConstPtr(null), descr=...) setfield_gc(p77, ConstPtr(ptr42), descr=...) setfield_gc(ConstPtr(ptr69), p77, descr=...) - jump(p0, p1, p3, p6, p7, p12, i74, p20, p26, i33, p77, p100, descr=...) + jump(..., descr=...) """) diff --git a/pypy/module/pypyjit/test_pypy_c/test_misc.py b/pypy/module/pypyjit/test_pypy_c/test_misc.py --- a/pypy/module/pypyjit/test_pypy_c/test_misc.py +++ b/pypy/module/pypyjit/test_pypy_c/test_misc.py @@ -145,9 +145,9 @@ i15 = int_lt(i10, i11) guard_true(i15, descr=...) i17 = int_add(i10, 1) - i18 = force_token() setfield_gc(p9, i17, descr=<.* .*W_XRangeIterator.inst_current .*>) guard_not_invalidated(descr=...) + i18 = force_token() i84 = int_sub(i14, 1) i21 = int_lt(i10, 0) guard_false(i21, descr=...) @@ -178,9 +178,9 @@ i16 = int_ge(i11, i12) guard_false(i16, descr=...) i20 = int_add(i11, 1) - i21 = force_token() setfield_gc(p4, i20, descr=<.* .*W_AbstractSeqIterObject.inst_index .*>) guard_not_invalidated? + i21 = force_token() i88 = int_sub(i9, 1) i25 = int_ge(i11, i9) guard_false(i25, descr=...) @@ -211,9 +211,9 @@ i17 = int_mul(i11, i14) i18 = int_add(i15, i17) i20 = int_add(i11, 1) - i21 = force_token() _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit
