Author: Ronan Lamy <[email protected]>
Branch: PEP393
Changeset: r89920:791a64ac3ea2
Date: 2017-02-04 17:56 +0000
http://bitbucket.org/pypy/pypy/changeset/791a64ac3ea2/
Log: hg merge py3.5
diff too long, truncating to 2000 out of 5539 lines
diff --git a/lib-python/3/datetime.py b/lib-python/3/datetime.py
--- a/lib-python/3/datetime.py
+++ b/lib-python/3/datetime.py
@@ -706,7 +706,7 @@
@classmethod
def fromordinal(cls, n):
- """Contruct a date from a proleptic Gregorian ordinal.
+ """Construct a date from a proleptic Gregorian ordinal.
January 1 of year 1 is day 1. Only the year, month and day are
non-zero in the result.
@@ -1145,18 +1145,17 @@
def __hash__(self):
"""Hash."""
if self._hashcode == -1:
+ # PyPy: uses the same algo as _datetimemodule.c, which
+ # unlike the pure Python version always relies on the
+ # nondeterministic hash on strings
+ temp1 = timedelta(hours=self._hour,
+ minutes=self._minute,
+ seconds=self._second,
+ microseconds=self._microsecond)
tzoff = self.utcoffset()
- if not tzoff: # zero or None
- self._hashcode = hash(self._getstate()[0])
- else:
- h, m = divmod(timedelta(hours=self.hour, minutes=self.minute)
- tzoff,
- timedelta(hours=1))
- assert not m % timedelta(minutes=1), "whole minute"
- m //= timedelta(minutes=1)
- if 0 <= h < 24:
- self._hashcode = hash(time(h, m, self.second,
self.microsecond))
- else:
- self._hashcode = hash((h, m, self.second,
self.microsecond))
+ if tzoff: # not zero, not None
+ temp1 -= tzoff
+ self._hashcode = hash(temp1)
return self._hashcode
# Conversion to string
diff --git a/lib-python/3/importlib/_bootstrap_external.py
b/lib-python/3/importlib/_bootstrap_external.py
--- a/lib-python/3/importlib/_bootstrap_external.py
+++ b/lib-python/3/importlib/_bootstrap_external.py
@@ -238,6 +238,12 @@
# different than CPython's, always < 3000. We get the 4-bytes string
# here via a hack: MAGIC_NUMBER is set in the module from
# module/_frozen_importlib/__init__.py before the module is executed.
+# FOR TESTS ONLY, we make it default to imp.get_magic().
+try:
+ MAGIC_NUMBER
+except NameError:
+ import imp
+ MAGIC_NUMBER = imp.get_magic()
_RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c
diff --git a/lib-python/3/test/_test_multiprocessing.py
b/lib-python/3/test/_test_multiprocessing.py
--- a/lib-python/3/test/_test_multiprocessing.py
+++ b/lib-python/3/test/_test_multiprocessing.py
@@ -2795,6 +2795,7 @@
self.assertTrue((arena != narena and nstart == 0) or
(stop == nstart))
+ @test.support.cpython_only
def test_free_from_gc(self):
# Check that freeing of blocks by the garbage collector doesn't
deadlock
# (issue #12352).
@@ -2893,12 +2894,14 @@
a = Foo()
util.Finalize(a, conn.send, args=('a',))
del a # triggers callback for a
+ import gc; gc.collect()
b = Foo()
close_b = util.Finalize(b, conn.send, args=('b',))
close_b() # triggers callback for b
close_b() # does nothing because callback has already been called
del b # does nothing because callback has already been called
+ import gc; gc.collect()
c = Foo()
util.Finalize(c, conn.send, args=('c',))
diff --git a/lib-python/3/test/test_hash.py b/lib-python/3/test/test_hash.py
--- a/lib-python/3/test/test_hash.py
+++ b/lib-python/3/test/test_hash.py
@@ -43,7 +43,7 @@
def skip_unless_internalhash(test):
"""Skip decorator for tests that depend on SipHash24 or FNV"""
- ok = sys.hash_info.algorithm in {"fnv", "siphash24", "rpython"}
+ ok = sys.hash_info.algorithm in {"fnv", "siphash24"}
msg = "Requires SipHash24 or FNV"
return test if ok else unittest.skip(msg)(test)
@@ -189,7 +189,6 @@
stdout = out[1].strip()
return int(stdout)
- @impl_detail("PyPy does not support hash randomization", pypy=False)
def test_randomized_hash(self):
# two runs should return different hashes
run1 = self.get_hash(self.repr_, seed='random')
@@ -197,10 +196,6 @@
self.assertNotEqual(run1, run2)
class StringlikeHashRandomizationTests(HashRandomizationTests):
- if check_impl_detail(pypy=True):
- EMPTY_STRING_HASH = -2
- else:
- EMPTY_STRING_HASH = 0
repr_ = None
repr_long = None
@@ -242,9 +237,9 @@
[-283066365, -4576729883824601543, -271871407,
-3927695501187247084],
],
- 'rpython': [
- # This is for PyPy. NOTE: PyPy does not support hash
- # randomization for now, so the results don't depend on the seed.
+ 'fnv-pypy': [
+ # This is for PyPy, whose fnv algorithm does not support hash
+ # randomization, so the results don't depend on the seed.
# seed 0, 'abc'
[-1600925533, 1453079729188098211, -1600925533,
1453079729188098211],
@@ -273,6 +268,8 @@
else:
assert(sys.byteorder == 'big')
platform = 3 if IS_64BIT else 2
+ if algorithm == 'fnv' and check_impl_detail(pypy=True):
+ algorithm = 'fnv-pypy'
return self.known_hashes[algorithm][position][platform]
def test_null_hash(self):
@@ -280,8 +277,7 @@
known_hash_of_obj = self.get_expected_hash(0, 3)
# Randomization is enabled by default:
- if check_impl_detail(pypy=False):
- self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
+ self.assertNotEqual(self.get_hash(self.repr_), known_hash_of_obj)
# It can also be disabled by setting the seed to 0:
self.assertEqual(self.get_hash(self.repr_, seed=0), known_hash_of_obj)
@@ -307,11 +303,12 @@
repr_long = repr('abcdefghijk')
repr_ucs2 = repr('äú∑ℇ')
- @impl_detail("hash('') == -2 on PyPy", pypy=False)
+ @impl_detail("hash('') != 0 on PyPy", pypy=False)
@skip_unless_internalhash
def test_empty_string(self):
self.assertEqual(hash(""), 0)
+ @impl_detail("hash(ucs2) differs on PyPy if unichar is 4 bytes",
pypy=False)
@skip_unless_internalhash
def test_ucs2_string(self):
h = self.get_expected_hash(3, 6)
@@ -324,18 +321,20 @@
repr_ = repr(b'abc')
repr_long = repr(b'abcdefghijk')
+ @impl_detail("hash('') != 0 on PyPy", pypy=False)
@skip_unless_internalhash
def test_empty_string(self):
- self.assertEqual(hash(b""), self.EMPTY_STRING_HASH)
+ self.assertEqual(hash(b""), 0)
class MemoryviewHashRandomizationTests(StringlikeHashRandomizationTests,
unittest.TestCase):
repr_ = "memoryview(b'abc')"
repr_long = "memoryview(b'abcdefghijk')"
+ @impl_detail("hash('') != 0 on PyPy", pypy=False)
@skip_unless_internalhash
def test_empty_string(self):
- self.assertEqual(hash(memoryview(b"")), self.EMPTY_STRING_HASH)
+ self.assertEqual(hash(memoryview(b"")), 0)
class DatetimeTests(HashRandomizationTests):
def get_hash_command(self, repr_):
diff --git a/lib-python/3/test/test_signal.py b/lib-python/3/test/test_signal.py
--- a/lib-python/3/test/test_signal.py
+++ b/lib-python/3/test/test_signal.py
@@ -358,6 +358,8 @@
assert_python_ok('-c', code)
+ @support.impl_detail("pypy writes the message to fd 2, not to sys.stderr",
+ pypy=False)
@unittest.skipIf(_testcapi is None, 'need _testcapi')
def test_wakeup_write_error(self):
# Issue #16105: write() errors in the C signal handler should not
diff --git a/lib-python/conftest.py b/lib-python/conftest.py
--- a/lib-python/conftest.py
+++ b/lib-python/conftest.py
@@ -261,7 +261,7 @@
RegrTest('test_imaplib.py'),
RegrTest('test_imghdr.py'),
RegrTest('test_import'),
- RegrTest('test_importlib', skip='XXX segfaults'),
+ RegrTest('test_importlib'),
RegrTest('test_imp.py', core=True, usemodules='thread'),
RegrTest('test_index.py'),
RegrTest('test_inspect.py', usemodules="struct unicodedata"),
@@ -304,10 +304,10 @@
RegrTest('test_modulefinder.py'),
RegrTest('test_msilib.py'),
RegrTest('test_multibytecodec.py', usemodules='_multibytecodec'),
- RegrTest('test_multiprocessing_fork.py', skip="XXX: deadlocks?"),
- RegrTest('test_multiprocessing_forkserver.py', skip="XXX: deadlocks?"),
- RegrTest('test_multiprocessing_main_handling.py', skip="XXX: deadlocks?"),
- RegrTest('test_multiprocessing_spawn.py', skip="XXX: deadlocks?"),
+ RegrTest('test_multiprocessing_fork.py'),
+ RegrTest('test_multiprocessing_forkserver.py'),
+ RegrTest('test_multiprocessing_main_handling.py'),
+ RegrTest('test_multiprocessing_spawn.py'),
RegrTest('test_netrc.py'),
RegrTest('test_nis.py'),
RegrTest('test_nntplib.py'),
diff --git a/lib_pypy/_ctypes/basics.py b/lib_pypy/_ctypes/basics.py
--- a/lib_pypy/_ctypes/basics.py
+++ b/lib_pypy/_ctypes/basics.py
@@ -92,7 +92,11 @@
% (buf.nbytes, offset + size))
raw_addr = buf._pypy_raw_address() + offset
result = self.from_address(raw_addr)
- result._ensure_objects()['ffffffff'] = obj
+ objects = result._ensure_objects()
+ if objects is not None:
+ objects['ffffffff'] = obj
+ else: # case e.g. of a primitive type like c_int
+ result._objects = obj
return result
def from_buffer_copy(self, obj, offset=0):
diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py
--- a/pypy/config/pypyoption.py
+++ b/pypy/config/pypyoption.py
@@ -201,6 +201,13 @@
"issue, you can disable them here",
default=True),
+ ChoiceOption("hash",
+ "The hash function to use for strings: fnv from CPython 2.7"
+ " or siphash24 from CPython >= 3.4",
+ ["fnv", "siphash24"],
+ default="siphash24",
+ cmdline="--hash"),
+
OptionDescription("std", "Standard Object Space Options", [
BoolOption("withtproxy", "support transparent proxies",
default=True),
diff --git a/pypy/doc/build.rst b/pypy/doc/build.rst
--- a/pypy/doc/build.rst
+++ b/pypy/doc/build.rst
@@ -56,6 +56,9 @@
.. _`arm document`: http://rpython.readthedocs.org/en/latest/arm.html
.. _`RPython documentation`: http://rpython.readthedocs.org
+The host Python needs to have CFFI installed. If translating on PyPy, CFFI is
+already installed. If translating on CPython, you need to install it, e.g.
+using ``pip install cffi``.
To build PyPy on Unix using the C translation backend, you need at least a C
compiler and ``make`` installed. Further, some optional modules have additional
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -134,3 +134,12 @@
rffi structures in cpyext can now be created by parsing simple C headers.
Additionally, the cts object that holds the parsed information can act like
cffi's ffi objects, with the methods cts.cast() and cts.gettype().
+
+.. branch: rpython-hash
+
+Don't freeze hashes in the translated pypy. In practice, that means
+that we can now translate PyPy with the option --hash=siphash24 and get
+the same hashes as CPython 3.5, which can be randomized (in a
+crypographically good way). It is the default in PyPy3. The default of
+PyPy2 remains unchanged: there are user programs out there that depend
+on constant hashes (or even sometimes on specific hash results).
diff --git a/pypy/goal/targetpypystandalone.py
b/pypy/goal/targetpypystandalone.py
--- a/pypy/goal/targetpypystandalone.py
+++ b/pypy/goal/targetpypystandalone.py
@@ -36,6 +36,7 @@
w_run_toplevel = space.getitem(w_dict, space.wrap('run_toplevel'))
w_initstdio = space.getitem(w_dict, space.wrap('initstdio'))
withjit = space.config.objspace.usemodules.pypyjit
+ hashfunc = space.config.objspace.hash
else:
w_initstdio = space.appexec([], """():
return lambda unbuffered: None
@@ -46,6 +47,10 @@
from rpython.jit.backend.hlinfo import highleveljitinfo
highleveljitinfo.sys_executable = argv[0]
+ if hashfunc == "siphash24":
+ from rpython.rlib import rsiphash
+ rsiphash.enable_siphash24()
+
#debug("entry point starting")
#for arg in argv:
# debug(" argv -> " + arg)
diff --git a/pypy/interpreter/app_main.py b/pypy/interpreter/app_main.py
--- a/pypy/interpreter/app_main.py
+++ b/pypy/interpreter/app_main.py
@@ -728,8 +728,26 @@
if IS_WINDOWS:
filename = filename.lower()
if filename.endswith('.pyc') or filename.endswith('.pyo'):
+ # We don't actually load via SourcelessFileLoader
+ # because '__main__' must not be listed inside
+ # 'importlib._bootstrap._module_locks' (it deadlocks
+ # test_multiprocessing_main_handling.test_script_compiled)
+ from importlib._bootstrap_external import MAGIC_NUMBER
+ import marshal
loader = SourcelessFileLoader('__main__', filename)
- args = (loader.load_module, loader.name)
+ mainmodule.__loader__ = loader
+ @hidden_applevel
+ def execfile(filename, namespace):
+ with open(filename, 'rb') as f:
+ if f.read(4) != MAGIC_NUMBER:
+ raise RuntimeError("Bad magic number in .pyc file")
+ if len(f.read(8)) != 8:
+ raise RuntimeError("Truncated .pyc file")
+ co = marshal.load(f)
+ if type(co) is not type((lambda:0).__code__):
+ raise RuntimeError("Bad code object in .pyc file")
+ exec_(co, namespace)
+ args = (execfile, filename, mainmodule.__dict__)
else:
filename = sys.argv[0]
for hook in sys.path_hooks:
diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py
--- a/pypy/interpreter/baseobjspace.py
+++ b/pypy/interpreter/baseobjspace.py
@@ -225,14 +225,14 @@
def buffer_w(self, space, flags):
if _does_override_buffer_w_ex(self.__class__):
return self.buffer_w_ex(space, flags)[0]
- return self._buffer(space, flags).buffer_w(space, flags)
+ return self.__buffer_w(space, flags).buffer_w(space, flags)
def buffer_w_ex(self, space, flags):
if _does_override_buffer_w(self.__class__):
return self.buffer_w(space, flags), 'B', 1
- return self._buffer(space, flags).buffer_w_ex(space, flags)
+ return self.__buffer_w(space, flags).buffer_w_ex(space, flags)
- def _buffer(self, space, flags):
+ def __buffer_w(self, space, flags):
if flags & space.BUF_WRITABLE:
w_impl = space.lookup(self, '__wbuffer__')
else:
diff --git a/pypy/interpreter/error.py b/pypy/interpreter/error.py
--- a/pypy/interpreter/error.py
+++ b/pypy/interpreter/error.py
@@ -7,7 +7,7 @@
from rpython.rlib import jit
from rpython.rlib.objectmodel import we_are_translated, specialize
-from rpython.rlib import rstackovf
+from rpython.rlib import rstack, rstackovf
from pypy.interpreter import debug
@@ -60,6 +60,7 @@
"Check if this is an exception that should better not be caught."
return (self.match(space, space.w_SystemExit) or
self.match(space, space.w_KeyboardInterrupt))
+ # note: an extra case is added in OpErrFmtNoArgs
def __str__(self):
"NOT_RPYTHON: Convenience for tracebacks."
@@ -512,6 +513,16 @@
def _compute_value(self, space):
return self._value.decode('utf-8')
+ def async(self, space):
+ # also matches a RuntimeError("maximum rec.") if the stack is
+ # still almost full, because in this case it might be a better
+ # idea to propagate the exception than eat it
+ if (self.w_type is space.w_RecursionError and
+ self._value == "maximum recursion depth exceeded" and
+ rstack.stack_almost_full()):
+ return True
+ return OperationError.async(self, space)
+
@specialize.memo()
def get_operr_class(valuefmt):
try:
diff --git a/pypy/interpreter/test/test_function.py
b/pypy/interpreter/test/test_function.py
--- a/pypy/interpreter/test/test_function.py
+++ b/pypy/interpreter/test/test_function.py
@@ -160,10 +160,8 @@
def test_write_code_builtin_forbidden(self):
def f(*args):
return 42
- if hasattr('dir', '__code__'):
- # only on PyPy, CPython does not expose these attrs
- raises(TypeError, "dir.__code__ = f.__code__")
- raises(TypeError, "list().append.__func__.__code__ = f.__code__")
+ raises(TypeError, "dir.__code__ = f.__code__")
+ raises(TypeError, "list.append.__code__ = f.__code__")
def test_set_module_to_name_eagerly(self):
skip("fails on PyPy but works on CPython. Unsure we want to care")
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -88,6 +88,7 @@
'_promote' : 'interp_magic._promote',
'normalize_exc' : 'interp_magic.normalize_exc',
'StdErrPrinter' : 'interp_stderrprinter.W_StdErrPrinter',
+ 'stack_almost_full' : 'interp_magic.stack_almost_full',
}
submodules = {
diff --git a/pypy/module/__pypy__/interp_magic.py
b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -9,7 +9,7 @@
from pypy.objspace.std.setobject import W_BaseSetObject
from pypy.objspace.std.typeobject import MethodCache
from pypy.objspace.std.mapdict import MapAttrCache
-from rpython.rlib import rposix, rgc
+from rpython.rlib import rposix, rgc, rstack
def internal_repr(space, w_object):
@@ -176,3 +176,7 @@
operr = OperationError(w_type, w_value, w_tb)
operr.normalize_exception(space)
return operr.get_w_value(space)
+
+def stack_almost_full(space):
+ """Return True if the stack is more than 15/16th full."""
+ return space.wrap(rstack.stack_almost_full())
diff --git a/pypy/module/_cffi_backend/newtype.py
b/pypy/module/_cffi_backend/newtype.py
--- a/pypy/module/_cffi_backend/newtype.py
+++ b/pypy/module/_cffi_backend/newtype.py
@@ -23,13 +23,34 @@
# ____________________________________________________________
class UniqueCache:
+ for_testing = False # set to True on the class level in test_c.py
+
def __init__(self, space):
self.ctvoid = None # Cache for the 'void' type
self.ctvoidp = None # Cache for the 'void *' type
self.ctchara = None # Cache for the 'char[]' type
self.primitives = {} # Cache for {name: primitive_type}
self.functions = [] # see _new_function_type()
- self.for_testing = False
+ self.functions_packed = None # only across translation
+
+ def _cleanup_(self):
+ import gc
+ assert self.functions_packed is None
+ # Note: a full PyPy translation may still have
+ # 'self.functions == []' at this point, possibly depending
+ # on details. Code tested directly in test_ffi_obj
+ gc.collect()
+ funcs = []
+ for weakdict in self.functions:
+ funcs += weakdict._dict.values()
+ del self.functions[:]
+ self.functions_packed = funcs if len(funcs) > 0 else None
+
+ def unpack_functions(self):
+ for fct in self.functions_packed:
+ _record_function_type(self, fct)
+ self.functions_packed = None
+
def _clean_cache(space):
"NOT_RPYTHON"
@@ -622,7 +643,7 @@
for w_arg in fargs:
y = compute_identity_hash(w_arg)
x = intmask((1000003 * x) ^ y)
- x ^= (ellipsis - abi)
+ x ^= ellipsis + 2 * abi
if unique_cache.for_testing: # constant-folded to False in translation;
x &= 3 # but for test, keep only 2 bits of hash
return x
@@ -646,6 +667,8 @@
# one such dict, but in case of hash collision, there might be
# more.
unique_cache = space.fromcache(UniqueCache)
+ if unique_cache.functions_packed is not None:
+ unique_cache.unpack_functions()
func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
for weakdict in unique_cache.functions:
ctype = weakdict.get(func_hash)
@@ -674,13 +697,18 @@
#
fct = ctypefunc.W_CTypeFunc(space, fargs, fresult, ellipsis, abi)
unique_cache = space.fromcache(UniqueCache)
- func_hash = _func_key_hash(unique_cache, fargs, fresult, ellipsis, abi)
+ _record_function_type(unique_cache, fct)
+ return fct
+
+def _record_function_type(unique_cache, fct):
+ from pypy.module._cffi_backend import ctypefunc
+ #
+ func_hash = _func_key_hash(unique_cache, fct.fargs, fct.ctitem,
+ fct.ellipsis, fct.abi)
for weakdict in unique_cache.functions:
if weakdict.get(func_hash) is None:
- weakdict.set(func_hash, fct)
break
else:
weakdict = rweakref.RWeakValueDictionary(int, ctypefunc.W_CTypeFunc)
unique_cache.functions.append(weakdict)
- weakdict.set(func_hash, fct)
- return fct
+ weakdict.set(func_hash, fct)
diff --git a/pypy/module/_cffi_backend/test/test_c.py
b/pypy/module/_cffi_backend/test/test_c.py
--- a/pypy/module/_cffi_backend/test/test_c.py
+++ b/pypy/module/_cffi_backend/test/test_c.py
@@ -37,6 +37,7 @@
def setup_class(cls):
testfuncs_w = []
keepalive_funcs = []
+ UniqueCache.for_testing = True
test_lib_c = tmpdir.join('_test_lib.c')
src_test_lib_c = py.path.local(__file__).dirpath().join('_test_lib.c')
@@ -100,11 +101,12 @@
_all_test_c.find_and_load_library = func
_all_test_c._testfunc = testfunc
""")
- UniqueCache.for_testing = True
def teardown_method(self, method):
+ _clean_cache(self.space)
+
+ def teardown_class(cls):
UniqueCache.for_testing = False
- _clean_cache(self.space)
all_names = ', '.join(Module.interpleveldefs.keys())
diff --git a/pypy/module/_cffi_backend/test/test_ffi_obj.py
b/pypy/module/_cffi_backend/test/test_ffi_obj.py
--- a/pypy/module/_cffi_backend/test/test_ffi_obj.py
+++ b/pypy/module/_cffi_backend/test/test_ffi_obj.py
@@ -1,5 +1,23 @@
+from pypy.module._cffi_backend import newtype
from pypy.module._cffi_backend.newtype import _clean_cache
+
+class TestFFIObj:
+ spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
+
+ def teardown_method(self, meth):
+ _clean_cache(self.space)
+
+ def test_new_function_type_during_translation(self):
+ space = self.space
+ BInt = newtype.new_primitive_type(space, "int")
+ BFunc = newtype.new_function_type(space, space.wrap([BInt]), BInt)
+ assert BFunc is
newtype.new_function_type(space,space.wrap([BInt]),BInt)
+ unique_cache = space.fromcache(newtype.UniqueCache)
+ unique_cache._cleanup_()
+ assert BFunc is
newtype.new_function_type(space,space.wrap([BInt]),BInt)
+
+
class AppTestFFIObj:
spaceconfig = dict(usemodules=('_cffi_backend', 'array'))
diff --git a/pypy/module/_multiprocessing/__init__.py
b/pypy/module/_multiprocessing/__init__.py
--- a/pypy/module/_multiprocessing/__init__.py
+++ b/pypy/module/_multiprocessing/__init__.py
@@ -5,7 +5,6 @@
class Module(MixedModule):
interpleveldefs = {
- 'Connection' : 'interp_connection.W_FileConnection',
'SemLock' : 'interp_semaphore.W_SemLock',
'sem_unlink' : 'interp_semaphore.semaphore_unlink',
'address_of_buffer' : 'interp_memory.address_of_buffer',
@@ -15,12 +14,5 @@
}
if sys.platform == 'win32':
- interpleveldefs['PipeConnection'] = \
- 'interp_connection.W_PipeConnection'
interpleveldefs['win32'] = 'interp_win32.win32_namespace(space)'
del interpleveldefs['sem_unlink']
-
- def init(self, space):
- MixedModule.init(self, space)
- from pypy.module._multiprocessing.interp_connection import State
- space.fromcache(State).init(space)
diff --git a/pypy/module/_multiprocessing/interp_connection.py
b/pypy/module/_multiprocessing/interp_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/interp_connection.py
+++ /dev/null
@@ -1,547 +0,0 @@
-import sys
-from errno import EINTR
-
-from rpython.rlib import rpoll, rsocket
-from rpython.rlib.objectmodel import we_are_translated
-from rpython.rlib.rarithmetic import intmask
-from rpython.rtyper.lltypesystem import lltype, rffi
-
-from pypy.interpreter.baseobjspace import W_Root
-from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
-from pypy.interpreter.gateway import (
- WrappedDefault, interp2app, interpindirect2app, unwrap_spec)
-from pypy.interpreter.typedef import GetSetProperty, TypeDef
-
-READABLE, WRITABLE = range(1, 3)
-PY_SSIZE_T_MAX = sys.maxint
-PY_SSIZE_T_MIN = -sys.maxint - 1
-
-class State(object):
- def __init__(self, space):
- pass
-
- def init(self, space):
- w_builtins = space.getbuiltinmodule('builtins')
- w_module = space.call_method(
- w_builtins, '__import__', space.wrap("multiprocessing"))
- self.w_BufferTooShort = space.getattr(w_module,
space.wrap("BufferTooShort"))
-
- self.w_picklemodule = space.call_method(
- w_builtins, '__import__', space.wrap("pickle"))
-
-def BufferTooShort(space, w_data):
- state = space.fromcache(State)
- if not we_are_translated() and not hasattr(state, 'w_BufferTooShort'):
- state.init(space) # xxx for test/test_connection.py
- w_BufferTooShort = state.w_BufferTooShort
- return OperationError(w_BufferTooShort, w_data)
-
-def w_handle(space, handle):
- return space.wrap(rffi.cast(rffi.INTPTR_T, handle))
-
-
-class W_BaseConnection(W_Root):
- BUFFER_SIZE = 1024
- buffer = lltype.nullptr(rffi.CCHARP.TO)
-
- def __init__(self, space, flags):
- self.flags = flags
- self.buffer = lltype.malloc(rffi.CCHARP.TO, self.BUFFER_SIZE,
- flavor='raw')
- self.register_finalizer(space)
-
- def _finalize_(self):
- buf = self.buffer
- if buf:
- self.buffer = lltype.nullptr(rffi.CCHARP.TO)
- lltype.free(buf, flavor='raw')
- try:
- self.do_close()
- except OSError:
- pass
-
- # Abstract methods
- def do_close(self):
- raise NotImplementedError
- def is_valid(self):
- return False
- def do_send_string(self, space, buf, offset, size):
- raise NotImplementedError
- def do_recv_string(self, space, buflength, maxlength):
- raise NotImplementedError
- def do_poll(self, space, timeout):
- raise NotImplementedError
-
- def close(self):
- self.do_close()
-
- def closed_get(self, space):
- return space.newbool(not self.is_valid())
- def readable_get(self, space):
- return space.newbool(bool(self.flags & READABLE))
- def writable_get(self, space):
- return space.newbool(bool(self.flags & WRITABLE))
-
- def _repr(self, space, handle):
- index = self.flags - 1
- conn_type = [u"read-only", u"write-only", u"read-write"][index]
- return space.wrap(u"<%s %s, handle %d>" % (
- conn_type, space.type(self).getname(space), handle))
-
- def descr_repr(self, space):
- raise NotImplementedError
-
- def _check_readable(self, space):
- if not self.flags & READABLE:
- raise oefmt(space.w_IOError, "connection is write-only")
- def _check_writable(self, space):
- if not self.flags & WRITABLE:
- raise oefmt(space.w_IOError, "connection is read-only")
-
- @unwrap_spec(offset='index', size='index')
- def send_bytes(self, space, w_buf, offset=0, size=PY_SSIZE_T_MIN):
- buf = space.getarg_w('s*', w_buf).as_str()
- length = len(buf)
- self._check_writable(space)
- if offset < 0:
- raise oefmt(space.w_ValueError, "offset is negative")
- if length < offset:
- raise oefmt(space.w_ValueError, "buffer length < offset")
-
- if size == PY_SSIZE_T_MIN:
- size = length - offset
- elif size < 0:
- raise oefmt(space.w_ValueError, "size is negative")
- elif offset + size > length:
- raise oefmt(space.w_ValueError, "buffer length > offset + size")
-
- self.do_send_string(space, buf, offset, size)
-
- @unwrap_spec(maxlength='index')
- def recv_bytes(self, space, maxlength=PY_SSIZE_T_MAX):
- self._check_readable(space)
- if maxlength < 0:
- raise oefmt(space.w_ValueError, "maxlength < 0")
-
- res, newbuf = self.do_recv_string(
- space, self.BUFFER_SIZE, maxlength)
- try:
- if newbuf:
- return space.newbytes(rffi.charpsize2str(newbuf, res))
- else:
- return space.newbytes(rffi.charpsize2str(self.buffer, res))
- finally:
- if newbuf:
- rffi.free_charp(newbuf)
-
- @unwrap_spec(offset='index')
- def recv_bytes_into(self, space, w_buffer, offset=0):
- rwbuffer = space.writebuf_w(w_buffer)
- length = rwbuffer.getlength()
-
- res, newbuf = self.do_recv_string(
- space, length - offset, PY_SSIZE_T_MAX)
- try:
- if newbuf:
- raise BufferTooShort(space, space.newbytes(
- rffi.charpsize2str(newbuf, res)))
- rwbuffer.setslice(offset, rffi.charpsize2str(self.buffer, res))
- finally:
- if newbuf:
- rffi.free_charp(newbuf)
-
- return space.wrap(res)
-
- def send(self, space, w_obj):
- self._check_writable(space)
-
- w_picklemodule = space.fromcache(State).w_picklemodule
- w_protocol = space.getattr(
- w_picklemodule, space.wrap("HIGHEST_PROTOCOL"))
- w_pickled = space.call_method(
- w_picklemodule, "dumps", w_obj, w_protocol)
-
- buf = space.str_w(w_pickled)
- self.do_send_string(space, buf, 0, len(buf))
-
- def recv(self, space):
- self._check_readable(space)
-
- res, newbuf = self.do_recv_string(
- space, self.BUFFER_SIZE, PY_SSIZE_T_MAX)
- try:
- if newbuf:
- w_received = space.newbytes(rffi.charpsize2str(newbuf, res))
- else:
- w_received = space.newbytes(rffi.charpsize2str(self.buffer,
res))
- finally:
- if newbuf:
- rffi.free_charp(newbuf)
-
- w_builtins = space.getbuiltinmodule('builtins')
- w_picklemodule = space.fromcache(State).w_picklemodule
- w_unpickled = space.call_method(
- w_picklemodule, "loads", w_received)
-
- return w_unpickled
-
- @unwrap_spec(w_timeout=WrappedDefault(0.0))
- def poll(self, space, w_timeout):
- self._check_readable(space)
- if space.is_w(w_timeout, space.w_None):
- timeout = -1.0 # block forever
- else:
- timeout = space.float_w(w_timeout)
- if timeout < 0.0:
- timeout = 0.0
- return space.newbool(self.do_poll(space, timeout))
-
-W_BaseConnection.typedef = TypeDef(
- 'BaseConnection',
- __repr__ = interpindirect2app(W_BaseConnection.descr_repr),
- closed = GetSetProperty(W_BaseConnection.closed_get),
- readable = GetSetProperty(W_BaseConnection.readable_get),
- writable = GetSetProperty(W_BaseConnection.writable_get),
-
- send_bytes = interp2app(W_BaseConnection.send_bytes),
- recv_bytes = interp2app(W_BaseConnection.recv_bytes),
- recv_bytes_into = interp2app(W_BaseConnection.recv_bytes_into),
- send = interp2app(W_BaseConnection.send),
- recv = interp2app(W_BaseConnection.recv),
- poll = interp2app(W_BaseConnection.poll),
- close = interp2app(W_BaseConnection.close),
- )
-
-class W_FileConnection(W_BaseConnection):
- INVALID_HANDLE_VALUE = -1
- fd = INVALID_HANDLE_VALUE
-
- if sys.platform == 'win32':
- def WRITE(self, data):
- from rpython.rlib._rsocket_rffi import send, geterrno
- length = send(self.fd, data, len(data), 0)
- if length < 0:
- raise WindowsError(geterrno(), "send")
- return length
- def READ(self, size):
- from rpython.rlib._rsocket_rffi import socketrecv, geterrno
- with rffi.scoped_alloc_buffer(size) as buf:
- length = socketrecv(self.fd, buf.raw, buf.size, 0)
- if length < 0:
- raise WindowsError(geterrno(), "recv")
- return buf.str(length)
- def CLOSE(self):
- from rpython.rlib._rsocket_rffi import socketclose
- socketclose(self.fd)
- else:
- def WRITE(self, data):
- import os
- return os.write(self.fd, data)
- def READ(self, length):
- import os
- return os.read(self.fd, length)
- def CLOSE(self):
- import os
- try:
- os.close(self.fd)
- except OSError:
- pass
-
- def __init__(self, space, fd, flags):
- if fd == self.INVALID_HANDLE_VALUE or fd < 0:
- raise oefmt(space.w_IOError, "invalid handle %d", fd)
- W_BaseConnection.__init__(self, space, flags)
- self.fd = fd
-
- @unwrap_spec(fd=int, readable=bool, writable=bool)
- def descr_new_file(space, w_subtype, fd, readable=True, writable=True):
- flags = (readable and READABLE) | (writable and WRITABLE)
-
- self = space.allocate_instance(W_FileConnection, w_subtype)
- W_FileConnection.__init__(self, space, fd, flags)
- return space.wrap(self)
-
- def descr_repr(self, space):
- return self._repr(space, self.fd)
-
- def fileno(self, space):
- return space.wrap(self.fd)
-
- def is_valid(self):
- return self.fd != self.INVALID_HANDLE_VALUE
-
- def do_close(self):
- if self.is_valid():
- self.CLOSE()
- self.fd = self.INVALID_HANDLE_VALUE
-
- def do_send_string(self, space, buf, offset, size):
- # Since str2charp copies the buf anyway, always combine the
- # "header" and the "body" of the message and send them at once.
- message = lltype.malloc(rffi.CCHARP.TO, size + 4, flavor='raw')
- try:
- length = rffi.r_uint(rsocket.htonl(
- rffi.cast(lltype.Unsigned, size)))
- rffi.cast(rffi.UINTP, message)[0] = length
- i = size - 1
- while i >= 0:
- message[4 + i] = buf[offset + i]
- i -= 1
- self._sendall(space, message, size + 4)
- finally:
- lltype.free(message, flavor='raw')
-
- def do_recv_string(self, space, buflength, maxlength):
- with lltype.scoped_alloc(rffi.CArrayPtr(rffi.UINT).TO, 1) as
length_ptr:
- self._recvall(space, rffi.cast(rffi.CCHARP, length_ptr), 4)
- length = intmask(rsocket.ntohl(
- rffi.cast(lltype.Unsigned, length_ptr[0])))
- if length > maxlength: # bad message, close connection
- self.flags &= ~READABLE
- if self.flags == 0:
- self.close()
- raise oefmt(space.w_IOError, "bad message length")
-
- if length <= buflength:
- self._recvall(space, self.buffer, length)
- return length, lltype.nullptr(rffi.CCHARP.TO)
- else:
- newbuf = lltype.malloc(rffi.CCHARP.TO, length, flavor='raw')
- self._recvall(space, newbuf, length)
- return length, newbuf
-
- def _sendall(self, space, message, size):
- while size > 0:
- # XXX inefficient
- data = rffi.charpsize2str(message, size)
- try:
- count = self.WRITE(data)
- except OSError as e:
- if e.errno == EINTR:
- space.getexecutioncontext().checksignals()
- continue
- raise wrap_oserror(space, e)
- size -= count
- message = rffi.ptradd(message, count)
-
- def _recvall(self, space, buf, length):
- length = intmask(length)
- remaining = length
- while remaining > 0:
- try:
- data = self.READ(remaining)
- except OSError as e:
- if e.errno == EINTR:
- space.getexecutioncontext().checksignals()
- continue
- raise wrap_oserror(space, e)
- count = len(data)
- if count == 0:
- if remaining == length:
- raise OperationError(space.w_EOFError, space.w_None)
- else:
- raise oefmt(space.w_IOError,
- "got end of file during message")
- # XXX inefficient
- for i in range(count):
- buf[i] = data[i]
- remaining -= count
- buf = rffi.ptradd(buf, count)
-
- if sys.platform == 'win32':
- def _check_fd(self):
- return self.fd >= 0
- else:
- def _check_fd(self):
- return self.fd >= 0 and self.fd < rpoll.FD_SETSIZE
-
- def do_poll(self, space, timeout):
- if not self._check_fd():
- raise oefmt(space.w_IOError, "handle out of range in select()")
- r, w, e = rpoll.select([self.fd], [], [], timeout, handle_eintr=True)
- return bool(r)
-
-W_FileConnection.typedef = TypeDef(
- '_multiprocessing.Connection', W_BaseConnection.typedef,
- __new__ = interp2app(W_FileConnection.descr_new_file.im_func),
- fileno = interp2app(W_FileConnection.fileno),
-)
-
-class W_PipeConnection(W_BaseConnection):
- if sys.platform == 'win32':
- from rpython.rlib.rwin32 import INVALID_HANDLE_VALUE
-
- def __init__(self, space, handle, flags):
- W_BaseConnection.__init__(self, space, flags)
- self.handle = handle
-
- @unwrap_spec(readable=bool, writable=bool)
- def descr_new_pipe(space, w_subtype, w_handle, readable=True,
- writable=True):
- from pypy.module._multiprocessing.interp_win32 import handle_w
- handle = handle_w(space, w_handle)
- flags = (readable and READABLE) | (writable and WRITABLE)
-
- self = space.allocate_instance(W_PipeConnection, w_subtype)
- W_PipeConnection.__init__(self, space, handle, flags)
- return space.wrap(self)
-
- def descr_repr(self, space):
- return self._repr(space, rffi.cast(rffi.INTPTR_T, self.handle))
-
- def is_valid(self):
- return self.handle != self.INVALID_HANDLE_VALUE
-
- def fileno(self, space):
- return w_handle(space, self.handle)
-
- def do_close(self):
- from rpython.rlib.rwin32 import CloseHandle
- if self.is_valid():
- CloseHandle(self.handle)
- self.handle = self.INVALID_HANDLE_VALUE
-
- def do_send_string(self, space, buf, offset, size):
- from pypy.module._multiprocessing.interp_win32 import (
- _WriteFile, ERROR_NO_SYSTEM_RESOURCES)
- from rpython.rlib import rwin32
-
- with rffi.scoped_view_charp(buf) as charp:
- written_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- try:
- result = _WriteFile(
- self.handle, rffi.ptradd(charp, offset),
- size, written_ptr, rffi.NULL)
-
- if (result == 0 and
- rwin32.GetLastError_saved() == ERROR_NO_SYSTEM_RESOURCES):
- raise oefmt(space.w_ValueError,
- "Cannot send %d bytes over connection", size)
- finally:
- lltype.free(written_ptr, flavor='raw')
-
- def do_recv_string(self, space, buflength, maxlength):
- from pypy.module._multiprocessing.interp_win32 import (
- _ReadFile, _PeekNamedPipe, ERROR_BROKEN_PIPE, ERROR_MORE_DATA)
- from rpython.rlib import rwin32
- from pypy.interpreter.error import wrap_windowserror
-
- read_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- left_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- try:
- result = _ReadFile(self.handle,
- self.buffer, min(self.BUFFER_SIZE, buflength),
- read_ptr, rffi.NULL)
- if result:
- return intmask(read_ptr[0]), lltype.nullptr(rffi.CCHARP.TO)
-
- err = rwin32.GetLastError_saved()
- if err == ERROR_BROKEN_PIPE:
- raise OperationError(space.w_EOFError, space.w_None)
- elif err != ERROR_MORE_DATA:
- raise wrap_windowserror(space, WindowsError(err, "_ReadFile"))
-
- # More data...
- if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
- lltype.nullptr(rwin32.LPDWORD.TO),
- lltype.nullptr(rwin32.LPDWORD.TO),
- left_ptr):
- raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
- length = intmask(read_ptr[0] + left_ptr[0])
- if length > maxlength: # bad message, close connection
- self.flags &= ~READABLE
- if self.flags == 0:
- self.close()
- raise oefmt(space.w_IOError, "bad message length")
-
- newbuf = lltype.malloc(rffi.CCHARP.TO, length + 1, flavor='raw')
- for i in range(read_ptr[0]):
- newbuf[i] = self.buffer[i]
-
- result = _ReadFile(self.handle,
- rffi.ptradd(newbuf, read_ptr[0]), left_ptr[0],
- read_ptr, rffi.NULL)
- if not result:
- rffi.free_charp(newbuf)
- raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
-
- assert read_ptr[0] == left_ptr[0]
- return length, newbuf
- finally:
- lltype.free(read_ptr, flavor='raw')
- lltype.free(left_ptr, flavor='raw')
-
- def do_poll(self, space, timeout):
- from pypy.module._multiprocessing.interp_win32 import (
- _PeekNamedPipe, _GetTickCount, _Sleep)
- from rpython.rlib import rwin32
- from pypy.interpreter.error import wrap_windowserror
- bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- try:
- if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
- lltype.nullptr(rwin32.LPDWORD.TO),
- bytes_ptr,
- lltype.nullptr(rwin32.LPDWORD.TO)):
- raise wrap_windowserror(space, rwin32.lastSavedWindowsError())
- bytes = bytes_ptr[0]
- finally:
- lltype.free(bytes_ptr, flavor='raw')
-
- if timeout == 0.0:
- return bytes > 0
-
- block = timeout < 0
- if not block:
- # XXX does not check for overflow
- deadline = intmask(_GetTickCount()) + int(1000 * timeout + 0.5)
- else:
- deadline = 0
-
- _Sleep(0)
-
- delay = 1
- while True:
- bytes_ptr = lltype.malloc(rffi.CArrayPtr(rwin32.DWORD).TO, 1,
- flavor='raw')
- try:
- if not _PeekNamedPipe(self.handle, rffi.NULL, 0,
- lltype.nullptr(rwin32.LPDWORD.TO),
- bytes_ptr,
- lltype.nullptr(rwin32.LPDWORD.TO)):
- raise wrap_windowserror(space,
- rwin32.lastSavedWindowsError())
- bytes = bytes_ptr[0]
- finally:
- lltype.free(bytes_ptr, flavor='raw')
-
- if bytes > 0:
- return True
-
- if not block:
- now = intmask(_GetTickCount())
- if now > deadline:
- return False
- diff = deadline - now
- if delay > diff:
- delay = diff
- else:
- delay += 1
-
- if delay >= 20:
- delay = 20
- _Sleep(delay)
-
- # check for signals
- # PyErr_CheckSignals()
-
-if sys.platform == 'win32':
- W_PipeConnection.typedef = TypeDef(
- '_multiprocessing.PipeConnection', W_BaseConnection.typedef,
- __new__ = interp2app(W_PipeConnection.descr_new_pipe.im_func),
- fileno = interp2app(W_PipeConnection.fileno),
- )
diff --git a/pypy/module/_multiprocessing/interp_semaphore.py
b/pypy/module/_multiprocessing/interp_semaphore.py
--- a/pypy/module/_multiprocessing/interp_semaphore.py
+++ b/pypy/module/_multiprocessing/interp_semaphore.py
@@ -13,9 +13,9 @@
from pypy.interpreter.error import oefmt, wrap_oserror
from pypy.interpreter.gateway import interp2app, unwrap_spec
from pypy.interpreter.typedef import GetSetProperty, TypeDef
-from pypy.module._multiprocessing.interp_connection import w_handle
RECURSIVE_MUTEX, SEMAPHORE = range(2)
+sys_platform = sys.platform
if sys.platform == 'win32':
from rpython.rlib import rwin32
@@ -216,9 +216,10 @@
def semaphore_unlink(space, w_name):
name = space.str_w(w_name)
- res = _sem_unlink(name)
- if res < 0:
- raise oefmt(space.w_OSError, "sem unlink failed with errno: %d",
rposix.get_saved_errno())
+ try:
+ sem_unlink(name)
+ except OSError as e:
+ raise wrap_oserror(space, e)
class CounterState:
def __init__(self, space):
@@ -331,6 +332,11 @@
rgc.add_memory_pressure(SEM_T_SIZE)
return sem
+ def reopen_semaphore(name):
+ sem = sem_open(name, 0, 0600, 0)
+ rgc.add_memory_pressure(SEM_T_SIZE)
+ return sem
+
def delete_semaphore(handle):
_sem_close_no_errno(handle)
@@ -454,7 +460,8 @@
return space.newint(self.maxvalue)
def handle_get(self, space):
- return w_handle(space, self.handle)
+ h = rffi.cast(rffi.INTPTR_T, self.handle)
+ return space.wrap(h)
def get_count(self, space):
return space.wrap(self.count)
@@ -521,8 +528,18 @@
@unwrap_spec(kind=int, maxvalue=int)
def rebuild(space, w_cls, w_handle, kind, maxvalue, w_name):
name = space.str_or_None_w(w_name)
+ #
+ if sys_platform != 'win32' and name is not None:
+ # like CPython, in this case ignore 'w_handle'
+ try:
+ handle = reopen_semaphore(name)
+ except OSError as e:
+ raise wrap_oserror(space, e)
+ else:
+ handle = handle_w(space, w_handle)
+ #
self = space.allocate_instance(W_SemLock, w_cls)
- self.__init__(space, handle_w(space, w_handle), kind, maxvalue, name)
+ self.__init__(space, handle, kind, maxvalue, name)
return space.wrap(self)
def enter(self, space):
diff --git a/pypy/module/_multiprocessing/test/test_connection.py
b/pypy/module/_multiprocessing/test/test_connection.py
deleted file mode 100644
--- a/pypy/module/_multiprocessing/test/test_connection.py
+++ /dev/null
@@ -1,200 +0,0 @@
-import py
-import sys
-from pypy.interpreter.gateway import interp2app, W_Root
-
-class TestImport:
- def test_simple(self):
- from pypy.module._multiprocessing import interp_connection
- from pypy.module._multiprocessing import interp_semaphore
-
-class AppTestBufferTooShort:
- spaceconfig = {'usemodules': ['_multiprocessing', 'thread', 'signal',
- 'select', 'struct', 'binascii',
- '_posixsubprocess']}
- if sys.platform == 'win32':
- spaceconfig['usemodules'].append('_rawffi')
- else:
- spaceconfig['usemodules'].append('fcntl')
-
-
- def setup_class(cls):
- if cls.runappdirect:
- def raiseBufferTooShort(self, data):
- import multiprocessing
- raise multiprocessing.BufferTooShort(data)
- cls.w_raiseBufferTooShort = raiseBufferTooShort
- else:
- from pypy.module._multiprocessing import interp_connection
- def raiseBufferTooShort(space, w_data):
- raise interp_connection.BufferTooShort(space, w_data)
- cls.w_raiseBufferTooShort = cls.space.wrap(
- interp2app(raiseBufferTooShort))
-
- def test_exception(self):
- import multiprocessing
- try:
- self.raiseBufferTooShort("data")
- except multiprocessing.BufferTooShort as e:
- assert isinstance(e, multiprocessing.ProcessError)
- assert e.args == ("data",)
-
-class BaseConnectionTest(object):
- def test_connection(self):
- rhandle, whandle = self.make_pair()
-
- whandle.send_bytes(b"abc")
- assert rhandle.recv_bytes(100) == b"abc"
-
- obj = [1, 2.0, "hello"]
- whandle.send(obj)
- obj2 = rhandle.recv()
- assert obj == obj2
-
- def test_poll(self):
- rhandle, whandle = self.make_pair()
-
- assert rhandle.poll() == False
- assert rhandle.poll(1) == False
- whandle.send(1)
- import time; time.sleep(0.1) # give it time to arrive :-)
- assert rhandle.poll() == True
- assert rhandle.poll(None) == True
- assert rhandle.recv() == 1
- assert rhandle.poll() == False
- raises(IOError, whandle.poll)
-
- def test_read_into(self):
- import array, multiprocessing
- rhandle, whandle = self.make_pair()
-
- obj = [1, 2.0, "hello"]
- whandle.send(obj)
- buffer = array.array('b', [0]*10)
- raises(multiprocessing.BufferTooShort, rhandle.recv_bytes_into, buffer)
- assert rhandle.readable
-
-class AppTestWinpipeConnection(BaseConnectionTest):
- spaceconfig = {
- "usemodules": [
- '_multiprocessing', 'thread', 'signal', 'struct', 'array',
- 'itertools', '_socket', 'binascii',
- ]
- }
- if sys.platform == 'win32':
- spaceconfig['usemodules'].append('_rawffi')
-
- def setup_class(cls):
- if sys.platform != "win32":
- py.test.skip("win32 only")
-
- if not cls.runappdirect:
- space = cls.space
- # stubs for some modules,
- # just for multiprocessing to import correctly on Windows
- w_modules = space.sys.get('modules')
- space.setitem(w_modules, space.wrap('msvcrt'), space.sys)
- else:
- import _multiprocessing
-
- def w_make_pair(self):
- import multiprocessing
-
- return multiprocessing.Pipe(duplex=False)
-
-
-class AppTestSocketConnection(BaseConnectionTest):
- spaceconfig = {
- "usemodules": [
- '_multiprocessing', 'thread', 'signal', 'struct', 'array',
- '_socket', 'binascii', 'select', '_posixsubprocess']
- }
- if sys.platform == 'win32':
- spaceconfig['usemodules'].append('_rawffi')
- else:
- spaceconfig['usemodules'].append('fcntl')
-
- def setup_class(cls):
- cls.w_connections = cls.space.newlist([])
-
- def w_socketpair(self):
- "A socket.socketpair() that works on Windows"
- import errno
- import socket
-
- serverSocket = socket.socket()
- serverSocket.bind(('127.0.0.1', 0))
- serverSocket.listen(1)
-
- client = socket.socket()
- client.setblocking(False)
- try:
- client.connect(('127.0.0.1', serverSocket.getsockname()[1]))
- except socket.error as e:
- assert e.args[0] in (errno.EINPROGRESS, errno.EWOULDBLOCK)
- server, addr = serverSocket.accept()
-
- # keep sockets alive during the test
- self.connections.append(server)
- self.connections.append(client)
-
- return server.fileno(), client.fileno()
-
- def w_make_pair(self):
- import _multiprocessing
-
- fd1, fd2 = self.socketpair()
- rhandle = _multiprocessing.Connection(fd1, writable=False)
- whandle = _multiprocessing.Connection(fd2, readable=False)
- self.connections.append(rhandle)
- self.connections.append(whandle)
- return rhandle, whandle
-
- def teardown_method(self, func):
- # Work hard to close all sockets and connections now!
- # since the fd is probably already closed, another unrelated
- # part of the program will probably reuse it;
- # And any object forgotten here will close it on destruction...
- try:
- w_connections = self.w_connections
- except AttributeError:
- return
- space = self.space
- for c in space.unpackiterable(w_connections):
- if isinstance(c, W_Root):
- space.call_method(c, "close")
- else:
- c.close()
- space.delslice(w_connections, space.wrap(0), space.wrap(100))
-
- def test_bad_fd(self):
- import _multiprocessing
-
- raises(IOError, _multiprocessing.Connection, -1)
- raises(IOError, _multiprocessing.Connection, -15)
-
- def test_byte_order(self):
- import socket
- if not 'fromfd' in dir(socket):
- skip('No fromfd in socket')
- # The exact format of net strings (length in network byte
- # order) is important for interoperation with others
- # implementations.
- rhandle, whandle = self.make_pair()
- whandle.send_bytes(b"abc")
- whandle.send_bytes(b"defg")
- sock = socket.fromfd(rhandle.fileno(),
- socket.AF_INET, socket.SOCK_STREAM)
- data1 = sock.recv(7)
- assert data1 == b'\x00\x00\x00\x03abc'
- data2 = sock.recv(8)
- assert data2 == b'\x00\x00\x00\x04defg'
-
- def test_repr(self):
- import _multiprocessing, os
- fd = os.dup(1) # closed by Connection.__del__
- c = _multiprocessing.Connection(fd)
- assert repr(c) == '<read-write Connection, handle %d>' % fd
- if hasattr(_multiprocessing, 'PipeConnection'):
- fd = os.dup(1) # closed by PipeConnection.__del__
- c = _multiprocessing.PipeConnection(fd)
- assert repr(c) == '<read-write PipeConnection, handle %d>' % fd
diff --git a/pypy/module/_multiprocessing/test/test_semaphore.py
b/pypy/module/_multiprocessing/test/test_semaphore.py
--- a/pypy/module/_multiprocessing/test/test_semaphore.py
+++ b/pypy/module/_multiprocessing/test/test_semaphore.py
@@ -22,10 +22,13 @@
@py.test.mark.skipif("sys.platform == 'win32'")
def test_sem_unlink(self):
from _multiprocessing import sem_unlink
+ import errno
try:
sem_unlink("non-existent")
- except OSError:
- pass
+ except OSError as e:
+ assert e.errno in (errno.ENOENT, errno.EINVAL)
+ else:
+ assert 0, "should have raised"
def test_semaphore(self):
from _multiprocessing import SemLock
@@ -99,14 +102,20 @@
assert res == False
def test_semaphore_rebuild(self):
- from _multiprocessing import SemLock
+ from _multiprocessing import SemLock, sem_unlink
kind = self.SEMAPHORE
value = 1
maxvalue = 1
- sem = SemLock(kind, value, maxvalue, "4", unlink=True)
-
- sem2 = SemLock._rebuild(sem.handle, kind, value, "10")
- assert sem.handle == sem2.handle
+ sem = SemLock(kind, value, maxvalue, "4.2", unlink=False)
+ try:
+ sem2 = SemLock._rebuild(-1, kind, value, "4.2")
+ #assert sem.handle != sem2.handle---even though they come
+ # from different calls to sem_open(), on Linux at least,
+ # they are the same pointer
+ sem2 = SemLock._rebuild(sem.handle, kind, value, None)
+ assert sem.handle == sem2.handle
+ finally:
+ sem_unlink("4.2")
def test_semaphore_contextmanager(self):
from _multiprocessing import SemLock
diff --git a/pypy/module/_weakref/interp__weakref.py
b/pypy/module/_weakref/interp__weakref.py
--- a/pypy/module/_weakref/interp__weakref.py
+++ b/pypy/module/_weakref/interp__weakref.py
@@ -194,6 +194,15 @@
W_WeakrefBase.__init__(self, space, w_obj, w_callable)
self.w_hash = None
+ def _cleanup_(self):
+ # When a prebuilt weakref is frozen inside a translation, if
+ # this weakref has got an already-cached w_hash, then throw it
+ # away. That's because the hash value will change after
+ # translation. It will be recomputed the first time we ask for
+ # it. Note that such a frozen weakref, if not dead, will point
+ # to a frozen object, so it will never die.
+ self.w_hash = None
+
def descr__init__weakref(self, space, w_obj, w_callable=None,
__args__=None):
if __args__.arguments_w:
diff --git a/pypy/module/array/interp_array.py
b/pypy/module/array/interp_array.py
--- a/pypy/module/array/interp_array.py
+++ b/pypy/module/array/interp_array.py
@@ -141,12 +141,120 @@
hints={'nolength': True}))
class W_ArrayBase(W_Root):
- _attrs_ = ('space', 'len', 'allocated', '_lifeline_') # no buffer
+ _attrs_ = ('space', 'len', 'allocated', '_lifeline_', '_buffer')
def __init__(self, space):
self.space = space
self.len = 0
self.allocated = 0
+ self._buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+ @rgc.must_be_light_finalizer
+ def __del__(self):
+ if self._buffer:
+ lltype.free(self._buffer, flavor='raw')
+
+ def setlen(self, size, zero=False, overallocate=True):
+ if size > 0:
+ if size > self.allocated or size < self.allocated / 2:
+ if overallocate:
+ if size < 9:
+ some = 3
+ else:
+ some = 6
+ some += size >> 3
+ else:
+ some = 0
+ self.allocated = size + some
+ byte_size = self.allocated * self.itemsize
+ if zero:
+ new_buffer = lltype.malloc(
+ rffi.CCHARP.TO, byte_size, flavor='raw',
+ add_memory_pressure=True, zero=True)
+ else:
+ new_buffer = lltype.malloc(
+ rffi.CCHARP.TO, byte_size, flavor='raw',
+ add_memory_pressure=True)
+ copy_bytes = min(size, self.len) * self.itemsize
+ rffi.c_memcpy(rffi.cast(rffi.VOIDP, new_buffer),
+ rffi.cast(rffi.VOIDP, self._buffer),
+ copy_bytes)
+ else:
+ self.len = size
+ return
+ else:
+ assert size == 0
+ self.allocated = 0
+ new_buffer = lltype.nullptr(rffi.CCHARP.TO)
+
+ if self._buffer:
+ lltype.free(self._buffer, flavor='raw')
+ self._buffer = new_buffer
+ self.len = size
+
+ def _fromiterable(self, w_seq):
+ # used by fromsequence().
+ # a more careful case if w_seq happens to be a very large
+ # iterable: don't copy the items into some intermediate list
+ w_iterator = self.space.iter(w_seq)
+ tp = self.space.type(w_iterator)
+ while True:
+ unpack_driver.jit_merge_point(selfclass=self.__class__,
+ tp=tp, self=self,
+ w_iterator=w_iterator)
+ space = self.space
+ try:
+ w_item = space.next(w_iterator)
+ except OperationError as e:
+ if not e.match(space, space.w_StopIteration):
+ raise
+ break # done
+ self.descr_append(space, w_item)
+
+ def _charbuf_start(self):
+ return self._buffer
+
+ def _buffer_as_unsigned(self):
+ return rffi.cast(lltype.Unsigned, self._buffer)
+
+ def _charbuf_stop(self):
+ keepalive_until_here(self)
+
+ def delitem(self, space, i, j):
+ if i < 0:
+ i += self.len
+ if i < 0:
+ i = 0
+ if j < 0:
+ j += self.len
+ if j < 0:
+ j = 0
+ if j > self.len:
+ j = self.len
+ if i >= j:
+ return None
+ oldbuffer = self._buffer
+ self._buffer = lltype.malloc(rffi.CCHARP.TO,
+ (self.len - (j - i)) * self.itemsize, flavor='raw',
+ add_memory_pressure=True)
+ if i:
+ rffi.c_memcpy(
+ rffi.cast(rffi.VOIDP, self._buffer),
+ rffi.cast(rffi.VOIDP, oldbuffer),
+ i * self.itemsize
+ )
+ if j < self.len:
+ rffi.c_memcpy(
+ rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+ i * self.itemsize)),
+ rffi.cast(rffi.VOIDP, rffi.ptradd(oldbuffer,
+ j * self.itemsize)),
+ (self.len - j) * self.itemsize
+ )
+ self.len -= j - i
+ self.allocated = self.len
+ if oldbuffer:
+ lltype.free(oldbuffer, flavor='raw')
def buffer_w_ex(self, space, flags):
return ArrayBuffer(self, False), self.typecode, self.itemsize
@@ -170,14 +278,24 @@
Return number of occurrences of x in the array.
"""
- raise NotImplementedError
+ cnt = 0
+ for i in range(self.len):
+ # XXX jitdriver
+ w_item = self.w_getitem(space, i)
+ if space.is_true(space.eq(w_item, w_val)):
+ cnt += 1
+ return space.wrap(cnt)
def descr_index(self, space, w_x):
""" index(x)
Return index of first occurrence of x in the array.
"""
- raise NotImplementedError
+ for i in range(self.len):
+ w_item = self.w_getitem(space, i)
+ if space.is_true(space.eq(w_item, w_x)):
+ return space.wrap(i)
+ raise oefmt(space.w_ValueError, "array.index(x): x not in list")
def descr_reverse(self, space):
""" reverse()
@@ -191,7 +309,8 @@
Remove the first occurrence of x in the array.
"""
- raise NotImplementedError
+ w_idx = self.descr_index(space, w_val)
+ self.descr_pop(space, space.int_w(w_idx))
@unwrap_spec(i=int)
def descr_pop(self, space, i=-1):
@@ -507,16 +626,102 @@
return space.newseqiter(self)
def descr_add(self, space, w_other):
- raise NotImplementedError
+ if (not isinstance(w_other, W_ArrayBase)
+ or w_other.typecode != self.typecode):
+ return space.w_NotImplemented
+ a = self.constructor(space)
+ a.setlen(self.len + w_other.len, overallocate=False)
+ if self.len:
+ rffi.c_memcpy(
+ rffi.cast(rffi.VOIDP, a._buffer),
+ rffi.cast(rffi.VOIDP, self._buffer),
+ self.len * self.itemsize
+ )
+ if w_other.len:
+ rffi.c_memcpy(
+ rffi.cast(rffi.VOIDP, rffi.ptradd(a._buffer,
+ self.len * self.itemsize)),
+ rffi.cast(rffi.VOIDP, w_other._buffer),
+ w_other.len * self.itemsize
+ )
+ keepalive_until_here(self)
+ keepalive_until_here(a)
+ return a
def descr_inplace_add(self, space, w_other):
- raise NotImplementedError
+ if (not isinstance(w_other, W_ArrayBase)
+ or w_other.typecode != self.typecode):
+ return space.w_NotImplemented
+ oldlen = self.len
+ otherlen = w_other.len
+ self.setlen(oldlen + otherlen)
+ if otherlen:
+ rffi.c_memcpy(
+ rffi.cast(rffi.VOIDP, rffi.ptradd(self._buffer,
+ oldlen * self.itemsize)),
+ rffi.cast(rffi.VOIDP, w_other._buffer),
+ otherlen * self.itemsize
+ )
+ keepalive_until_here(self)
+ keepalive_until_here(w_other)
+ return self
+
+ def _mul_helper(self, space, w_repeat, is_inplace):
+ try:
+ repeat = space.getindex_w(w_repeat, space.w_OverflowError)
+ except OperationError as e:
+ if e.match(space, space.w_TypeError):
+ return space.w_NotImplemented
+ raise
+ if is_inplace:
+ a = self
+ start = 1
+ else:
+ a = self.constructor(space)
+ start = 0
+ if repeat <= start:
+ if repeat <= 0:
+ a.setlen(0, overallocate=False)
+ return a
+ oldlen = self.len
+ try:
+ newlen = ovfcheck(oldlen * repeat)
+ except OverflowError:
+ raise MemoryError
+ #
+ srcbuf = self._buffer
+ srcsize = self.len * self.itemsize
+ for i in range(srcsize):
+ if srcbuf[i] != '\x00':
+ break
+ else:
+ # the source is entirely zero: initialize the target
+ # with zeroes too
+ a.setlen(newlen, zero=True, overallocate=False)
+ return a
+ #
+ a.setlen(newlen, overallocate=False)
+ srcbuf = self._buffer # reload this, in case self is a
+ if oldlen == 1:
+ self._repeat_single_item(a, start, repeat)
+ else:
+ dstbuf = a._buffer
+ if start == 1:
+ dstbuf = rffi.ptradd(dstbuf, srcsize)
+ for r in range(start, repeat):
+ rffi.c_memcpy(rffi.cast(rffi.VOIDP, dstbuf),
+ rffi.cast(rffi.VOIDP, srcbuf),
+ srcsize)
+ dstbuf = rffi.ptradd(dstbuf, srcsize)
+ keepalive_until_here(self)
+ keepalive_until_here(a)
+ return a
def descr_mul(self, space, w_repeat):
- raise NotImplementedError
+ return self._mul_helper(space, w_repeat, False)
def descr_inplace_mul(self, space, w_repeat):
- raise NotImplementedError
+ return self._mul_helper(space, w_repeat, True)
def descr_radd(self, space, w_other):
return self.descr_add(space, w_other)
@@ -600,6 +805,7 @@
self.itemtype = itemtype
self.bytes = rffi.sizeof(itemtype)
self.arraytype = lltype.Array(itemtype, hints={'nolength': True})
+ self.arrayptrtype = lltype.Ptr(self.arraytype)
self.unwrap, _, self.convert = unwrap.partition('.')
self.signed = signed
self.canoverflow = canoverflow
@@ -694,22 +900,21 @@
return self.array._charbuf_start()
+unpack_driver = jit.JitDriver(name='unpack_array',
+ greens=['selfclass', 'tp'],
+ reds=['self', 'w_iterator'])
+
def make_array(mytype):
W_ArrayBase = globals()['W_ArrayBase']
- unpack_driver = jit.JitDriver(name='unpack_array',
- greens=['tp'],
- reds=['self', 'w_iterator'])
-
class W_Array(W_ArrayBase):
itemsize = mytype.bytes
typecode = mytype.typecode
- _attrs_ = ('space', 'len', 'allocated', '_lifeline_', 'buffer')
+ _attrs_ = W_ArrayBase._attrs_
- def __init__(self, space):
- W_ArrayBase.__init__(self, space)
- self.buffer = lltype.nullptr(mytype.arraytype)
+ def get_buffer(self):
+ return rffi.cast(mytype.arrayptrtype, self._buffer)
def item_w(self, w_item):
space = self.space
@@ -765,46 +970,6 @@
self.space.wrap(msg))
return result
- @rgc.must_be_light_finalizer
- def __del__(self):
- if self.buffer:
- lltype.free(self.buffer, flavor='raw')
-
- def setlen(self, size, zero=False, overallocate=True):
- if size > 0:
- if size > self.allocated or size < self.allocated / 2:
- if overallocate:
- if size < 9:
- some = 3
- else:
- some = 6
- some += size >> 3
- else:
- some = 0
- self.allocated = size + some
- if zero:
- new_buffer = lltype.malloc(
- mytype.arraytype, self.allocated, flavor='raw',
- add_memory_pressure=True, zero=True)
- else:
- new_buffer = lltype.malloc(
- mytype.arraytype, self.allocated, flavor='raw',
- add_memory_pressure=True)
- for i in range(min(size, self.len)):
- new_buffer[i] = self.buffer[i]
- else:
- self.len = size
- return
- else:
- assert size == 0
- self.allocated = 0
- new_buffer = lltype.nullptr(mytype.arraytype)
-
- if self.buffer:
- lltype.free(self.buffer, flavor='raw')
- self.buffer = new_buffer
- self.len = size
-
def fromsequence(self, w_seq):
space = self.space
oldlen = self.len
@@ -820,20 +985,21 @@
if lst is not None:
self.setlen(oldlen + len(lst))
try:
- buf = self.buffer
+ buf = self.get_buffer()
for num in lst:
buf[newlen] = self.item_from_int_or_float(num)
newlen += 1
except OperationError:
self.setlen(newlen)
raise
+ keepalive_until_here(self)
return
# this is the common case: w_seq is a list or a tuple
lst_w = space.listview_no_unpack(w_seq)
if lst_w is not None:
self.setlen(oldlen + len(lst_w))
- buf = self.buffer
+ buf = self.get_buffer()
try:
for w_num in lst_w:
# note: self.item_w() might invoke arbitrary code.
@@ -844,30 +1010,14 @@
buf[newlen] = self.item_w(w_num)
newlen += 1
except OperationError:
- if buf == self.buffer:
+ if buf == self.get_buffer():
self.setlen(newlen)
raise
+ keepalive_until_here(self)
return
self._fromiterable(w_seq)
- def _fromiterable(self, w_seq):
- # a more careful case if w_seq happens to be a very large
- # iterable: don't copy the items into some intermediate list
- w_iterator = self.space.iter(w_seq)
- tp = self.space.type(w_iterator)
- while True:
- unpack_driver.jit_merge_point(tp=tp, self=self,
- w_iterator=w_iterator)
- space = self.space
- try:
- w_item = space.next(w_iterator)
- except OperationError as e:
- if not e.match(space, space.w_StopIteration):
- raise
- break # done
- self.descr_append(space, w_item)
-
def extend(self, w_iterable, accept_different_array=False):
space = self.space
if isinstance(w_iterable, W_Array):
@@ -875,11 +1025,14 @@
new = w_iterable.len
self.setlen(self.len + new)
i = 0
+ buf = self.get_buffer()
+ srcbuf = w_iterable.get_buffer()
while i < new:
if oldlen + i >= self.len:
self.setlen(oldlen + i + 1)
- self.buffer[oldlen + i] = w_iterable.buffer[i]
+ buf[oldlen + i] = srcbuf[i]
i += 1
+ keepalive_until_here(w_iterable)
self.setlen(oldlen + i)
elif (not accept_different_array
and isinstance(w_iterable, W_ArrayBase)):
@@ -888,17 +1041,9 @@
else:
self.fromsequence(w_iterable)
- def _charbuf_start(self):
- return rffi.cast(rffi.CCHARP, self.buffer)
-
- def _buffer_as_unsigned(self):
- return rffi.cast(lltype.Unsigned, self.buffer)
-
- def _charbuf_stop(self):
+ def w_getitem(self, space, idx):
+ item = self.get_buffer()[idx]
keepalive_until_here(self)
-
- def w_getitem(self, space, idx):
- item = self.buffer[idx]
if mytype.typecode in 'bBhHil':
item = rffi.cast(lltype.Signed, item)
elif mytype.typecode == 'f':
@@ -916,29 +1061,16 @@
x = self.item_w(w_x)
index = self.len
self.setlen(index + 1)
- self.buffer[index] = x
+ self.get_buffer()[index] = x
+ keepalive_until_here(self)
# List interface
- def descr_count(self, space, w_val):
- cnt = 0
- for i in range(self.len):
- # XXX jitdriver
- w_item = self.w_getitem(space, i)
- if space.is_true(space.eq(w_item, w_val)):
- cnt += 1
- return space.wrap(cnt)
-
- def descr_index(self, space, w_val):
- for i in range(self.len):
- w_item = self.w_getitem(space, i)
- if space.is_true(space.eq(w_item, w_val)):
- return space.wrap(i)
- raise oefmt(space.w_ValueError, "array.index(x): x not in list")
def descr_reverse(self, space):
- b = self.buffer
+ b = self.get_buffer()
for i in range(self.len / 2):
b[i], b[self.len - i - 1] = b[self.len - i - 1], b[i]
+ keepalive_until_here(self)
def descr_pop(self, space, i):
if i < 0:
@@ -946,16 +1078,14 @@
if i < 0 or i >= self.len:
raise oefmt(space.w_IndexError, "pop index out of range")
w_val = self.w_getitem(space, i)
+ b = self.get_buffer()
while i < self.len - 1:
- self.buffer[i] = self.buffer[i + 1]
+ b[i] = b[i + 1]
i += 1
+ keepalive_until_here(self)
self.setlen(self.len - 1)
return w_val
- def descr_remove(self, space, w_val):
- w_idx = self.descr_index(space, w_val)
- self.descr_pop(space, space.int_w(w_idx))
-
def descr_insert(self, space, idx, w_val):
if idx < 0:
idx += self.len
@@ -967,10 +1097,12 @@
val = self.item_w(w_val)
self.setlen(self.len + 1)
i = self.len - 1
+ b = self.get_buffer()
while i > idx:
- self.buffer[i] = self.buffer[i - 1]
+ b[i] = b[i - 1]
i -= 1
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
