Author: Armin Rigo <ar...@tunes.org>
Branch: cffi-static-callback-embedding
Changeset: r81585:7b81fa1c3fa9
Date: 2016-01-05 15:01 +0100
http://bitbucket.org/pypy/pypy/changeset/7b81fa1c3fa9/
Log: hg merge default (including the ec-keepalive branch)
diff --git a/lib-python/2.7/pickle.py b/lib-python/2.7/pickle.py
--- a/lib-python/2.7/pickle.py
+++ b/lib-python/2.7/pickle.py
@@ -1376,6 +1376,7 @@
def decode_long(data):
r"""Decode a long from a two's complement little-endian binary string.
+ This is overriden on PyPy by a RPython version that has linear complexity.
>>> decode_long('')
0L
@@ -1402,6 +1403,11 @@
n -= 1L << (nbytes * 8)
return n
+try:
+ from __pypy__ import decode_long
+except ImportError:
+ pass
+
# Shorthands
try:
diff --git a/lib_pypy/cPickle.py b/lib_pypy/cPickle.py
--- a/lib_pypy/cPickle.py
+++ b/lib_pypy/cPickle.py
@@ -559,6 +559,7 @@
def decode_long(data):
r"""Decode a long from a two's complement little-endian binary string.
+ This is overriden on PyPy by a RPython version that has linear complexity.
>>> decode_long('')
0L
@@ -592,6 +593,11 @@
n -= 1L << (nbytes << 3)
return n
+try:
+ from __pypy__ import decode_long
+except ImportError:
+ pass
+
def load(f):
return Unpickler(f).load()
diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst
--- a/pypy/doc/whatsnew-head.rst
+++ b/pypy/doc/whatsnew-head.rst
@@ -103,3 +103,9 @@
Fix the cryptic exception message when attempting to use extended slicing
in rpython. Was issue #2211.
+
+.. branch: ec-keepalive
+
+Optimize the case where, in a new C-created thread, we keep invoking
+short-running Python callbacks. (CFFI on CPython has a hack to achieve
+the same result.)
diff --git a/pypy/module/__pypy__/__init__.py b/pypy/module/__pypy__/__init__.py
--- a/pypy/module/__pypy__/__init__.py
+++ b/pypy/module/__pypy__/__init__.py
@@ -89,6 +89,7 @@
'set_code_callback' : 'interp_magic.set_code_callback',
'save_module_content_for_future_reload':
'interp_magic.save_module_content_for_future_reload',
+ 'decode_long' : 'interp_magic.decode_long',
}
if sys.platform == 'win32':
interpleveldefs['get_console_cp'] = 'interp_magic.get_console_cp'
diff --git a/pypy/module/__pypy__/interp_magic.py
b/pypy/module/__pypy__/interp_magic.py
--- a/pypy/module/__pypy__/interp_magic.py
+++ b/pypy/module/__pypy__/interp_magic.py
@@ -1,4 +1,4 @@
-from pypy.interpreter.error import OperationError, wrap_oserror
+from pypy.interpreter.error import OperationError, oefmt, wrap_oserror
from pypy.interpreter.gateway import unwrap_spec
from pypy.interpreter.pycode import CodeHookCache
from pypy.interpreter.pyframe import PyFrame
@@ -158,4 +158,13 @@
if space.is_none(w_callable):
cache._code_hook = None
else:
- cache._code_hook = w_callable
\ No newline at end of file
+ cache._code_hook = w_callable
+
+@unwrap_spec(string=str, byteorder=str, signed=int)
+def decode_long(space, string, byteorder='little', signed=1):
+ from rpython.rlib.rbigint import rbigint, InvalidEndiannessError
+ try:
+ result = rbigint.frombytes(string, byteorder, bool(signed))
+ except InvalidEndiannessError:
+ raise oefmt(space.w_ValueError, "invalid byteorder argument")
+ return space.newlong_from_rbigint(result)
diff --git a/pypy/module/__pypy__/test/test_magic.py
b/pypy/module/__pypy__/test/test_magic.py
--- a/pypy/module/__pypy__/test/test_magic.py
+++ b/pypy/module/__pypy__/test/test_magic.py
@@ -30,4 +30,20 @@
""" in d
finally:
__pypy__.set_code_callback(None)
- assert d['f'].__code__ in l
\ No newline at end of file
+ assert d['f'].__code__ in l
+
+ def test_decode_long(self):
+ from __pypy__ import decode_long
+ assert decode_long('') == 0
+ assert decode_long('\xff\x00') == 255
+ assert decode_long('\xff\x7f') == 32767
+ assert decode_long('\x00\xff') == -256
+ assert decode_long('\x00\x80') == -32768
+ assert decode_long('\x80') == -128
+ assert decode_long('\x7f') == 127
+ assert decode_long('\x55' * 97) == (1 << (97 * 8)) // 3
+ assert decode_long('\x00\x80', 'big') == 128
+ assert decode_long('\xff\x7f', 'little', False) == 32767
+ assert decode_long('\x00\x80', 'little', False) == 32768
+ assert decode_long('\x00\x80', 'little', True) == -32768
+ raises(ValueError, decode_long, '', 'foo')
diff --git a/pypy/module/posix/interp_posix.py
b/pypy/module/posix/interp_posix.py
--- a/pypy/module/posix/interp_posix.py
+++ b/pypy/module/posix/interp_posix.py
@@ -299,7 +299,7 @@
return build_stat_result(space, st)
def lstat(space, w_path):
- "Like stat(path), but do no follow symbolic links."
+ "Like stat(path), but do not follow symbolic links."
try:
st = dispatch_filename(rposix_stat.lstat)(space, w_path)
except OSError, e:
diff --git a/pypy/objspace/std/test/test_longobject.py
b/pypy/objspace/std/test/test_longobject.py
--- a/pypy/objspace/std/test/test_longobject.py
+++ b/pypy/objspace/std/test/test_longobject.py
@@ -358,3 +358,10 @@
assert 3L.__coerce__(4L) == (3L, 4L)
assert 3L.__coerce__(4) == (3, 4)
assert 3L.__coerce__(object()) == NotImplemented
+
+ def test_linear_long_base_16(self):
+ # never finishes if long(_, 16) is not linear-time
+ size = 100000
+ n = "a" * size
+ expected = (2 << (size * 4)) // 3
+ assert long(n, 16) == expected
diff --git a/rpython/rlib/rbigint.py b/rpython/rlib/rbigint.py
--- a/rpython/rlib/rbigint.py
+++ b/rpython/rlib/rbigint.py
@@ -2794,8 +2794,10 @@
def parse_digit_string(parser):
# helper for fromstr
+ base = parser.base
+ if (base & (base - 1)) == 0:
+ return parse_string_from_binary_base(parser)
a = rbigint()
- base = parser.base
digitmax = BASE_MAX[base]
tens, dig = 1, 0
while True:
@@ -2811,3 +2813,52 @@
tens *= base
a.sign *= parser.sign
return a
+
+def parse_string_from_binary_base(parser):
+ # The point to this routine is that it takes time linear in the number of
+ # string characters.
+ from rpython.rlib.rstring import ParseStringError
+
+ base = parser.base
+ if base == 2: bits_per_char = 1
+ elif base == 4: bits_per_char = 2
+ elif base == 8: bits_per_char = 3
+ elif base == 16: bits_per_char = 4
+ elif base == 32: bits_per_char = 5
+ else:
+ raise AssertionError
+
+ # n <- total number of bits needed, while moving 'parser' to the end
+ n = 0
+ while parser.next_digit() >= 0:
+ n += 1
+
+ # b <- number of Python digits needed, = ceiling(n/SHIFT). */
+ try:
+ b = ovfcheck(n * bits_per_char)
+ b = ovfcheck(b + (SHIFT - 1))
+ except OverflowError:
+ raise ParseStringError("long string too large to convert")
+ b = (b // SHIFT) or 1
+ z = rbigint([NULLDIGIT] * b, sign=parser.sign)
+
+ # Read string from right, and fill in long from left; i.e.,
+ # from least to most significant in both.
+ accum = _widen_digit(0)
+ bits_in_accum = 0
+ pdigit = 0
+ for _ in range(n):
+ k = parser.prev_digit()
+ accum |= _widen_digit(k) << bits_in_accum
+ bits_in_accum += bits_per_char
+ if bits_in_accum >= SHIFT:
+ z.setdigit(pdigit, accum)
+ pdigit += 1
+ assert pdigit <= b
+ accum >>= SHIFT
+ bits_in_accum -= SHIFT
+
+ if bits_in_accum:
+ z.setdigit(pdigit, accum)
+ z._normalize()
+ return z
diff --git a/rpython/rlib/rstring.py b/rpython/rlib/rstring.py
--- a/rpython/rlib/rstring.py
+++ b/rpython/rlib/rstring.py
@@ -485,6 +485,24 @@
else:
return -1
+ def prev_digit(self):
+ # After exhausting all n digits in next_digit(), you can walk them
+ # again in reverse order by calling prev_digit() exactly n times
+ i = self.i - 1
+ assert i >= 0
+ self.i = i
+ c = self.s[i]
+ digit = ord(c)
+ if '0' <= c <= '9':
+ digit -= ord('0')
+ elif 'A' <= c <= 'Z':
+ digit = (digit - ord('A')) + 10
+ elif 'a' <= c <= 'z':
+ digit = (digit - ord('a')) + 10
+ else:
+ raise AssertionError
+ return digit
+
# -------------- public API ---------------------------------
INIT_SIZE = 100 # XXX tweak
diff --git a/rpython/rlib/rthread.py b/rpython/rlib/rthread.py
--- a/rpython/rlib/rthread.py
+++ b/rpython/rlib/rthread.py
@@ -394,11 +394,13 @@
def _trace_tlref(gc, obj, callback, arg):
p = llmemory.NULL
+ llop.threadlocalref_acquire(lltype.Void)
while True:
p = llop.threadlocalref_enum(llmemory.Address, p)
if not p:
break
gc._trace_callback(callback, arg, p + offset)
+ llop.threadlocalref_release(lltype.Void)
_lambda_trace_tlref = lambda: _trace_tlref
TRACETLREF = lltype.GcStruct('TRACETLREF')
_tracetlref_obj = lltype.malloc(TRACETLREF, immortal=True)
@@ -407,9 +409,12 @@
def automatic_keepalive(config):
"""Returns True if translated with a GC that keeps alive
the set() value until the end of the thread. Returns False
- if you need to keep it alive yourself.
+ if you need to keep it alive yourself (but in that case, you
+ should also reset it to None before the thread finishes).
"""
- return config.translation.gctransformer == "framework"
+ return (config.translation.gctransformer == "framework" and
+ # see translator/c/src/threadlocal.c for the following line
+ (not _win32 or config.translation.shared))
tlfield_thread_ident = ThreadLocalField(lltype.Signed, "thread_ident",
@@ -418,7 +423,8 @@
loop_invariant=True)
tlfield_rpy_errno = ThreadLocalField(rffi.INT, "rpy_errno")
tlfield_alt_errno = ThreadLocalField(rffi.INT, "alt_errno")
-if sys.platform == "win32":
+_win32 = (sys.platform == "win32")
+if _win32:
from rpython.rlib import rwin32
tlfield_rpy_lasterror = ThreadLocalField(rwin32.DWORD, "rpy_lasterror")
tlfield_alt_lasterror = ThreadLocalField(rwin32.DWORD, "alt_lasterror")
diff --git a/rpython/rlib/test/test_rbigint.py
b/rpython/rlib/test/test_rbigint.py
--- a/rpython/rlib/test/test_rbigint.py
+++ b/rpython/rlib/test/test_rbigint.py
@@ -825,7 +825,19 @@
def __init__(self, base, sign, digits):
self.base = base
self.sign = sign
- self.next_digit = iter(digits + [-1]).next
+ self.i = 0
+ self._digits = digits
+ def next_digit(self):
+ i = self.i
+ if i == len(self._digits):
+ return -1
+ self.i = i + 1
+ return self._digits[i]
+ def prev_digit(self):
+ i = self.i - 1
+ assert i >= 0
+ self.i = i
+ return self._digits[i]
x = parse_digit_string(Parser(10, 1, [6]))
assert x.eq(rbigint.fromint(6))
x = parse_digit_string(Parser(10, 1, [6, 2, 3]))
@@ -847,6 +859,16 @@
x = parse_digit_string(Parser(7, -1, [0, 0, 0]))
assert x.tobool() is False
+ for base in [2, 4, 8, 16, 32]:
+ for inp in [[0], [1], [1, 0], [0, 1], [1, 0, 1], [1, 0, 0, 1],
+ [1, 0, 0, base-1, 0, 1], [base-1, 1, 0, 0, 0, 1, 0],
+ [base-1]]:
+ inp = inp * 97
+ x = parse_digit_string(Parser(base, -1, inp))
+ num = sum(inp[i] * (base ** (len(inp)-1-i))
+ for i in range(len(inp)))
+ assert x.eq(rbigint.fromlong(-num))
+
BASE = 2 ** SHIFT
diff --git a/rpython/rtyper/llinterp.py b/rpython/rtyper/llinterp.py
--- a/rpython/rtyper/llinterp.py
+++ b/rpython/rtyper/llinterp.py
@@ -950,6 +950,10 @@
return self.op_raw_load(RESTYPE, _address_of_thread_local(), offset)
op_threadlocalref_get.need_result_type = True
+ def op_threadlocalref_acquire(self, prev):
+ raise NotImplementedError
+ def op_threadlocalref_release(self, prev):
+ raise NotImplementedError
def op_threadlocalref_enum(self, prev):
raise NotImplementedError
diff --git a/rpython/rtyper/lltypesystem/lloperation.py
b/rpython/rtyper/lltypesystem/lloperation.py
--- a/rpython/rtyper/lltypesystem/lloperation.py
+++ b/rpython/rtyper/lltypesystem/lloperation.py
@@ -547,6 +547,8 @@
'threadlocalref_addr': LLOp(), # get (or make) addr of
tl
'threadlocalref_get': LLOp(sideeffects=False), # read field (no check)
+ 'threadlocalref_acquire': LLOp(), # lock for enum
+ 'threadlocalref_release': LLOp(), # lock for enum
'threadlocalref_enum': LLOp(sideeffects=False), # enum all
threadlocalrefs
# __________ debugging __________
diff --git a/rpython/translator/c/src/thread.h
b/rpython/translator/c/src/thread.h
--- a/rpython/translator/c/src/thread.h
+++ b/rpython/translator/c/src/thread.h
@@ -48,7 +48,7 @@
}
static inline void _RPyGilRelease(void) {
assert(RPY_FASTGIL_LOCKED(rpy_fastgil));
- rpy_fastgil = 0;
+ lock_release(&rpy_fastgil);
}
static inline long *_RPyFetchFastGil(void) {
return &rpy_fastgil;
diff --git a/rpython/translator/c/src/thread_nt.c
b/rpython/translator/c/src/thread_nt.c
--- a/rpython/translator/c/src/thread_nt.c
+++ b/rpython/translator/c/src/thread_nt.c
@@ -231,10 +231,19 @@
return (result != WAIT_TIMEOUT);
}
-#define mutex1_t mutex2_t
-#define mutex1_init mutex2_init
-#define mutex1_lock mutex2_lock
-#define mutex1_unlock mutex2_unlock
+typedef CRITICAL_SECTION mutex1_t;
+
+static inline void mutex1_init(mutex1_t *mutex) {
+ InitializeCriticalSection(mutex);
+}
+
+static inline void mutex1_lock(mutex1_t *mutex) {
+ EnterCriticalSection(mutex);
+}
+
+static inline void mutex1_unlock(mutex1_t *mutex) {
+ LeaveCriticalSection(mutex);
+}
//#define lock_test_and_set(ptr, value) see thread_nt.h
#define atomic_increment(ptr) InterlockedIncrement(ptr)
diff --git a/rpython/translator/c/src/thread_nt.h
b/rpython/translator/c/src/thread_nt.h
--- a/rpython/translator/c/src/thread_nt.h
+++ b/rpython/translator/c/src/thread_nt.h
@@ -38,3 +38,4 @@
#else
#define lock_test_and_set(ptr, value) InterlockedExchange(ptr, value)
#endif
+#define lock_release(ptr) (*((volatile long *)ptr) = 0)
diff --git a/rpython/translator/c/src/thread_pthread.h
b/rpython/translator/c/src/thread_pthread.h
--- a/rpython/translator/c/src/thread_pthread.h
+++ b/rpython/translator/c/src/thread_pthread.h
@@ -81,3 +81,4 @@
#define lock_test_and_set(ptr, value) __sync_lock_test_and_set(ptr, value)
+#define lock_release(ptr) __sync_lock_release(ptr)
diff --git a/rpython/translator/c/src/threadlocal.c
b/rpython/translator/c/src/threadlocal.c
--- a/rpython/translator/c/src/threadlocal.c
+++ b/rpython/translator/c/src/threadlocal.c
@@ -3,7 +3,27 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
+#include <assert.h>
#include "src/threadlocal.h"
+#include "src/thread.h"
+
+
+/* this is a spin-lock that must be acquired around each doubly-linked-list
+ manipulation (because such manipulations can occur without the GIL) */
+static long pypy_threadlocal_lock = 0;
+
+static int check_valid(void);
+
+void _RPython_ThreadLocals_Acquire(void) {
+ while (!lock_test_and_set(&pypy_threadlocal_lock, 1)) {
+ /* busy loop */
+ }
+ assert(check_valid());
+}
+void _RPython_ThreadLocals_Release(void) {
+ assert(check_valid());
+ lock_release(&pypy_threadlocal_lock);
+}
pthread_key_t pypy_threadlocal_key
@@ -18,6 +38,43 @@
&linkedlist_head, /* prev */
&linkedlist_head }; /* next */
+static int check_valid(void)
+{
+ struct pypy_threadlocal_s *prev, *cur;
+ prev = &linkedlist_head;
+ while (1) {
+ cur = prev->next;
+ assert(cur->prev == prev);
+ if (cur == &linkedlist_head)
+ break;
+ assert(cur->ready == 42);
+ assert(cur->next != cur);
+ prev = cur;
+ }
+ assert(cur->ready == -1);
+ return 1;
+}
+
+static void cleanup_after_fork(void)
+{
+ /* assume that at most one pypy_threadlocal_s survived, the current one */
+ struct pypy_threadlocal_s *cur;
+#ifdef USE___THREAD
+ cur = &pypy_threadlocal;
+#else
+ cur = (struct pypy_threadlocal_s *)_RPy_ThreadLocals_Get();
+#endif
+ if (cur && cur->ready == 42) {
+ cur->next = cur->prev = &linkedlist_head;
+ linkedlist_head.next = linkedlist_head.prev = cur;
+ }
+ else {
+ linkedlist_head.next = linkedlist_head.prev = &linkedlist_head;
+ }
+ _RPython_ThreadLocals_Release();
+}
+
+
struct pypy_threadlocal_s *
_RPython_ThreadLocals_Enum(struct pypy_threadlocal_s *prev)
{
@@ -48,23 +105,29 @@
where it is not the case are rather old nowadays. */
# endif
#endif
+ _RPython_ThreadLocals_Acquire();
oldnext = linkedlist_head.next;
tls->prev = &linkedlist_head;
tls->next = oldnext;
linkedlist_head.next = tls;
oldnext->prev = tls;
tls->ready = 42;
+ _RPython_ThreadLocals_Release();
}
static void threadloc_unlink(void *p)
{
+ /* warning: this can be called at completely random times without
+ the GIL. */
struct pypy_threadlocal_s *tls = (struct pypy_threadlocal_s *)p;
+ _RPython_ThreadLocals_Acquire();
if (tls->ready == 42) {
- tls->ready = 0;
tls->next->prev = tls->prev;
tls->prev->next = tls->next;
memset(tls, 0xDD, sizeof(struct pypy_threadlocal_s)); /* debug */
+ tls->ready = 0;
}
+ _RPython_ThreadLocals_Release();
#ifndef USE___THREAD
free(p);
#endif
@@ -77,7 +140,10 @@
There are some alternatives known, but they are horrible in other
ways (e.g. using undocumented behavior). This seems to be the
simplest, but feel free to fix if you need that.
- */
+
+ For this reason we have the line 'not _win32 or config.translation.shared'
+ in rpython.rlib.rthread.
+*/
BOOL WINAPI DllMain(HINSTANCE hinstDLL,
DWORD reason_for_call,
LPVOID reserved)
@@ -107,6 +173,7 @@
a non-null thread-local value). This is needed even in the
case where we use '__thread' below, for the destructor.
*/
+ assert(pypy_threadlocal_lock == 0);
#ifdef _WIN32
pypy_threadlocal_key = TlsAlloc();
if (pypy_threadlocal_key == TLS_OUT_OF_INDEXES)
@@ -119,6 +186,12 @@
abort();
}
_RPython_ThreadLocals_Build();
+
+#ifndef _WIN32
+ pthread_atfork(_RPython_ThreadLocals_Acquire,
+ _RPython_ThreadLocals_Release,
+ cleanup_after_fork);
+#endif
}
@@ -133,7 +206,7 @@
char *_RPython_ThreadLocals_Build(void)
{
- RPyAssert(pypy_threadlocal.ready == 0, "corrupted thread-local");
+ RPyAssert(pypy_threadlocal.ready == 0, "unclean thread-local");
_RPy_ThreadLocals_Init(&pypy_threadlocal);
/* we also set up &pypy_threadlocal as a POSIX thread-local variable,
diff --git a/rpython/translator/c/src/threadlocal.h
b/rpython/translator/c/src/threadlocal.h
--- a/rpython/translator/c/src/threadlocal.h
+++ b/rpython/translator/c/src/threadlocal.h
@@ -19,11 +19,17 @@
current thread, and if not, calls the following helper. */
RPY_EXTERN char *_RPython_ThreadLocals_Build(void);
+RPY_EXTERN void _RPython_ThreadLocals_Acquire(void);
+RPY_EXTERN void _RPython_ThreadLocals_Release(void);
+
+/* Must acquire/release the thread-local lock around a series of calls
+ to the following function */
RPY_EXTERN struct pypy_threadlocal_s *
_RPython_ThreadLocals_Enum(struct pypy_threadlocal_s *prev);
-#define OP_THREADLOCALREF_ENUM(p, r) \
- r = _RPython_ThreadLocals_Enum(p)
+#define OP_THREADLOCALREF_ACQUIRE(r) _RPython_ThreadLocals_Acquire()
+#define OP_THREADLOCALREF_RELEASE(r) _RPython_ThreadLocals_Release()
+#define OP_THREADLOCALREF_ENUM(p, r) r = _RPython_ThreadLocals_Enum(p)
/* ------------------------------------------------------------ */
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
