Author: Ronan Lamy <ronan.l...@gmail.com> Branch: llimpl Changeset: r82274:e6a89c683651 Date: 2016-02-15 17:45 +0000 http://bitbucket.org/pypy/pypy/changeset/e6a89c683651/
Log: hg merge default diff too long, truncating to 2000 out of 3299 lines diff --git a/lib-python/2.7/distutils/command/build_ext.py b/lib-python/2.7/distutils/command/build_ext.py --- a/lib-python/2.7/distutils/command/build_ext.py +++ b/lib-python/2.7/distutils/command/build_ext.py @@ -188,7 +188,7 @@ # the 'libs' directory is for binary installs - we assume that # must be the *native* platform. But we don't really support # cross-compiling via a binary install anyway, so we let it go. - self.library_dirs.append(os.path.join(sys.exec_prefix, 'include')) + self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) if self.debug: self.build_temp = os.path.join(self.build_temp, "Debug") else: diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.5.0 +Version: 1.5.2 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.5.0" -__version_info__ = (1, 5, 0) +__version__ = "1.5.2" +__version_info__ = (1, 5, 2) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -231,6 +231,12 @@ ((got_nonpos) == (expected <= 0) && \ (got) == (unsigned long long)expected) +#ifdef MS_WIN32 +# define _cffi_stdcall __stdcall +#else +# define _cffi_stdcall /* nothing */ +#endif + #ifdef __cplusplus } #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/_embedding.h @@ -0,0 +1,517 @@ + +/***** Support code for embedding *****/ + +#if defined(_MSC_VER) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include <Windows.h> +# define cffi_compare_and_swap(l,o,n) \ + (InterlockedCompareExchangePointer(l,n,o) == (o)) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include <pthread.h> + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static void *volatile lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + + XXX if importing 'site' fails, CPython (any version) calls + exit(). Should we try to work around this behavior here? + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + +#if PY_MAJOR_VERSION >= 3 + /* see comments in _cffi_carefully_make_gil() about the + Python2/Python3 difference + */ +#else + /* Acquire the GIL. We have no threadstate here. If Python is + already initialized, it is possible that there is already one + existing for this thread, but it is not made current now. + */ + PyEval_AcquireLock(); + + _cffi_py_initialize(); + + /* The Py_InitializeEx() sometimes made a threadstate for us, but + not always. Indeed Py_InitializeEx() could be called and do + nothing. So do we have a threadstate, or not? We don't know, + but we can replace it with NULL in all cases. + */ + (void)PyThreadState_Swap(NULL); + + /* Now we can release the GIL and re-acquire immediately using the + logic of PyGILState(), which handles making or installing the + correct threadstate. + */ + PyEval_ReleaseLock(); +#endif + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "<init code for '" _CFFI_MODULE_NAME "'>", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", + PyThreadState_GET()->interp->builtins) < 0) + goto error; + x = PyEval_EvalCode( +#if PY_MAJOR_VERSION < 3 + (PyCodeObject *) +#endif + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 1.5.2" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + } + result = -1; + goto done; +} + +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + What it really does is completely different in Python 2 and + Python 3. + + Python 2 + ======== + + Initialize the GIL, without initializing the rest of Python, + by calling PyEval_InitThreads(). + + PyEval_InitThreads() must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. We choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + Python 3 + ======== + + In Python 3, PyEval_InitThreads() cannot be called before + Py_InitializeEx() any more. So this function calls + Py_InitializeEx() first. It uses the same obscure logic to + make sure we never call it concurrently. + + Arguably, this is less good on the spinlock, because + Py_InitializeEx() takes much longer to run than + PyEval_InitThreads(). But I didn't find a way around it. + */ + +#ifdef WITH_THREAD + char *volatile *lock = (char *volatile *)_PyParser_TokenNames; + char *old_value; + + while (1) { /* spin loop */ + old_value = *lock; + if (old_value[0] == 'E') { + assert(old_value[1] == 'N'); + if (cffi_compare_and_swap(lock, old_value, old_value + 1)) + break; + } + else { + assert(old_value[0] == 'N'); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +#endif + +#if PY_MAJOR_VERSION >= 3 + /* Python 3: call Py_InitializeEx() */ + { + PyGILState_STATE state = PyGILState_UNLOCKED; + if (!Py_IsInitialized()) + _cffi_py_initialize(); + else + state = PyGILState_Ensure(); + + PyEval_InitThreads(); + PyGILState_Release(state); + } +#else + /* Python 2: call PyEval_InitThreads() */ +# ifdef WITH_THREAD + if (!PyEval_ThreadsInitialized()) { + PyEval_InitThreads(); /* makes the GIL */ + PyEval_ReleaseLock(); /* then release it */ + } + /* else: there is already a GIL, but we still needed to do the + spinlock dance to make sure that we see it as fully ready */ +# endif +#endif + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, old_value + 1, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void (*func)(const void *[]); + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -544,28 +544,45 @@ def _apply_embedding_fix(self, kwds): # must include an argument like "-lpython2.7" for the compiler + def ensure(key, value): + lst = kwds.setdefault(key, []) + if value not in lst: + lst.append(value) + # if '__pypy__' in sys.builtin_module_names: - if hasattr(sys, 'prefix'): - import os - libdir = os.path.join(sys.prefix, 'bin') - dirs = kwds.setdefault('library_dirs', []) - if libdir not in dirs: - dirs.append(libdir) - pythonlib = "pypy-c" + if sys.platform == "win32": + # we need 'libpypy-c.lib'. Current distributions of + # pypy (>= 4.1) contain it as 'libs/python27.lib'. + pythonlib = "python27" + if hasattr(sys, 'prefix'): + ensure('library_dirs', os.path.join(sys.prefix, 'libs')) + else: + # we need 'libpypy-c.{so,dylib}', which should be by + # default located in 'sys.prefix/bin' + pythonlib = "pypy-c" + if hasattr(sys, 'prefix'): + import os + ensure('library_dirs', os.path.join(sys.prefix, 'bin')) else: if sys.platform == "win32": template = "python%d%d" - if sys.flags.debug: - template = template + '_d' + if hasattr(sys, 'gettotalrefcount'): + template += '_d' else: + try: + import sysconfig + except ImportError: # 2.6 + from distutils import sysconfig template = "python%d.%d" + if sysconfig.get_config_var('DEBUG_EXT'): + template += sysconfig.get_config_var('DEBUG_EXT') pythonlib = (template % (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) if hasattr(sys, 'abiflags'): pythonlib += sys.abiflags - libraries = kwds.setdefault('libraries', []) - if pythonlib not in libraries: - libraries.append(pythonlib) + ensure('libraries', pythonlib) + if sys.platform == "win32": + ensure('extra_link_args', '/MANIFEST') def set_source(self, module_name, source, source_extension='.c', **kwds): if hasattr(self, '_assigned_source'): @@ -631,7 +648,7 @@ compiled DLL. Use '*' to force distutils' choice, suitable for regular CPython C API modules. Use a file name ending in '.*' to ask for the system's default extension for dynamic libraries - (.so/.dll). + (.so/.dll/.dylib). The default is '*' when building a non-embedded C API extension, and (module_name + '.*') when building an embedded library. @@ -695,6 +712,10 @@ # self._embedding = pysource + def def_extern(self, *args, **kwds): + raise ValueError("ffi.def_extern() is only available on API-mode FFI " + "objects") + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -220,7 +220,7 @@ self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() - self._options = None + self._options = {} self._int_constants = {} self._recomplete = [] self._uses_new_feature = None @@ -374,7 +374,7 @@ def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) - if self._options['dllexport']: + if self._options.get('dllexport'): tag = 'dllexport_python ' elif self._inside_extern_python: tag = 'extern_python ' @@ -450,7 +450,7 @@ prevobj, prevquals = self._declarations[name] if prevobj is obj and prevquals == quals: return - if not self._options['override']: + if not self._options.get('override'): raise api.FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) @@ -729,7 +729,7 @@ if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" % (tp,)) - tp.packed = self._options['packed'] + tp.packed = self._options.get('packed') if tp.completed: # must be re-completed: it is not opaque any more tp.completed = 0 self._recomplete.append(tp) diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -21,14 +21,12 @@ allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0, target_extension=None, - embedding=False): +def compile(tmpdir, ext, compiler_verbose=0): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose, - target_extension, embedding) + outputfilename = _build(tmpdir, ext, compiler_verbose) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -38,32 +36,7 @@ os.environ[key] = value return outputfilename -def _save_val(name): - import distutils.sysconfig - config_vars = distutils.sysconfig.get_config_vars() - return config_vars.get(name, Ellipsis) - -def _restore_val(name, value): - import distutils.sysconfig - config_vars = distutils.sysconfig.get_config_vars() - config_vars[name] = value - if value is Ellipsis: - del config_vars[name] - -def _win32_hack_for_embedding(): - from distutils.msvc9compiler import MSVCCompiler - if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'): - MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \ - MSVCCompiler._remove_visual_c_ref - MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file: manifest_file - -def _win32_unhack_for_embedding(): - from distutils.msvc9compiler import MSVCCompiler - MSVCCompiler._remove_visual_c_ref = \ - MSVCCompiler._remove_visual_c_ref_CFFI_BAK - -def _build(tmpdir, ext, compiler_verbose=0, target_extension=None, - embedding=False): +def _build(tmpdir, ext, compiler_verbose=0): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -76,25 +49,14 @@ options['build_temp'] = ('ffiplatform', tmpdir) # try: - if sys.platform == 'win32' and embedding: - _win32_hack_for_embedding() old_level = distutils.log.set_threshold(0) or 0 - old_SO = _save_val('SO') - old_EXT_SUFFIX = _save_val('EXT_SUFFIX') try: - if target_extension is not None: - _restore_val('SO', target_extension) - _restore_val('EXT_SUFFIX', target_extension) distutils.log.set_verbosity(compiler_verbose) dist.run_command('build_ext') cmd_obj = dist.get_command_obj('build_ext') [soname] = cmd_obj.get_outputs() finally: distutils.log.set_threshold(old_level) - _restore_val('SO', old_SO) - _restore_val('EXT_SUFFIX', old_EXT_SUFFIX) - if sys.platform == 'win32' and embedding: - _win32_unhack_for_embedding() except (distutils.errors.CompileError, distutils.errors.LinkError) as e: raise VerificationError('%s: %s' % (e.__class__.__name__, e)) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1170,6 +1170,8 @@ repr_arguments = ', '.join(arguments) repr_arguments = repr_arguments or 'void' name_and_arguments = '%s(%s)' % (name, repr_arguments) + if tp.abi == "__stdcall": + name_and_arguments = '_cffi_stdcall ' + name_and_arguments # def may_need_128_bits(tp): return (isinstance(tp, model.PrimitiveType) and @@ -1357,6 +1359,58 @@ parts[-1] += extension return os.path.join(outputdir, *parts), parts + +# Aaargh. Distutils is not tested at all for the purpose of compiling +# DLLs that are not extension modules. Here are some hacks to work +# around that, in the _patch_for_*() functions... + +def _patch_meth(patchlist, cls, name, new_meth): + old = getattr(cls, name) + patchlist.append((cls, name, old)) + setattr(cls, name, new_meth) + return old + +def _unpatch_meths(patchlist): + for cls, name, old_meth in reversed(patchlist): + setattr(cls, name, old_meth) + +def _patch_for_embedding(patchlist): + if sys.platform == 'win32': + # we must not remove the manifest when building for embedding! + from distutils.msvc9compiler import MSVCCompiler + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) + + if sys.platform == 'darwin': + # we must not make a '-bundle', but a '-dynamiclib' instead + from distutils.ccompiler import CCompiler + def my_link_shared_object(self, *args, **kwds): + if '-bundle' in self.linker_so: + self.linker_so = list(self.linker_so) + i = self.linker_so.index('-bundle') + self.linker_so[i] = '-dynamiclib' + return old_link_shared_object(self, *args, **kwds) + old_link_shared_object = _patch_meth(patchlist, CCompiler, + 'link_shared_object', + my_link_shared_object) + +def _patch_for_target(patchlist, target): + from distutils.command.build_ext import build_ext + # if 'target' is different from '*', we need to patch some internal + # method to just return this 'target' value, instead of having it + # built from module_name + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + elif sys.platform == 'darwin': + target += '.dylib' + else: + target += '.so' + _patch_meth(patchlist, build_ext, 'get_ext_filename', + lambda self, ext_name: target) + + def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, compiler_verbose=1, target=None, **kwds): @@ -1382,36 +1436,22 @@ target = '%s.*' % module_name else: target = '*' - if target == '*': - target_module_name = module_name - target_extension = None # use default - else: - if target.endswith('.*'): - target = target[:-2] - if sys.platform == 'win32': - target += '.dll' - else: - target += '.so' - # split along the first '.' (not the last one, otherwise the - # preceeding dots are interpreted as splitting package names) - index = target.find('.') - if index < 0: - raise ValueError("target argument %r should be a file name " - "containing a '.'" % (target,)) - target_module_name = target[:index] - target_extension = target[index:] # - ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds) + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) updated = make_c_source(ffi, module_name, preamble, c_file) if call_c_compiler: + patchlist = [] cwd = os.getcwd() try: + if embedding: + _patch_for_embedding(patchlist) + if target != '*': + _patch_for_target(patchlist, target) os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose, - target_extension, - embedding=embedding) + outputfilename = ffiplatform.compile('.', ext, compiler_verbose) finally: os.chdir(cwd) + _unpatch_meths(patchlist) return outputfilename else: return ext, updated diff --git a/lib_pypy/cffi/vengine_cpy.py b/lib_pypy/cffi/vengine_cpy.py --- a/lib_pypy/cffi/vengine_cpy.py +++ b/lib_pypy/cffi/vengine_cpy.py @@ -1,3 +1,6 @@ +# +# DEPRECATED: implementation for ffi.verify() +# import sys, imp from . import model, ffiplatform diff --git a/lib_pypy/cffi/vengine_gen.py b/lib_pypy/cffi/vengine_gen.py --- a/lib_pypy/cffi/vengine_gen.py +++ b/lib_pypy/cffi/vengine_gen.py @@ -1,3 +1,6 @@ +# +# DEPRECATED: implementation for ffi.verify() +# import sys, os import types diff --git a/lib_pypy/cffi/verifier.py b/lib_pypy/cffi/verifier.py --- a/lib_pypy/cffi/verifier.py +++ b/lib_pypy/cffi/verifier.py @@ -1,3 +1,6 @@ +# +# DEPRECATED: implementation for ffi.verify() +# import sys, os, binascii, shutil, io from . import __version_verifier_modules__ from . import ffiplatform diff --git a/pypy/config/pypyoption.py b/pypy/config/pypyoption.py --- a/pypy/config/pypyoption.py +++ b/pypy/config/pypyoption.py @@ -36,13 +36,13 @@ "cStringIO", "thread", "itertools", "pyexpat", "_ssl", "cpyext", "array", "binascii", "_multiprocessing", '_warnings', "_collections", "_multibytecodec", "micronumpy", "_continuation", "_cffi_backend", - "_csv", "cppyy", "_pypyjson" + "_csv", "cppyy", "_pypyjson", "_vmprof", ]) -if ((sys.platform.startswith('linux') or sys.platform == 'darwin') - and os.uname()[4] == 'x86_64' and sys.maxint > 2**32): +#if ((sys.platform.startswith('linux') or sys.platform == 'darwin') +# and os.uname()[4] == 'x86_64' and sys.maxint > 2**32): # it's not enough that we get x86_64 - working_modules.add('_vmprof') +# working_modules.add('_vmprof') translation_modules = default_modules.copy() translation_modules.update([ diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -7,6 +7,9 @@ Fixed ``_PyLong_FromByteArray()``, which was buggy. +Fixed a crash with stacklets (or greenlets) on non-Linux machines +which showed up if you forget stacklets without resuming them. + .. branch: numpy-1.10 Fix tests to run cleanly with -A and start to fix micronumpy for upstream numpy @@ -38,7 +41,8 @@ .. branch: compress-numbering -Improve the memory signature of numbering instances in the JIT. +Improve the memory signature of numbering instances in the JIT. This should massively +decrease the amount of memory consumed by the JIT, which is significant for most programs. .. branch: fix-trace-too-long-heuristic @@ -142,4 +146,9 @@ .. branch: vmprof-newstack -Refactor vmprof to work cross-operating-system. \ No newline at end of file +Refactor vmprof to work cross-operating-system. + +.. branch: seperate-strucmember_h + +Seperate structmember.h from Python.h Also enhance creating api functions +to specify which header file they appear in (previously only pypy_decl.h) diff --git a/pypy/goal/targetpypystandalone.py b/pypy/goal/targetpypystandalone.py --- a/pypy/goal/targetpypystandalone.py +++ b/pypy/goal/targetpypystandalone.py @@ -239,6 +239,9 @@ raise Exception("Cannot use the --output option with PyPy " "when --shared is on (it is by default). " "See issue #1971.") + if sys.platform == 'win32': + config.translation.libname = '..\\..\\libs\\python27.lib' + thisdir.join('..', '..', 'libs').ensure(dir=1) if config.translation.thread: config.objspace.usemodules.thread = True diff --git a/pypy/interpreter/baseobjspace.py b/pypy/interpreter/baseobjspace.py --- a/pypy/interpreter/baseobjspace.py +++ b/pypy/interpreter/baseobjspace.py @@ -306,7 +306,7 @@ return None -class W_InterpIterable(W_Root): +class InterpIterable(object): def __init__(self, space, w_iterable): self.w_iter = space.iter(w_iterable) self.space = space @@ -745,9 +745,13 @@ return self.int_w(self.hash(w_obj)) def len_w(self, w_obj): - """shotcut for space.int_w(space.len(w_obj))""" + """shortcut for space.int_w(space.len(w_obj))""" return self.int_w(self.len(w_obj)) + def contains_w(self, w_container, w_item): + """shortcut for space.is_true(space.contains(w_container, w_item))""" + return self.is_true(self.contains(w_container, w_item)) + def setitem_str(self, w_obj, key, w_value): return self.setitem(w_obj, self.wrap(key), w_value) @@ -846,7 +850,7 @@ return lst_w[:] # make the resulting list resizable def iteriterable(self, w_iterable): - return W_InterpIterable(self, w_iterable) + return InterpIterable(self, w_iterable) def _unpackiterable_unknown_length(self, w_iterator, w_iterable): """Unpack an iterable of unknown length into an interp-level @@ -1237,7 +1241,7 @@ if not isinstance(statement, PyCode): raise TypeError('space.exec_(): expected a string, code or PyCode object') w_key = self.wrap('__builtins__') - if not self.is_true(self.contains(w_globals, w_key)): + if not self.contains_w(w_globals, w_key): self.setitem(w_globals, w_key, self.wrap(self.builtin)) return statement.exec_code(self, w_globals, w_locals) diff --git a/pypy/module/__builtin__/interp_classobj.py b/pypy/module/__builtin__/interp_classobj.py --- a/pypy/module/__builtin__/interp_classobj.py +++ b/pypy/module/__builtin__/interp_classobj.py @@ -20,7 +20,7 @@ if not space.isinstance_w(w_dict, space.w_dict): raise_type_err(space, 'bases', 'tuple', w_bases) - if not space.is_true(space.contains(w_dict, space.wrap("__doc__"))): + if not space.contains_w(w_dict, space.wrap("__doc__")): space.setitem(w_dict, space.wrap("__doc__"), space.w_None) # XXX missing: lengthy and obscure logic about "__module__" diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.5.0" +VERSION = "1.5.2" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: @@ -69,6 +69,7 @@ def startup(self, space): from pypy.module._cffi_backend import embedding embedding.glob.space = space + embedding.glob.patched_sys = False def get_dict_rtld_constants(): diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -45,6 +45,26 @@ pass glob = Global() +def patch_sys(space): + # Annoying: CPython would just use the C-level std{in,out,err} as + # configured by the main application, for example in binary mode + # on Windows or with buffering turned off. We can't easily do the + # same. Instead, go for the safest bet (but possibly bad for + # performance) and open sys.std{in,out,err} unbuffered. On + # Windows I guess binary mode is a better default choice. + # + # XXX if needed, we could add support for a flag passed to + # pypy_init_embedded_cffi_module(). + if not glob.patched_sys: + space.appexec([], """(): + import os + sys.stdin = sys.__stdin__ = os.fdopen(0, 'rb', 0) + sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0) + sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0) + """) + glob.patched_sys = True + + def pypy_init_embedded_cffi_module(version, init_struct): # called from __init__.py name = "?" @@ -56,6 +76,7 @@ must_leave = False try: must_leave = space.threadlocals.try_enter_thread(space) + patch_sys(space) load_embedded_cffi_module(space, version, init_struct) res = 0 except OperationError, operr: @@ -84,72 +105,87 @@ return rffi.cast(rffi.INT, res) # ____________________________________________________________ + if os.name == 'nt': - do_startup = r''' -#include <stdio.h> -#define WIN32_LEAN_AND_MEAN + + do_includes = r""" +#define _WIN32_WINNT 0x0501 #include <windows.h> -RPY_EXPORTED void rpython_startup_code(void); -RPY_EXPORTED int pypy_setup_home(char *, int); -static unsigned char _cffi_ready = 0; -static const char *volatile _cffi_module_name; +#define CFFI_INIT_HOME_PATH_MAX _MAX_PATH +static void _cffi_init(void); +static void _cffi_init_error(const char *msg, const char *extra); -static void _cffi_init_error(const char *msg, const char *extra) +static int _cffi_init_home(char *output_home_path) { - fprintf(stderr, - "\nPyPy initialization failure when loading module '%s':\n%s%s\n", - _cffi_module_name, msg, extra); -} - -BOOL CALLBACK _cffi_init(PINIT_ONCE InitOnce, PVOID Parameter, PVOID *lpContex) -{ - - HMODULE hModule; - TCHAR home[_MAX_PATH]; - rpython_startup_code(); - RPyGilAllocate(); + HMODULE hModule = 0; + DWORD res; GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCTSTR)&_cffi_init, &hModule); + if (hModule == 0 ) { - /* TODO turn the int into a string with FormatMessage */ - - _cffi_init_error("dladdr() failed: ", ""); - return TRUE; + _cffi_init_error("GetModuleHandleEx() failed", ""); + return -1; } - GetModuleFileName(hModule, home, _MAX_PATH); - if (pypy_setup_home(home, 1) != 0) { - _cffi_init_error("pypy_setup_home() failed", ""); - return TRUE; + res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX); + if (res >= CFFI_INIT_HOME_PATH_MAX) { + return -1; } - _cffi_ready = 1; - fprintf(stderr, "startup succeeded, home %s\n", home); - return TRUE; + return 0; } -RPY_EXPORTED -int pypy_carefully_make_gil(const char *name) +static void _cffi_init_once(void) { - /* For CFFI: this initializes the GIL and loads the home path. - It can be called completely concurrently from unrelated threads. - It assumes that we don't hold the GIL before (if it exists), and we - don't hold it afterwards. - */ - static INIT_ONCE s_init_once; + static LONG volatile lock = 0; + static int _init_called = 0; - _cffi_module_name = name; /* not really thread-safe, but better than - nothing */ - InitOnceExecuteOnce(&s_init_once, _cffi_init, NULL, NULL); - return (int)_cffi_ready - 1; -}''' + while (InterlockedCompareExchange(&lock, 1, 0) != 0) { + SwitchToThread(); /* spin loop */ + } + if (!_init_called) { + _cffi_init(); + _init_called = 1; + } + InterlockedCompareExchange(&lock, 0, 1); +} +""" + else: - do_startup = r""" -#include <stdio.h> + + do_includes = r""" #include <dlfcn.h> #include <pthread.h> +#define CFFI_INIT_HOME_PATH_MAX PATH_MAX +static void _cffi_init(void); +static void _cffi_init_error(const char *msg, const char *extra); + +static int _cffi_init_home(char *output_home_path) +{ + Dl_info info; + dlerror(); /* reset */ + if (dladdr(&_cffi_init, &info) == 0) { + _cffi_init_error("dladdr() failed: ", dlerror()); + return -1; + } + if (realpath(info.dli_fname, output_home_path) == NULL) { + perror("realpath() failed"); + _cffi_init_error("realpath() failed", ""); + return -1; + } + return 0; +} + +static void _cffi_init_once(void) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + pthread_once(&once_control, _cffi_init); +} +""" + +do_startup = do_includes + r""" RPY_EXPORTED void rpython_startup_code(void); RPY_EXPORTED int pypy_setup_home(char *, int); @@ -165,17 +201,13 @@ static void _cffi_init(void) { - Dl_info info; - char *home; + char home[CFFI_INIT_HOME_PATH_MAX + 1]; rpython_startup_code(); RPyGilAllocate(); - if (dladdr(&_cffi_init, &info) == 0) { - _cffi_init_error("dladdr() failed: ", dlerror()); + if (_cffi_init_home(home) != 0) return; - } - home = realpath(info.dli_fname, NULL); if (pypy_setup_home(home, 1) != 0) { _cffi_init_error("pypy_setup_home() failed", ""); return; @@ -191,11 +223,9 @@ It assumes that we don't hold the GIL before (if it exists), and we don't hold it afterwards. */ - static pthread_once_t once_control = PTHREAD_ONCE_INIT; - _cffi_module_name = name; /* not really thread-safe, but better than nothing */ - pthread_once(&once_control, _cffi_init); + _cffi_init_once(); return (int)_cffi_ready - 1; } """ diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.5.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.5.2", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_ztranslation.py b/pypy/module/_cffi_backend/test/test_ztranslation.py --- a/pypy/module/_cffi_backend/test/test_ztranslation.py +++ b/pypy/module/_cffi_backend/test/test_ztranslation.py @@ -4,15 +4,18 @@ # side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule() from pypy.module._cffi_backend import misc -from pypy.module._cffi_backend import cffi1_module +from pypy.module._cffi_backend import embedding def test_checkmodule(): # prepare_file_argument() is not working without translating the _file # module too def dummy_prepare_file_argument(space, fileobj): - # call load_cffi1_module() too, from a random place like here - cffi1_module.load_cffi1_module(space, "foo", "foo", 42) + # call pypy_init_embedded_cffi_module() from a random place like here + # --- this calls load_cffi1_module(), too + embedding.pypy_init_embedded_cffi_module( + rffi.cast(rffi.INT, embedding.EMBED_VERSION_MIN), + 42) return lltype.nullptr(rffi.CCHARP.TO) old = ctypeptr.prepare_file_argument try: diff --git a/pypy/module/_demo/test/test_import.py b/pypy/module/_demo/test/test_import.py --- a/pypy/module/_demo/test/test_import.py +++ b/pypy/module/_demo/test/test_import.py @@ -12,8 +12,7 @@ w_modules = space.sys.get('modules') assert _demo.Module.demo_events == ['setup'] - assert not space.is_true(space.contains(w_modules, - space.wrap('_demo'))) + assert not space.contains_w(w_modules, space.wrap('_demo')) # first import w_import = space.builtin.get('__import__') diff --git a/pypy/module/_vmprof/interp_vmprof.py b/pypy/module/_vmprof/interp_vmprof.py --- a/pypy/module/_vmprof/interp_vmprof.py +++ b/pypy/module/_vmprof/interp_vmprof.py @@ -60,7 +60,7 @@ Must be smaller than 1.0 """ w_modules = space.sys.get('modules') - if space.is_true(space.contains(w_modules, space.wrap('_continuation'))): + if space.contains_w(w_modules, space.wrap('_continuation')): space.warn(space.wrap("Using _continuation/greenlet/stacklet together " "with vmprof will crash"), space.w_RuntimeWarning) diff --git a/pypy/module/_vmprof/test/test_direct.py b/pypy/module/_vmprof/test/test_direct.py --- a/pypy/module/_vmprof/test/test_direct.py +++ b/pypy/module/_vmprof/test/test_direct.py @@ -69,4 +69,4 @@ result = ffi.cast("void**", buf) res = lib.vmprof_write_header_for_jit_addr(result, 0, ffi.NULL, 100) assert res == 10 - assert [x for x in buf] == [6, 0, 3, 16, 3, 12, 3, 8, 3, 4 + assert [x for x in buf] == [6, 0, 3, 16, 3, 12, 3, 8, 3, 4] diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -59,7 +59,7 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( include_dirs=include_dirs, - includes=['Python.h', 'stdarg.h'], + includes=['Python.h', 'stdarg.h', 'structmember.h'], compile_extra=['-DPy_BUILD_CORE'], ) @@ -129,6 +129,7 @@ for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) udir.join('pypy_decl.h').write("/* Will be filled later */\n") +udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n") udir.join('pypy_macros.h').write("/* Will be filled later */\n") globals().update(rffi_platform.configure(CConfig_constants)) @@ -147,7 +148,7 @@ # XXX: 20 lines of code to recursively copy a directory, really?? assert dstdir.check(dir=True) headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl') - for name in ("pypy_decl.h", "pypy_macros.h"): + for name in ("pypy_decl.h", "pypy_macros.h", "pypy_structmember_decl.h"): headers.append(udir.join(name)) _copy_header_files(headers, dstdir) @@ -232,7 +233,7 @@ wrapper.c_name = cpyext_namespace.uniquename(self.c_name) return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', gil=None): """ Declares a function to be exported. @@ -241,8 +242,8 @@ special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual exception into a wrapped SystemError. Unwrapped exceptions also cause a SytemError. - - set `external` to False to get a C function pointer, but not exported by - the API headers. + - `header` is the header file to export the function in, Set to None to get + a C function pointer, but not exported by the API headers. - set `gil` to "acquire", "release" or "around" to acquire the GIL, release the GIL, or both """ @@ -263,7 +264,7 @@ def decorate(func): func_name = func.func_name - if external: + if header is not None: c_name = None else: c_name = func_name @@ -271,7 +272,7 @@ c_name=c_name, gil=gil) func.api_func = api_function - if external: + if header is not None: assert func_name not in FUNCTIONS, ( "%s already registered" % func_name) @@ -363,8 +364,9 @@ unwrapper_catch = make_unwrapper(True) unwrapper_raise = make_unwrapper(False) - if external: + if header is not None: FUNCTIONS[func_name] = api_function + FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests return unwrapper_raise # used in 'normal' RPython code. return decorate @@ -383,6 +385,7 @@ INTERPLEVEL_API = {} FUNCTIONS = {} +FUNCTIONS_BY_HEADER = {} # These are C symbols which cpyext will export, but which are defined in .c # files somewhere in the implementation of cpyext (rather than being defined in @@ -811,6 +814,7 @@ global_code = '\n'.join(global_objects) prologue = ("#include <Python.h>\n" + "#include <structmember.h>\n" "#include <src/thread.c>\n") code = (prologue + struct_declaration_code + @@ -960,7 +964,8 @@ "NOT_RPYTHON" # implement function callbacks and generate function decls functions = [] - pypy_decls = [] + decls = {} + pypy_decls = decls['pypy_decl.h'] = [] pypy_decls.append("#ifndef _PYPY_PYPY_DECL_H\n") pypy_decls.append("#define _PYPY_PYPY_DECL_H\n") pypy_decls.append("#ifndef PYPY_STANDALONE\n") @@ -973,17 +978,23 @@ for decl in FORWARD_DECLS: pypy_decls.append("%s;" % (decl,)) - for name, func in sorted(FUNCTIONS.iteritems()): - restype, args = c_function_signature(db, func) - pypy_decls.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args)) - if api_struct: - callargs = ', '.join('arg%d' % (i,) - for i in range(len(func.argtypes))) - if func.restype is lltype.Void: - body = "{ _pypyAPI.%s(%s); }" % (name, callargs) - else: - body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) - functions.append('%s %s(%s)\n%s' % (restype, name, args, body)) + for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems(): + if header_name not in decls: + header = decls[header_name] = [] + else: + header = decls[header_name] + + for name, func in sorted(header_functions.iteritems()): + restype, args = c_function_signature(db, func) + header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args)) + if api_struct: + callargs = ', '.join('arg%d' % (i,) + for i in range(len(func.argtypes))) + if func.restype is lltype.Void: + body = "{ _pypyAPI.%s(%s); }" % (name, callargs) + else: + body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) + functions.append('%s %s(%s)\n%s' % (restype, name, args, body)) for name in VA_TP_LIST: name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % @@ -1007,8 +1018,9 @@ pypy_decls.append("#endif /*PYPY_STANDALONE*/\n") pypy_decls.append("#endif /*_PYPY_PYPY_DECL_H*/\n") - pypy_decl_h = udir.join('pypy_decl.h') - pypy_decl_h.write('\n'.join(pypy_decls)) + for header_name, header_decls in decls.iteritems(): + decl_h = udir.join(header_name) + decl_h.write('\n'.join(header_decls)) return functions separate_module_files = [source_dir / "varargwrapper.c", diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -73,7 +73,7 @@ "Don't know how to realize a buffer")) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def buffer_dealloc(space, py_obj): py_buf = rffi.cast(PyBufferObject, py_obj) if py_buf.c_b_base: diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -39,7 +39,7 @@ py_frame.c_f_locals = make_ref(space, frame.get_w_locals()) rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def frame_dealloc(space, py_obj): py_frame = rffi.cast(PyFrameObject, py_obj) py_code = rffi.cast(PyObject, py_frame.c_f_code) diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -56,7 +56,7 @@ assert isinstance(w_obj, Function) py_func.c_func_name = make_ref(space, space.wrap(w_obj.name)) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) @@ -75,7 +75,7 @@ rffi.setintfield(py_code, 'c_co_flags', co_flags) rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def code_dealloc(space, py_obj): py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -84,6 +84,7 @@ #include "pyconfig.h" #include "object.h" +#include "pymath.h" #include "pyport.h" #include "warnings.h" @@ -115,7 +116,6 @@ #include "compile.h" #include "frameobject.h" #include "eval.h" -#include "pymath.h" #include "pymem.h" #include "pycobject.h" #include "pycapsule.h" @@ -132,9 +132,6 @@ /* Missing definitions */ #include "missing.h" -// XXX This shouldn't be included here -#include "structmember.h" - #include <pypy_decl.h> /* Define macros for inline documentation. */ diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -7,6 +7,18 @@ extern "C" { #endif +#define PyFloat_STR_PRECISION 12 + +#ifdef Py_NAN +#define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) +#endif + +#define Py_RETURN_INF(sign) do \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(Py_HUGE_VAL); \ + } else { \ + return PyFloat_FromDouble(-Py_HUGE_VAL); \ + } while(0) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/include/pymath.h b/pypy/module/cpyext/include/pymath.h --- a/pypy/module/cpyext/include/pymath.h +++ b/pypy/module/cpyext/include/pymath.h @@ -17,4 +17,35 @@ #define Py_HUGE_VAL HUGE_VAL #endif +/* Py_NAN + * A value that evaluates to a NaN. On IEEE 754 platforms INF*0 or + * INF/INF works. Define Py_NO_NAN in pyconfig.h if your platform + * doesn't support NaNs. + */ +#if !defined(Py_NAN) && !defined(Py_NO_NAN) +#if !defined(__INTEL_COMPILER) + #define Py_NAN (Py_HUGE_VAL * 0.) +#else /* __INTEL_COMPILER */ + #if defined(ICC_NAN_STRICT) + #pragma float_control(push) + #pragma float_control(precise, on) + #pragma float_control(except, on) + #if defined(_MSC_VER) + __declspec(noinline) + #else /* Linux */ + __attribute__((noinline)) + #endif /* _MSC_VER */ + static double __icc_nan() + { + return sqrt(-1.0); + } + #pragma float_control (pop) + #define Py_NAN __icc_nan() + #else /* ICC_NAN_RELAXED as default for Intel Compiler */ + static union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; + #define Py_NAN (__nan_store.__icc_nan) + #endif /* ICC_NAN_STRICT */ +#endif /* __INTEL_COMPILER */ +#endif + #endif /* Py_PYMATH_H */ diff --git a/pypy/module/cpyext/include/structmember.h b/pypy/module/cpyext/include/structmember.h --- a/pypy/module/cpyext/include/structmember.h +++ b/pypy/module/cpyext/include/structmember.h @@ -4,54 +4,85 @@ extern "C" { #endif + +/* Interface to map C struct members to Python object attributes */ + #include <stddef.h> /* For offsetof */ + +/* The offsetof() macro calculates the offset of a structure member + in its structure. Unfortunately this cannot be written down + portably, hence it is provided by a Standard C header file. + For pre-Standard C compilers, here is a version that usually works + (but watch out!): */ + #ifndef offsetof #define offsetof(type, member) ( (int) & ((type*)0) -> member ) #endif +/* An array of memberlist structures defines the name, type and offset + of selected members of a C structure. These can be read by + PyMember_Get() and set by PyMember_Set() (except if their READONLY flag + is set). The array must be terminated with an entry whose name + pointer is NULL. */ + + typedef struct PyMemberDef { - /* Current version, use this */ - char *name; - int type; - Py_ssize_t offset; - int flags; - char *doc; + /* Current version, use this */ + char *name; + int type; + Py_ssize_t offset; + int flags; + char *doc; } PyMemberDef; +/* Types */ +#define T_SHORT 0 +#define T_INT 1 +#define T_LONG 2 +#define T_FLOAT 3 +#define T_DOUBLE 4 +#define T_STRING 5 +#define T_OBJECT 6 +/* XXX the ordering here is weird for binary compatibility */ +#define T_CHAR 7 /* 1-character string */ +#define T_BYTE 8 /* 8-bit signed int */ +/* unsigned variants: */ +#define T_UBYTE 9 +#define T_USHORT 10 +#define T_UINT 11 +#define T_ULONG 12 -/* Types. These constants are also in structmemberdefs.py. */ -#define T_SHORT 0 -#define T_INT 1 -#define T_LONG 2 -#define T_FLOAT 3 -#define T_DOUBLE 4 -#define T_STRING 5 -#define T_OBJECT 6 -#define T_CHAR 7 /* 1-character string */ -#define T_BYTE 8 /* 8-bit signed int */ -#define T_UBYTE 9 -#define T_USHORT 10 -#define T_UINT 11 -#define T_ULONG 12 -#define T_STRING_INPLACE 13 /* Strings contained in the structure */ -#define T_BOOL 14 -#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError - when the value is NULL, instead of - converting to None. */ -#define T_LONGLONG 17 -#define T_ULONGLONG 18 -#define T_PYSSIZET 19 +/* Added by Jack: strings contained in the structure */ +#define T_STRING_INPLACE 13 + +/* Added by Lillo: bools contained in the structure (assumed char) */ +#define T_BOOL 14 + +#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError + when the value is NULL, instead of + converting to None. */ +#ifdef HAVE_LONG_LONG +#define T_LONGLONG 17 +#define T_ULONGLONG 18 +#endif /* HAVE_LONG_LONG */ + +#define T_PYSSIZET 19 /* Py_ssize_t */ /* Flags. These constants are also in structmemberdefs.py. */ -#define READONLY 1 -#define RO READONLY /* Shorthand */ +#define READONLY 1 +#define RO READONLY /* Shorthand */ #define READ_RESTRICTED 2 #define PY_WRITE_RESTRICTED 4 -#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) +#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) + + +/* API functions. */ +#include "pypy_structmember_decl.h" #ifdef __cplusplus } #endif #endif /* !Py_STRUCTMEMBER_H */ + diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -50,7 +50,7 @@ py_func.c_m_self = make_ref(space, w_obj.w_self) py_func.c_m_module = make_ref(space, w_obj.w_module) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -70,7 +70,7 @@ alloc : allocate and basic initialization of a raw PyObject attach : Function called to tie a raw structure to a pypy object realize : Function called to create a pypy object from a raw struct - dealloc : a cpython_api(external=False), similar to PyObject_dealloc + dealloc : a cpython_api(header=None), similar to PyObject_dealloc """ tp_basestruct = kw.pop('basestruct', PyObject.TO) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -41,7 +41,7 @@ rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti) rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno()) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def traceback_dealloc(space, py_obj): py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -36,7 +36,7 @@ py_slice.c_stop = make_ref(space, w_obj.w_stop) py_slice.c_step = make_ref(space, w_obj.w_step) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -309,7 +309,7 @@ return space.wrap(generic_cpy_call(space, func_target, w_self, w_other)) -@cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) +@cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) @@ -320,30 +320,30 @@ w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) -@cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) +@cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, header=None) def slot_tp_init(space, w_self, w_args, w_kwds): w_descr = space.lookup(w_self, '__init__') args = Arguments.frompacked(space, w_args, w_kwds) space.get_and_call_args(w_descr, w_self, args) return 0 -@cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) +@cpython_api([PyObject, PyObject, PyObject], PyObject, header=None) def slot_tp_call(space, w_self, w_args, w_kwds): return space.call(w_self, w_args, w_kwds) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_str(space, w_self): return space.str(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_nb_int(space, w_self): return space.int(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_iter(space, w_self): return space.iter(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_iternext(space, w_self): return space.next(w_self) @@ -371,7 +371,7 @@ return @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1, external=True) # XXX should not be exported + error=-1) # XXX should be header=None @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) def slot_tp_setattro(space, w_self, w_name, w_value): if w_value is not None: @@ -385,8 +385,7 @@ if getattr_fn is None: return - @cpython_api([PyObject, PyObject], PyObject, - external=True) + @cpython_api([PyObject, PyObject], PyObject) @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) def slot_tp_getattro(space, w_self, w_name): return space.call_function(getattr_fn, w_self, w_name) diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py --- a/pypy/module/cpyext/stringobject.py +++ b/pypy/module/cpyext/stringobject.py @@ -103,7 +103,7 @@ track_reference(space, py_obj, w_obj) return w_obj -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py --- a/pypy/module/cpyext/structmember.py +++ b/pypy/module/cpyext/structmember.py @@ -31,8 +31,10 @@ (T_PYSSIZET, rffi.SSIZE_T, PyLong_AsSsize_t), ]) +_HEADER = 'pypy_structmember_decl.h' -@cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject) + +@cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject, header=_HEADER) def PyMember_GetOne(space, obj, w_member): addr = rffi.cast(ADDR, obj) addr += w_member.c_offset @@ -83,7 +85,8 @@ return w_result -@cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, error=-1) +@cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, + error=-1, header=_HEADER) def PyMember_SetOne(space, obj, w_member, w_value): addr = rffi.cast(ADDR, obj) addr += w_member.c_offset diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -863,3 +863,15 @@ os.unlink('_imported_already') except OSError: pass + + def test_no_structmember(self): + """structmember.h should not be included by default.""" + mod = self.import_extension('foo', [ + ('bar', 'METH_NOARGS', + ''' + /* reuse a name that is #defined in structmember.h */ + int RO; + Py_RETURN_NONE; + ''' + ), + ]) diff --git a/pypy/module/cpyext/test/test_dictobject.py b/pypy/module/cpyext/test/test_dictobject.py --- a/pypy/module/cpyext/test/test_dictobject.py +++ b/pypy/module/cpyext/test/test_dictobject.py @@ -146,7 +146,7 @@ def test_dictproxy(self, space, api): w_dict = space.sys.get('modules') w_proxy = api.PyDictProxy_New(w_dict) - assert space.is_true(space.contains(w_proxy, space.wrap('sys'))) + assert space.contains_w(w_proxy, space.wrap('sys')) raises(OperationError, space.setitem, w_proxy, space.wrap('sys'), space.w_None) raises(OperationError, space.delitem, diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -45,3 +45,35 @@ ]) assert module.from_string() == 1234.56 assert type(module.from_string()) is float + +class AppTestFloatMacros(AppTestCpythonExtensionBase): + def test_return_nan(self): + import math + + module = self.import_extension('foo', [ + ("return_nan", "METH_NOARGS", + "Py_RETURN_NAN;"), + ]) + assert math.isnan(module.return_nan()) + + def test_return_inf(self): + import math + + module = self.import_extension('foo', [ + ("return_inf", "METH_NOARGS", + "Py_RETURN_INF(10);"), + ]) + inf = module.return_inf() + assert inf > 0 + assert math.isinf(inf) + + def test_return_inf_negative(self): + import math + + module = self.import_extension('foo', [ + ("return_neginf", "METH_NOARGS", + "Py_RETURN_INF(-10);"), + ]) + neginf = module.return_neginf() + assert neginf < 0 + assert math.isinf(neginf) diff --git a/pypy/module/cpyext/test/test_import.py b/pypy/module/cpyext/test/test_import.py --- a/pypy/module/cpyext/test/test_import.py +++ b/pypy/module/cpyext/test/test_import.py @@ -21,7 +21,7 @@ def test_getmoduledict(self, space, api): testmod = "_functools" w_pre_dict = api.PyImport_GetModuleDict() - assert not space.is_true(space.contains(w_pre_dict, space.wrap(testmod))) + assert not space.contains_w(w_pre_dict, space.wrap(testmod)) with rffi.scoped_str2charp(testmod) as modname: w_module = api.PyImport_ImportModule(modname) @@ -29,7 +29,7 @@ assert w_module w_dict = api.PyImport_GetModuleDict() - assert space.is_true(space.contains(w_dict, space.wrap(testmod))) + assert space.contains_w(w_dict, space.wrap(testmod)) def test_reload(self, space, api): stat = api.PyImport_Import(space.wrap("stat")) diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py --- a/pypy/module/cpyext/test/test_intobject.py +++ b/pypy/module/cpyext/test/test_intobject.py @@ -99,6 +99,7 @@ """), ], prologue=""" _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit