Author: Nikolay Zinov <nzi...@gmail.com> Branch: Changeset: r82187:e418c04f44ad Date: 2016-02-12 19:28 +0300 http://bitbucket.org/pypy/pypy/changeset/e418c04f44ad/
Log: merge default diff too long, truncating to 2000 out of 2205 lines diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.5.0 +Version: 1.5.1 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.5.0" -__version_info__ = (1, 5, 0) +__version__ = "1.5.1" +__version_info__ = (1, 5, 1) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/_cffi_include.h b/lib_pypy/cffi/_cffi_include.h --- a/lib_pypy/cffi/_cffi_include.h +++ b/lib_pypy/cffi/_cffi_include.h @@ -231,6 +231,12 @@ ((got_nonpos) == (expected <= 0) && \ (got) == (unsigned long long)expected) +#ifdef MS_WIN32 +# define _cffi_stdcall __stdcall +#else +# define _cffi_stdcall /* nothing */ +#endif + #ifdef __cplusplus } #endif diff --git a/lib_pypy/cffi/_embedding.h b/lib_pypy/cffi/_embedding.h new file mode 100644 --- /dev/null +++ b/lib_pypy/cffi/_embedding.h @@ -0,0 +1,517 @@ + +/***** Support code for embedding *****/ + +#if defined(_MSC_VER) +# define CFFI_DLLEXPORT __declspec(dllexport) +#elif defined(__GNUC__) +# define CFFI_DLLEXPORT __attribute__((visibility("default"))) +#else +# define CFFI_DLLEXPORT /* nothing */ +#endif + + +/* There are two global variables of type _cffi_call_python_fnptr: + + * _cffi_call_python, which we declare just below, is the one called + by ``extern "Python"`` implementations. + + * _cffi_call_python_org, which on CPython is actually part of the + _cffi_exports[] array, is the function pointer copied from + _cffi_backend. + + After initialization is complete, both are equal. However, the + first one remains equal to &_cffi_start_and_call_python until the + very end of initialization, when we are (or should be) sure that + concurrent threads also see a completely initialized world, and + only then is it changed. +*/ +#undef _cffi_call_python +typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *); +static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *); +static _cffi_call_python_fnptr _cffi_call_python = &_cffi_start_and_call_python; + + +#ifndef _MSC_VER + /* --- Assuming a GCC not infinitely old --- */ +# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n) +# define cffi_write_barrier() __sync_synchronize() +# if !defined(__amd64__) && !defined(__x86_64__) && \ + !defined(__i386__) && !defined(__i386) +# define cffi_read_barrier() __sync_synchronize() +# else +# define cffi_read_barrier() (void)0 +# endif +#else + /* --- Windows threads version --- */ +# include <Windows.h> +# define cffi_compare_and_swap(l,o,n) \ + (InterlockedCompareExchangePointer(l,n,o) == (o)) +# define cffi_write_barrier() InterlockedCompareExchange(&_cffi_dummy,0,0) +# define cffi_read_barrier() (void)0 +static volatile LONG _cffi_dummy; +#endif + +#ifdef WITH_THREAD +# ifndef _MSC_VER +# include <pthread.h> + static pthread_mutex_t _cffi_embed_startup_lock; +# else + static CRITICAL_SECTION _cffi_embed_startup_lock; +# endif + static char _cffi_embed_startup_lock_ready = 0; +#endif + +static void _cffi_acquire_reentrant_mutex(void) +{ + static void *volatile lock = NULL; + + while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) { + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: pthread_mutex_init() should be very fast, and + this is only run at start-up anyway. */ + } + +#ifdef WITH_THREAD + if (!_cffi_embed_startup_lock_ready) { +# ifndef _MSC_VER + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + pthread_mutex_init(&_cffi_embed_startup_lock, &attr); +# else + InitializeCriticalSection(&_cffi_embed_startup_lock); +# endif + _cffi_embed_startup_lock_ready = 1; + } +#endif + + while (!cffi_compare_and_swap(&lock, (void *)1, NULL)) + ; + +#ifndef _MSC_VER + pthread_mutex_lock(&_cffi_embed_startup_lock); +#else + EnterCriticalSection(&_cffi_embed_startup_lock); +#endif +} + +static void _cffi_release_reentrant_mutex(void) +{ +#ifndef _MSC_VER + pthread_mutex_unlock(&_cffi_embed_startup_lock); +#else + LeaveCriticalSection(&_cffi_embed_startup_lock); +#endif +} + + +/********** CPython-specific section **********/ +#ifndef PYPY_VERSION + + +#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX] + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */ + +static void _cffi_py_initialize(void) +{ + /* XXX use initsigs=0, which "skips initialization registration of + signal handlers, which might be useful when Python is + embedded" according to the Python docs. But review and think + if it should be a user-controllable setting. + + XXX we should also give a way to write errors to a buffer + instead of to stderr. + + XXX if importing 'site' fails, CPython (any version) calls + exit(). Should we try to work around this behavior here? + */ + Py_InitializeEx(0); +} + +static int _cffi_initialize_python(void) +{ + /* This initializes Python, imports _cffi_backend, and then the + present .dll/.so is set up as a CPython C extension module. + */ + int result; + PyGILState_STATE state; + PyObject *pycode=NULL, *global_dict=NULL, *x; + +#if PY_MAJOR_VERSION >= 3 + /* see comments in _cffi_carefully_make_gil() about the + Python2/Python3 difference + */ +#else + /* Acquire the GIL. We have no threadstate here. If Python is + already initialized, it is possible that there is already one + existing for this thread, but it is not made current now. + */ + PyEval_AcquireLock(); + + _cffi_py_initialize(); + + /* The Py_InitializeEx() sometimes made a threadstate for us, but + not always. Indeed Py_InitializeEx() could be called and do + nothing. So do we have a threadstate, or not? We don't know, + but we can replace it with NULL in all cases. + */ + (void)PyThreadState_Swap(NULL); + + /* Now we can release the GIL and re-acquire immediately using the + logic of PyGILState(), which handles making or installing the + correct threadstate. + */ + PyEval_ReleaseLock(); +#endif + state = PyGILState_Ensure(); + + /* Call the initxxx() function from the present module. It will + create and initialize us as a CPython extension module, instead + of letting the startup Python code do it---it might reimport + the same .dll/.so and get maybe confused on some platforms. + It might also have troubles locating the .dll/.so again for all + I know. + */ + (void)_CFFI_PYTHON_STARTUP_FUNC(); + if (PyErr_Occurred()) + goto error; + + /* Now run the Python code provided to ffi.embedding_init_code(). + */ + pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE, + "<init code for '" _CFFI_MODULE_NAME "'>", + Py_file_input); + if (pycode == NULL) + goto error; + global_dict = PyDict_New(); + if (global_dict == NULL) + goto error; + if (PyDict_SetItemString(global_dict, "__builtins__", + PyThreadState_GET()->interp->builtins) < 0) + goto error; + x = PyEval_EvalCode( +#if PY_MAJOR_VERSION < 3 + (PyCodeObject *) +#endif + pycode, global_dict, global_dict); + if (x == NULL) + goto error; + Py_DECREF(x); + + /* Done! Now if we've been called from + _cffi_start_and_call_python() in an ``extern "Python"``, we can + only hope that the Python code did correctly set up the + corresponding @ffi.def_extern() function. Otherwise, the + general logic of ``extern "Python"`` functions (inside the + _cffi_backend module) will find that the reference is still + missing and print an error. + */ + result = 0; + done: + Py_XDECREF(pycode); + Py_XDECREF(global_dict); + PyGILState_Release(state); + return result; + + error:; + { + /* Print as much information as potentially useful. + Debugging load-time failures with embedding is not fun + */ + PyObject *exception, *v, *tb, *f, *modules, *mod; + PyErr_Fetch(&exception, &v, &tb); + if (exception != NULL) { + PyErr_NormalizeException(&exception, &v, &tb); + PyErr_Display(exception, v, tb); + } + Py_XDECREF(exception); + Py_XDECREF(v); + Py_XDECREF(tb); + + f = PySys_GetObject((char *)"stderr"); + if (f != NULL && f != Py_None) { + PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME + "\ncompiled with cffi version: 1.5.1" + "\n_cffi_backend module: ", f); + modules = PyImport_GetModuleDict(); + mod = PyDict_GetItemString(modules, "_cffi_backend"); + if (mod == NULL) { + PyFile_WriteString("not loaded", f); + } + else { + v = PyObject_GetAttrString(mod, "__file__"); + PyFile_WriteObject(v, f, 0); + Py_XDECREF(v); + } + PyFile_WriteString("\nsys.path: ", f); + PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0); + PyFile_WriteString("\n\n", f); + } + } + result = -1; + goto done; +} + +PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */ + +static int _cffi_carefully_make_gil(void) +{ + /* This does the basic initialization of Python. It can be called + completely concurrently from unrelated threads. It assumes + that we don't hold the GIL before (if it exists), and we don't + hold it afterwards. + + What it really does is completely different in Python 2 and + Python 3. + + Python 2 + ======== + + Initialize the GIL, without initializing the rest of Python, + by calling PyEval_InitThreads(). + + PyEval_InitThreads() must not be called concurrently at all. + So we use a global variable as a simple spin lock. This global + variable must be from 'libpythonX.Y.so', not from this + cffi-based extension module, because it must be shared from + different cffi-based extension modules. We choose + _PyParser_TokenNames[0] as a completely arbitrary pointer value + that is never written to. The default is to point to the + string "ENDMARKER". We change it temporarily to point to the + next character in that string. (Yes, I know it's REALLY + obscure.) + + Python 3 + ======== + + In Python 3, PyEval_InitThreads() cannot be called before + Py_InitializeEx() any more. So this function calls + Py_InitializeEx() first. It uses the same obscure logic to + make sure we never call it concurrently. + + Arguably, this is less good on the spinlock, because + Py_InitializeEx() takes much longer to run than + PyEval_InitThreads(). But I didn't find a way around it. + */ + +#ifdef WITH_THREAD + char *volatile *lock = (char *volatile *)_PyParser_TokenNames; + char *old_value; + + while (1) { /* spin loop */ + old_value = *lock; + if (old_value[0] == 'E') { + assert(old_value[1] == 'N'); + if (cffi_compare_and_swap(lock, old_value, old_value + 1)) + break; + } + else { + assert(old_value[0] == 'N'); + /* should ideally do a spin loop instruction here, but + hard to do it portably and doesn't really matter I + think: PyEval_InitThreads() should be very fast, and + this is only run at start-up anyway. */ + } + } +#endif + +#if PY_MAJOR_VERSION >= 3 + /* Python 3: call Py_InitializeEx() */ + { + PyGILState_STATE state = PyGILState_UNLOCKED; + if (!Py_IsInitialized()) + _cffi_py_initialize(); + else + state = PyGILState_Ensure(); + + PyEval_InitThreads(); + PyGILState_Release(state); + } +#else + /* Python 2: call PyEval_InitThreads() */ +# ifdef WITH_THREAD + if (!PyEval_ThreadsInitialized()) { + PyEval_InitThreads(); /* makes the GIL */ + PyEval_ReleaseLock(); /* then release it */ + } + /* else: there is already a GIL, but we still needed to do the + spinlock dance to make sure that we see it as fully ready */ +# endif +#endif + +#ifdef WITH_THREAD + /* release the lock */ + while (!cffi_compare_and_swap(lock, old_value + 1, old_value)) + ; +#endif + + return 0; +} + +/********** end CPython-specific section **********/ + + +#else + + +/********** PyPy-specific section **********/ + +PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */ + +static struct _cffi_pypy_init_s { + const char *name; + void (*func)(const void *[]); + const char *code; +} _cffi_pypy_init = { + _CFFI_MODULE_NAME, + _CFFI_PYTHON_STARTUP_FUNC, + _CFFI_PYTHON_STARTUP_CODE, +}; + +extern int pypy_carefully_make_gil(const char *); +extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *); + +static int _cffi_carefully_make_gil(void) +{ + return pypy_carefully_make_gil(_CFFI_MODULE_NAME); +} + +static int _cffi_initialize_python(void) +{ + return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init); +} + +/********** end PyPy-specific section **********/ + + +#endif + + +#ifdef __GNUC__ +__attribute__((noinline)) +#endif +static _cffi_call_python_fnptr _cffi_start_python(void) +{ + /* Delicate logic to initialize Python. This function can be + called multiple times concurrently, e.g. when the process calls + its first ``extern "Python"`` functions in multiple threads at + once. It can also be called recursively, in which case we must + ignore it. We also have to consider what occurs if several + different cffi-based extensions reach this code in parallel + threads---it is a different copy of the code, then, and we + can't have any shared global variable unless it comes from + 'libpythonX.Y.so'. + + Idea: + + * _cffi_carefully_make_gil(): "carefully" call + PyEval_InitThreads() (possibly with Py_InitializeEx() first). + + * then we use a (local) custom lock to make sure that a call to this + cffi-based extension will wait if another call to the *same* + extension is running the initialization in another thread. + It is reentrant, so that a recursive call will not block, but + only one from a different thread. + + * then we grab the GIL and (Python 2) we call Py_InitializeEx(). + At this point, concurrent calls to Py_InitializeEx() are not + possible: we have the GIL. + + * do the rest of the specific initialization, which may + temporarily release the GIL but not the custom lock. + Only release the custom lock when we are done. + */ + static char called = 0; + + if (_cffi_carefully_make_gil() != 0) + return NULL; + + _cffi_acquire_reentrant_mutex(); + + /* Here the GIL exists, but we don't have it. We're only protected + from concurrency by the reentrant mutex. */ + + /* This file only initializes the embedded module once, the first + time this is called, even if there are subinterpreters. */ + if (!called) { + called = 1; /* invoke _cffi_initialize_python() only once, + but don't set '_cffi_call_python' right now, + otherwise concurrent threads won't call + this function at all (we need them to wait) */ + if (_cffi_initialize_python() == 0) { + /* now initialization is finished. Switch to the fast-path. */ + + /* We would like nobody to see the new value of + '_cffi_call_python' without also seeing the rest of the + data initialized. However, this is not possible. But + the new value of '_cffi_call_python' is the function + 'cffi_call_python()' from _cffi_backend. So: */ + cffi_write_barrier(); + /* ^^^ we put a write barrier here, and a corresponding + read barrier at the start of cffi_call_python(). This + ensures that after that read barrier, we see everything + done here before the write barrier. + */ + + assert(_cffi_call_python_org != NULL); + _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org; + } + else { + /* initialization failed. Reset this to NULL, even if it was + already set to some other value. Future calls to + _cffi_start_python() are still forced to occur, and will + always return NULL from now on. */ + _cffi_call_python_org = NULL; + } + } + + _cffi_release_reentrant_mutex(); + + return (_cffi_call_python_fnptr)_cffi_call_python_org; +} + +static +void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args) +{ + _cffi_call_python_fnptr fnptr; + int current_err = errno; +#ifdef _MSC_VER + int current_lasterr = GetLastError(); +#endif + fnptr = _cffi_start_python(); + if (fnptr == NULL) { + fprintf(stderr, "function %s() called, but initialization code " + "failed. Returning 0.\n", externpy->name); + memset(args, 0, externpy->size_of_result); + } +#ifdef _MSC_VER + SetLastError(current_lasterr); +#endif + errno = current_err; + + if (fnptr != NULL) + fnptr(externpy, args); +} + + +/* The cffi_start_python() function makes sure Python is initialized + and our cffi module is set up. It can be called manually from the + user C code. The same effect is obtained automatically from any + dll-exported ``extern "Python"`` function. This function returns + -1 if initialization failed, 0 if all is OK. */ +_CFFI_UNUSED_FN +static int cffi_start_python(void) +{ + if (_cffi_call_python == &_cffi_start_and_call_python) { + if (_cffi_start_python() == NULL) + return -1; + } + cffi_read_barrier(); + return 0; +} + +#undef cffi_compare_and_swap +#undef cffi_write_barrier +#undef cffi_read_barrier diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -1,4 +1,4 @@ -import sys, types +import sys, sysconfig, types from .lock import allocate_lock try: @@ -544,28 +544,32 @@ def _apply_embedding_fix(self, kwds): # must include an argument like "-lpython2.7" for the compiler + def ensure(key, value): + lst = kwds.setdefault(key, []) + if value not in lst: + lst.append(value) + # if '__pypy__' in sys.builtin_module_names: if hasattr(sys, 'prefix'): import os - libdir = os.path.join(sys.prefix, 'bin') - dirs = kwds.setdefault('library_dirs', []) - if libdir not in dirs: - dirs.append(libdir) + ensure('library_dirs', os.path.join(sys.prefix, 'bin')) pythonlib = "pypy-c" else: if sys.platform == "win32": template = "python%d%d" - if sys.flags.debug: - template = template + '_d' + if hasattr(sys, 'gettotalrefcount'): + template += '_d' else: template = "python%d.%d" + if sysconfig.get_config_var('DEBUG_EXT'): + template += sysconfig.get_config_var('DEBUG_EXT') pythonlib = (template % (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) if hasattr(sys, 'abiflags'): pythonlib += sys.abiflags - libraries = kwds.setdefault('libraries', []) - if pythonlib not in libraries: - libraries.append(pythonlib) + ensure('libraries', pythonlib) + if sys.platform == "win32": + ensure('extra_link_args', '/MANIFEST') def set_source(self, module_name, source, source_extension='.c', **kwds): if hasattr(self, '_assigned_source'): @@ -631,7 +635,7 @@ compiled DLL. Use '*' to force distutils' choice, suitable for regular CPython C API modules. Use a file name ending in '.*' to ask for the system's default extension for dynamic libraries - (.so/.dll). + (.so/.dll/.dylib). The default is '*' when building a non-embedded C API extension, and (module_name + '.*') when building an embedded library. @@ -695,6 +699,10 @@ # self._embedding = pysource + def def_extern(self, *args, **kwds): + raise ValueError("ffi.def_extern() is only available on API-mode FFI " + "objects") + def _load_backend_lib(backend, name, flags): if name is None: diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -220,7 +220,7 @@ self._included_declarations = set() self._anonymous_counter = 0 self._structnode2type = weakref.WeakKeyDictionary() - self._options = None + self._options = {} self._int_constants = {} self._recomplete = [] self._uses_new_feature = None @@ -374,7 +374,7 @@ def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) - if self._options['dllexport']: + if self._options.get('dllexport'): tag = 'dllexport_python ' elif self._inside_extern_python: tag = 'extern_python ' @@ -450,7 +450,7 @@ prevobj, prevquals = self._declarations[name] if prevobj is obj and prevquals == quals: return - if not self._options['override']: + if not self._options.get('override'): raise api.FFIError( "multiple declarations of %s (for interactive usage, " "try cdef(xx, override=True))" % (name,)) @@ -729,7 +729,7 @@ if isinstance(tp, model.StructType) and tp.partial: raise NotImplementedError("%s: using both bitfields and '...;'" % (tp,)) - tp.packed = self._options['packed'] + tp.packed = self._options.get('packed') if tp.completed: # must be re-completed: it is not opaque any more tp.completed = 0 self._recomplete.append(tp) diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -21,14 +21,12 @@ allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0, target_extension=None, - embedding=False): +def compile(tmpdir, ext, compiler_verbose=0): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose, - target_extension, embedding) + outputfilename = _build(tmpdir, ext, compiler_verbose) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -38,32 +36,7 @@ os.environ[key] = value return outputfilename -def _save_val(name): - import distutils.sysconfig - config_vars = distutils.sysconfig.get_config_vars() - return config_vars.get(name, Ellipsis) - -def _restore_val(name, value): - import distutils.sysconfig - config_vars = distutils.sysconfig.get_config_vars() - config_vars[name] = value - if value is Ellipsis: - del config_vars[name] - -def _win32_hack_for_embedding(): - from distutils.msvc9compiler import MSVCCompiler - if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'): - MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \ - MSVCCompiler._remove_visual_c_ref - MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file: manifest_file - -def _win32_unhack_for_embedding(): - from distutils.msvc9compiler import MSVCCompiler - MSVCCompiler._remove_visual_c_ref = \ - MSVCCompiler._remove_visual_c_ref_CFFI_BAK - -def _build(tmpdir, ext, compiler_verbose=0, target_extension=None, - embedding=False): +def _build(tmpdir, ext, compiler_verbose=0): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -76,25 +49,14 @@ options['build_temp'] = ('ffiplatform', tmpdir) # try: - if sys.platform == 'win32' and embedding: - _win32_hack_for_embedding() old_level = distutils.log.set_threshold(0) or 0 - old_SO = _save_val('SO') - old_EXT_SUFFIX = _save_val('EXT_SUFFIX') try: - if target_extension is not None: - _restore_val('SO', target_extension) - _restore_val('EXT_SUFFIX', target_extension) distutils.log.set_verbosity(compiler_verbose) dist.run_command('build_ext') cmd_obj = dist.get_command_obj('build_ext') [soname] = cmd_obj.get_outputs() finally: distutils.log.set_threshold(old_level) - _restore_val('SO', old_SO) - _restore_val('EXT_SUFFIX', old_EXT_SUFFIX) - if sys.platform == 'win32' and embedding: - _win32_unhack_for_embedding() except (distutils.errors.CompileError, distutils.errors.LinkError) as e: raise VerificationError('%s: %s' % (e.__class__.__name__, e)) diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -1170,6 +1170,8 @@ repr_arguments = ', '.join(arguments) repr_arguments = repr_arguments or 'void' name_and_arguments = '%s(%s)' % (name, repr_arguments) + if tp.abi == "__stdcall": + name_and_arguments = '_cffi_stdcall ' + name_and_arguments # def may_need_128_bits(tp): return (isinstance(tp, model.PrimitiveType) and @@ -1357,6 +1359,58 @@ parts[-1] += extension return os.path.join(outputdir, *parts), parts + +# Aaargh. Distutils is not tested at all for the purpose of compiling +# DLLs that are not extension modules. Here are some hacks to work +# around that, in the _patch_for_*() functions... + +def _patch_meth(patchlist, cls, name, new_meth): + old = getattr(cls, name) + patchlist.append((cls, name, old)) + setattr(cls, name, new_meth) + return old + +def _unpatch_meths(patchlist): + for cls, name, old_meth in reversed(patchlist): + setattr(cls, name, old_meth) + +def _patch_for_embedding(patchlist): + if sys.platform == 'win32': + # we must not remove the manifest when building for embedding! + from distutils.msvc9compiler import MSVCCompiler + _patch_meth(patchlist, MSVCCompiler, '_remove_visual_c_ref', + lambda self, manifest_file: manifest_file) + + if sys.platform == 'darwin': + # we must not make a '-bundle', but a '-dynamiclib' instead + from distutils.ccompiler import CCompiler + def my_link_shared_object(self, *args, **kwds): + if '-bundle' in self.linker_so: + self.linker_so = list(self.linker_so) + i = self.linker_so.index('-bundle') + self.linker_so[i] = '-dynamiclib' + return old_link_shared_object(self, *args, **kwds) + old_link_shared_object = _patch_meth(patchlist, CCompiler, + 'link_shared_object', + my_link_shared_object) + +def _patch_for_target(patchlist, target): + from distutils.command.build_ext import build_ext + # if 'target' is different from '*', we need to patch some internal + # method to just return this 'target' value, instead of having it + # built from module_name + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + elif sys.platform == 'darwin': + target += '.dylib' + else: + target += '.so' + _patch_meth(patchlist, build_ext, 'get_ext_filename', + lambda self, ext_name: target) + + def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, compiler_verbose=1, target=None, **kwds): @@ -1382,36 +1436,22 @@ target = '%s.*' % module_name else: target = '*' - if target == '*': - target_module_name = module_name - target_extension = None # use default - else: - if target.endswith('.*'): - target = target[:-2] - if sys.platform == 'win32': - target += '.dll' - else: - target += '.so' - # split along the first '.' (not the last one, otherwise the - # preceeding dots are interpreted as splitting package names) - index = target.find('.') - if index < 0: - raise ValueError("target argument %r should be a file name " - "containing a '.'" % (target,)) - target_module_name = target[:index] - target_extension = target[index:] # - ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds) + ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) updated = make_c_source(ffi, module_name, preamble, c_file) if call_c_compiler: + patchlist = [] cwd = os.getcwd() try: + if embedding: + _patch_for_embedding(patchlist) + if target != '*': + _patch_for_target(patchlist, target) os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose, - target_extension, - embedding=embedding) + outputfilename = ffiplatform.compile('.', ext, compiler_verbose) finally: os.chdir(cwd) + _unpatch_meths(patchlist) return outputfilename else: return ext, updated diff --git a/pypy/doc/whatsnew-head.rst b/pypy/doc/whatsnew-head.rst --- a/pypy/doc/whatsnew-head.rst +++ b/pypy/doc/whatsnew-head.rst @@ -142,4 +142,9 @@ .. branch: vmprof-newstack -Refactor vmprof to work cross-operating-system. \ No newline at end of file +Refactor vmprof to work cross-operating-system. + +.. branch: seperate-strucmember_h + +Seperate structmember.h from Python.h Also enhance creating api functions +to specify which header file they appear in (previously only pypy_decl.h) diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.5.0" +VERSION = "1.5.1" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: @@ -69,6 +69,7 @@ def startup(self, space): from pypy.module._cffi_backend import embedding embedding.glob.space = space + embedding.glob.patched_sys = False def get_dict_rtld_constants(): diff --git a/pypy/module/_cffi_backend/embedding.py b/pypy/module/_cffi_backend/embedding.py --- a/pypy/module/_cffi_backend/embedding.py +++ b/pypy/module/_cffi_backend/embedding.py @@ -45,6 +45,26 @@ pass glob = Global() +def patch_sys(space): + # Annoying: CPython would just use the C-level std{in,out,err} as + # configured by the main application, for example in binary mode + # on Windows or with buffering turned off. We can't easily do the + # same. Instead, go for the safest bet (but possibly bad for + # performance) and open sys.std{in,out,err} unbuffered. On + # Windows I guess binary mode is a better default choice. + # + # XXX if needed, we could add support for a flag passed to + # pypy_init_embedded_cffi_module(). + if not glob.patched_sys: + space.appexec([], """(): + import os + sys.stdin = sys.__stdin__ = os.fdopen(0, 'rb', 0) + sys.stdout = sys.__stdout__ = os.fdopen(1, 'wb', 0) + sys.stderr = sys.__stderr__ = os.fdopen(2, 'wb', 0) + """) + glob.patched_sys = True + + def pypy_init_embedded_cffi_module(version, init_struct): # called from __init__.py name = "?" @@ -56,6 +76,7 @@ must_leave = False try: must_leave = space.threadlocals.try_enter_thread(space) + patch_sys(space) load_embedded_cffi_module(space, version, init_struct) res = 0 except OperationError, operr: @@ -84,72 +105,87 @@ return rffi.cast(rffi.INT, res) # ____________________________________________________________ + if os.name == 'nt': - do_startup = r''' -#include <stdio.h> -#define WIN32_LEAN_AND_MEAN + + do_includes = r""" +#define _WIN32_WINNT 0x0501 #include <windows.h> -RPY_EXPORTED void rpython_startup_code(void); -RPY_EXPORTED int pypy_setup_home(char *, int); -static unsigned char _cffi_ready = 0; -static const char *volatile _cffi_module_name; +#define CFFI_INIT_HOME_PATH_MAX _MAX_PATH +static void _cffi_init(void); +static void _cffi_init_error(const char *msg, const char *extra); -static void _cffi_init_error(const char *msg, const char *extra) +static int _cffi_init_home(char *output_home_path) { - fprintf(stderr, - "\nPyPy initialization failure when loading module '%s':\n%s%s\n", - _cffi_module_name, msg, extra); -} - -BOOL CALLBACK _cffi_init(PINIT_ONCE InitOnce, PVOID Parameter, PVOID *lpContex) -{ - - HMODULE hModule; - TCHAR home[_MAX_PATH]; - rpython_startup_code(); - RPyGilAllocate(); + HMODULE hModule = 0; + DWORD res; GetModuleHandleEx(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, (LPCTSTR)&_cffi_init, &hModule); + if (hModule == 0 ) { - /* TODO turn the int into a string with FormatMessage */ - - _cffi_init_error("dladdr() failed: ", ""); - return TRUE; + _cffi_init_error("GetModuleHandleEx() failed", ""); + return -1; } - GetModuleFileName(hModule, home, _MAX_PATH); - if (pypy_setup_home(home, 1) != 0) { - _cffi_init_error("pypy_setup_home() failed", ""); - return TRUE; + res = GetModuleFileName(hModule, output_home_path, CFFI_INIT_HOME_PATH_MAX); + if (res >= CFFI_INIT_HOME_PATH_MAX) { + return -1; } - _cffi_ready = 1; - fprintf(stderr, "startup succeeded, home %s\n", home); - return TRUE; + return 0; } -RPY_EXPORTED -int pypy_carefully_make_gil(const char *name) +static void _cffi_init_once(void) { - /* For CFFI: this initializes the GIL and loads the home path. - It can be called completely concurrently from unrelated threads. - It assumes that we don't hold the GIL before (if it exists), and we - don't hold it afterwards. - */ - static INIT_ONCE s_init_once; + static LONG volatile lock = 0; + static int _init_called = 0; - _cffi_module_name = name; /* not really thread-safe, but better than - nothing */ - InitOnceExecuteOnce(&s_init_once, _cffi_init, NULL, NULL); - return (int)_cffi_ready - 1; -}''' + while (InterlockedCompareExchange(&lock, 1, 0) != 0) { + SwitchToThread(); /* spin loop */ + } + if (!_init_called) { + _cffi_init(); + _init_called = 1; + } + InterlockedCompareExchange(&lock, 0, 1); +} +""" + else: - do_startup = r""" -#include <stdio.h> + + do_includes = r""" #include <dlfcn.h> #include <pthread.h> +#define CFFI_INIT_HOME_PATH_MAX PATH_MAX +static void _cffi_init(void); +static void _cffi_init_error(const char *msg, const char *extra); + +static int _cffi_init_home(char *output_home_path) +{ + Dl_info info; + dlerror(); /* reset */ + if (dladdr(&_cffi_init, &info) == 0) { + _cffi_init_error("dladdr() failed: ", dlerror()); + return -1; + } + if (realpath(info.dli_fname, output_home_path) == NULL) { + perror("realpath() failed"); + _cffi_init_error("realpath() failed", ""); + return -1; + } + return 0; +} + +static void _cffi_init_once(void) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + pthread_once(&once_control, _cffi_init); +} +""" + +do_startup = do_includes + r""" RPY_EXPORTED void rpython_startup_code(void); RPY_EXPORTED int pypy_setup_home(char *, int); @@ -165,17 +201,13 @@ static void _cffi_init(void) { - Dl_info info; - char *home; + char home[CFFI_INIT_HOME_PATH_MAX + 1]; rpython_startup_code(); RPyGilAllocate(); - if (dladdr(&_cffi_init, &info) == 0) { - _cffi_init_error("dladdr() failed: ", dlerror()); + if (_cffi_init_home(home) != 0) return; - } - home = realpath(info.dli_fname, NULL); if (pypy_setup_home(home, 1) != 0) { _cffi_init_error("pypy_setup_home() failed", ""); return; @@ -191,11 +223,9 @@ It assumes that we don't hold the GIL before (if it exists), and we don't hold it afterwards. */ - static pthread_once_t once_control = PTHREAD_ONCE_INIT; - _cffi_module_name = name; /* not really thread-safe, but better than nothing */ - pthread_once(&once_control, _cffi_init); + _cffi_init_once(); return (int)_cffi_ready - 1; } """ diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.5.0", ("This test_c.py file is for testing a version" +assert __version__ == "1.5.1", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/_cffi_backend/test/test_ztranslation.py b/pypy/module/_cffi_backend/test/test_ztranslation.py --- a/pypy/module/_cffi_backend/test/test_ztranslation.py +++ b/pypy/module/_cffi_backend/test/test_ztranslation.py @@ -4,15 +4,18 @@ # side-effect: FORMAT_LONGDOUBLE must be built before test_checkmodule() from pypy.module._cffi_backend import misc -from pypy.module._cffi_backend import cffi1_module +from pypy.module._cffi_backend import embedding def test_checkmodule(): # prepare_file_argument() is not working without translating the _file # module too def dummy_prepare_file_argument(space, fileobj): - # call load_cffi1_module() too, from a random place like here - cffi1_module.load_cffi1_module(space, "foo", "foo", 42) + # call pypy_init_embedded_cffi_module() from a random place like here + # --- this calls load_cffi1_module(), too + embedding.pypy_init_embedded_cffi_module( + rffi.cast(rffi.INT, embedding.EMBED_VERSION_MIN), + 42) return lltype.nullptr(rffi.CCHARP.TO) old = ctypeptr.prepare_file_argument try: diff --git a/pypy/module/cpyext/api.py b/pypy/module/cpyext/api.py --- a/pypy/module/cpyext/api.py +++ b/pypy/module/cpyext/api.py @@ -59,7 +59,7 @@ class CConfig: _compilation_info_ = ExternalCompilationInfo( include_dirs=include_dirs, - includes=['Python.h', 'stdarg.h'], + includes=['Python.h', 'stdarg.h', 'structmember.h'], compile_extra=['-DPy_BUILD_CORE'], ) @@ -129,6 +129,7 @@ for name in constant_names: setattr(CConfig_constants, name, rffi_platform.ConstantInteger(name)) udir.join('pypy_decl.h').write("/* Will be filled later */\n") +udir.join('pypy_structmember_decl.h').write("/* Will be filled later */\n") udir.join('pypy_macros.h').write("/* Will be filled later */\n") globals().update(rffi_platform.configure(CConfig_constants)) @@ -147,7 +148,7 @@ # XXX: 20 lines of code to recursively copy a directory, really?? assert dstdir.check(dir=True) headers = include_dir.listdir('*.h') + include_dir.listdir('*.inl') - for name in ("pypy_decl.h", "pypy_macros.h"): + for name in ("pypy_decl.h", "pypy_macros.h", "pypy_structmember_decl.h"): headers.append(udir.join(name)) _copy_header_files(headers, dstdir) @@ -232,7 +233,7 @@ wrapper.c_name = cpyext_namespace.uniquename(self.c_name) return wrapper -def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, external=True, +def cpython_api(argtypes, restype, error=_NOT_SPECIFIED, header='pypy_decl.h', gil=None): """ Declares a function to be exported. @@ -241,8 +242,8 @@ special value 'CANNOT_FAIL' (also when restype is Void) turns an eventual exception into a wrapped SystemError. Unwrapped exceptions also cause a SytemError. - - set `external` to False to get a C function pointer, but not exported by - the API headers. + - `header` is the header file to export the function in, Set to None to get + a C function pointer, but not exported by the API headers. - set `gil` to "acquire", "release" or "around" to acquire the GIL, release the GIL, or both """ @@ -263,7 +264,7 @@ def decorate(func): func_name = func.func_name - if external: + if header is not None: c_name = None else: c_name = func_name @@ -271,7 +272,7 @@ c_name=c_name, gil=gil) func.api_func = api_function - if external: + if header is not None: assert func_name not in FUNCTIONS, ( "%s already registered" % func_name) @@ -363,8 +364,9 @@ unwrapper_catch = make_unwrapper(True) unwrapper_raise = make_unwrapper(False) - if external: + if header is not None: FUNCTIONS[func_name] = api_function + FUNCTIONS_BY_HEADER.setdefault(header, {})[func_name] = api_function INTERPLEVEL_API[func_name] = unwrapper_catch # used in tests return unwrapper_raise # used in 'normal' RPython code. return decorate @@ -383,6 +385,7 @@ INTERPLEVEL_API = {} FUNCTIONS = {} +FUNCTIONS_BY_HEADER = {} # These are C symbols which cpyext will export, but which are defined in .c # files somewhere in the implementation of cpyext (rather than being defined in @@ -811,6 +814,7 @@ global_code = '\n'.join(global_objects) prologue = ("#include <Python.h>\n" + "#include <structmember.h>\n" "#include <src/thread.c>\n") code = (prologue + struct_declaration_code + @@ -960,7 +964,8 @@ "NOT_RPYTHON" # implement function callbacks and generate function decls functions = [] - pypy_decls = [] + decls = {} + pypy_decls = decls['pypy_decl.h'] = [] pypy_decls.append("#ifndef _PYPY_PYPY_DECL_H\n") pypy_decls.append("#define _PYPY_PYPY_DECL_H\n") pypy_decls.append("#ifndef PYPY_STANDALONE\n") @@ -973,17 +978,23 @@ for decl in FORWARD_DECLS: pypy_decls.append("%s;" % (decl,)) - for name, func in sorted(FUNCTIONS.iteritems()): - restype, args = c_function_signature(db, func) - pypy_decls.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args)) - if api_struct: - callargs = ', '.join('arg%d' % (i,) - for i in range(len(func.argtypes))) - if func.restype is lltype.Void: - body = "{ _pypyAPI.%s(%s); }" % (name, callargs) - else: - body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) - functions.append('%s %s(%s)\n%s' % (restype, name, args, body)) + for header_name, header_functions in FUNCTIONS_BY_HEADER.iteritems(): + if header_name not in decls: + header = decls[header_name] = [] + else: + header = decls[header_name] + + for name, func in sorted(header_functions.iteritems()): + restype, args = c_function_signature(db, func) + header.append("PyAPI_FUNC(%s) %s(%s);" % (restype, name, args)) + if api_struct: + callargs = ', '.join('arg%d' % (i,) + for i in range(len(func.argtypes))) + if func.restype is lltype.Void: + body = "{ _pypyAPI.%s(%s); }" % (name, callargs) + else: + body = "{ return _pypyAPI.%s(%s); }" % (name, callargs) + functions.append('%s %s(%s)\n%s' % (restype, name, args, body)) for name in VA_TP_LIST: name_no_star = process_va_name(name) header = ('%s pypy_va_get_%s(va_list* vp)' % @@ -1007,8 +1018,9 @@ pypy_decls.append("#endif /*PYPY_STANDALONE*/\n") pypy_decls.append("#endif /*_PYPY_PYPY_DECL_H*/\n") - pypy_decl_h = udir.join('pypy_decl.h') - pypy_decl_h.write('\n'.join(pypy_decls)) + for header_name, header_decls in decls.iteritems(): + decl_h = udir.join(header_name) + decl_h.write('\n'.join(header_decls)) return functions separate_module_files = [source_dir / "varargwrapper.c", diff --git a/pypy/module/cpyext/bufferobject.py b/pypy/module/cpyext/bufferobject.py --- a/pypy/module/cpyext/bufferobject.py +++ b/pypy/module/cpyext/bufferobject.py @@ -73,7 +73,7 @@ "Don't know how to realize a buffer")) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def buffer_dealloc(space, py_obj): py_buf = rffi.cast(PyBufferObject, py_obj) if py_buf.c_b_base: diff --git a/pypy/module/cpyext/frameobject.py b/pypy/module/cpyext/frameobject.py --- a/pypy/module/cpyext/frameobject.py +++ b/pypy/module/cpyext/frameobject.py @@ -39,7 +39,7 @@ py_frame.c_f_locals = make_ref(space, frame.get_w_locals()) rffi.setintfield(py_frame, 'c_f_lineno', frame.getorcreatedebug().f_lineno) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def frame_dealloc(space, py_obj): py_frame = rffi.cast(PyFrameObject, py_obj) py_code = rffi.cast(PyObject, py_frame.c_f_code) diff --git a/pypy/module/cpyext/funcobject.py b/pypy/module/cpyext/funcobject.py --- a/pypy/module/cpyext/funcobject.py +++ b/pypy/module/cpyext/funcobject.py @@ -56,7 +56,7 @@ assert isinstance(w_obj, Function) py_func.c_func_name = make_ref(space, space.wrap(w_obj.name)) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def function_dealloc(space, py_obj): py_func = rffi.cast(PyFunctionObject, py_obj) Py_DecRef(space, py_func.c_func_name) @@ -75,7 +75,7 @@ rffi.setintfield(py_code, 'c_co_flags', co_flags) rffi.setintfield(py_code, 'c_co_argcount', w_obj.co_argcount) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def code_dealloc(space, py_obj): py_code = rffi.cast(PyCodeObject, py_obj) Py_DecRef(space, py_code.c_co_name) diff --git a/pypy/module/cpyext/include/Python.h b/pypy/module/cpyext/include/Python.h --- a/pypy/module/cpyext/include/Python.h +++ b/pypy/module/cpyext/include/Python.h @@ -84,6 +84,7 @@ #include "pyconfig.h" #include "object.h" +#include "pymath.h" #include "pyport.h" #include "warnings.h" @@ -115,7 +116,6 @@ #include "compile.h" #include "frameobject.h" #include "eval.h" -#include "pymath.h" #include "pymem.h" #include "pycobject.h" #include "pycapsule.h" @@ -132,9 +132,6 @@ /* Missing definitions */ #include "missing.h" -// XXX This shouldn't be included here -#include "structmember.h" - #include <pypy_decl.h> /* Define macros for inline documentation. */ diff --git a/pypy/module/cpyext/include/floatobject.h b/pypy/module/cpyext/include/floatobject.h --- a/pypy/module/cpyext/include/floatobject.h +++ b/pypy/module/cpyext/include/floatobject.h @@ -7,6 +7,18 @@ extern "C" { #endif +#define PyFloat_STR_PRECISION 12 + +#ifdef Py_NAN +#define Py_RETURN_NAN return PyFloat_FromDouble(Py_NAN) +#endif + +#define Py_RETURN_INF(sign) do \ + if (copysign(1., sign) == 1.) { \ + return PyFloat_FromDouble(Py_HUGE_VAL); \ + } else { \ + return PyFloat_FromDouble(-Py_HUGE_VAL); \ + } while(0) #ifdef __cplusplus } diff --git a/pypy/module/cpyext/include/pymath.h b/pypy/module/cpyext/include/pymath.h --- a/pypy/module/cpyext/include/pymath.h +++ b/pypy/module/cpyext/include/pymath.h @@ -17,4 +17,35 @@ #define Py_HUGE_VAL HUGE_VAL #endif +/* Py_NAN + * A value that evaluates to a NaN. On IEEE 754 platforms INF*0 or + * INF/INF works. Define Py_NO_NAN in pyconfig.h if your platform + * doesn't support NaNs. + */ +#if !defined(Py_NAN) && !defined(Py_NO_NAN) +#if !defined(__INTEL_COMPILER) + #define Py_NAN (Py_HUGE_VAL * 0.) +#else /* __INTEL_COMPILER */ + #if defined(ICC_NAN_STRICT) + #pragma float_control(push) + #pragma float_control(precise, on) + #pragma float_control(except, on) + #if defined(_MSC_VER) + __declspec(noinline) + #else /* Linux */ + __attribute__((noinline)) + #endif /* _MSC_VER */ + static double __icc_nan() + { + return sqrt(-1.0); + } + #pragma float_control (pop) + #define Py_NAN __icc_nan() + #else /* ICC_NAN_RELAXED as default for Intel Compiler */ + static union { unsigned char buf[8]; double __icc_nan; } __nan_store = {0,0,0,0,0,0,0xf8,0x7f}; + #define Py_NAN (__nan_store.__icc_nan) + #endif /* ICC_NAN_STRICT */ +#endif /* __INTEL_COMPILER */ +#endif + #endif /* Py_PYMATH_H */ diff --git a/pypy/module/cpyext/include/structmember.h b/pypy/module/cpyext/include/structmember.h --- a/pypy/module/cpyext/include/structmember.h +++ b/pypy/module/cpyext/include/structmember.h @@ -4,54 +4,85 @@ extern "C" { #endif + +/* Interface to map C struct members to Python object attributes */ + #include <stddef.h> /* For offsetof */ + +/* The offsetof() macro calculates the offset of a structure member + in its structure. Unfortunately this cannot be written down + portably, hence it is provided by a Standard C header file. + For pre-Standard C compilers, here is a version that usually works + (but watch out!): */ + #ifndef offsetof #define offsetof(type, member) ( (int) & ((type*)0) -> member ) #endif +/* An array of memberlist structures defines the name, type and offset + of selected members of a C structure. These can be read by + PyMember_Get() and set by PyMember_Set() (except if their READONLY flag + is set). The array must be terminated with an entry whose name + pointer is NULL. */ + + typedef struct PyMemberDef { - /* Current version, use this */ - char *name; - int type; - Py_ssize_t offset; - int flags; - char *doc; + /* Current version, use this */ + char *name; + int type; + Py_ssize_t offset; + int flags; + char *doc; } PyMemberDef; +/* Types */ +#define T_SHORT 0 +#define T_INT 1 +#define T_LONG 2 +#define T_FLOAT 3 +#define T_DOUBLE 4 +#define T_STRING 5 +#define T_OBJECT 6 +/* XXX the ordering here is weird for binary compatibility */ +#define T_CHAR 7 /* 1-character string */ +#define T_BYTE 8 /* 8-bit signed int */ +/* unsigned variants: */ +#define T_UBYTE 9 +#define T_USHORT 10 +#define T_UINT 11 +#define T_ULONG 12 -/* Types. These constants are also in structmemberdefs.py. */ -#define T_SHORT 0 -#define T_INT 1 -#define T_LONG 2 -#define T_FLOAT 3 -#define T_DOUBLE 4 -#define T_STRING 5 -#define T_OBJECT 6 -#define T_CHAR 7 /* 1-character string */ -#define T_BYTE 8 /* 8-bit signed int */ -#define T_UBYTE 9 -#define T_USHORT 10 -#define T_UINT 11 -#define T_ULONG 12 -#define T_STRING_INPLACE 13 /* Strings contained in the structure */ -#define T_BOOL 14 -#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError - when the value is NULL, instead of - converting to None. */ -#define T_LONGLONG 17 -#define T_ULONGLONG 18 -#define T_PYSSIZET 19 +/* Added by Jack: strings contained in the structure */ +#define T_STRING_INPLACE 13 + +/* Added by Lillo: bools contained in the structure (assumed char) */ +#define T_BOOL 14 + +#define T_OBJECT_EX 16 /* Like T_OBJECT, but raises AttributeError + when the value is NULL, instead of + converting to None. */ +#ifdef HAVE_LONG_LONG +#define T_LONGLONG 17 +#define T_ULONGLONG 18 +#endif /* HAVE_LONG_LONG */ + +#define T_PYSSIZET 19 /* Py_ssize_t */ /* Flags. These constants are also in structmemberdefs.py. */ -#define READONLY 1 -#define RO READONLY /* Shorthand */ +#define READONLY 1 +#define RO READONLY /* Shorthand */ #define READ_RESTRICTED 2 #define PY_WRITE_RESTRICTED 4 -#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) +#define RESTRICTED (READ_RESTRICTED | PY_WRITE_RESTRICTED) + + +/* API functions. */ +#include "pypy_structmember_decl.h" #ifdef __cplusplus } #endif #endif /* !Py_STRUCTMEMBER_H */ + diff --git a/pypy/module/cpyext/methodobject.py b/pypy/module/cpyext/methodobject.py --- a/pypy/module/cpyext/methodobject.py +++ b/pypy/module/cpyext/methodobject.py @@ -50,7 +50,7 @@ py_func.c_m_self = make_ref(space, w_obj.w_self) py_func.c_m_module = make_ref(space, w_obj.w_module) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def cfunction_dealloc(space, py_obj): py_func = rffi.cast(PyCFunctionObject, py_obj) Py_DecRef(space, py_func.c_m_self) diff --git a/pypy/module/cpyext/pyobject.py b/pypy/module/cpyext/pyobject.py --- a/pypy/module/cpyext/pyobject.py +++ b/pypy/module/cpyext/pyobject.py @@ -70,7 +70,7 @@ alloc : allocate and basic initialization of a raw PyObject attach : Function called to tie a raw structure to a pypy object realize : Function called to create a pypy object from a raw struct - dealloc : a cpython_api(external=False), similar to PyObject_dealloc + dealloc : a cpython_api(header=None), similar to PyObject_dealloc """ tp_basestruct = kw.pop('basestruct', PyObject.TO) diff --git a/pypy/module/cpyext/pytraceback.py b/pypy/module/cpyext/pytraceback.py --- a/pypy/module/cpyext/pytraceback.py +++ b/pypy/module/cpyext/pytraceback.py @@ -41,7 +41,7 @@ rffi.setintfield(py_traceback, 'c_tb_lasti', traceback.lasti) rffi.setintfield(py_traceback, 'c_tb_lineno',traceback.get_lineno()) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def traceback_dealloc(space, py_obj): py_traceback = rffi.cast(PyTracebackObject, py_obj) Py_DecRef(space, rffi.cast(PyObject, py_traceback.c_tb_next)) diff --git a/pypy/module/cpyext/sliceobject.py b/pypy/module/cpyext/sliceobject.py --- a/pypy/module/cpyext/sliceobject.py +++ b/pypy/module/cpyext/sliceobject.py @@ -36,7 +36,7 @@ py_slice.c_stop = make_ref(space, w_obj.w_stop) py_slice.c_step = make_ref(space, w_obj.w_step) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def slice_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ diff --git a/pypy/module/cpyext/slotdefs.py b/pypy/module/cpyext/slotdefs.py --- a/pypy/module/cpyext/slotdefs.py +++ b/pypy/module/cpyext/slotdefs.py @@ -309,7 +309,7 @@ return space.wrap(generic_cpy_call(space, func_target, w_self, w_other)) -@cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, external=False) +@cpython_api([PyTypeObjectPtr, PyObject, PyObject], PyObject, header=None) def slot_tp_new(space, type, w_args, w_kwds): from pypy.module.cpyext.tupleobject import PyTuple_Check pyo = rffi.cast(PyObject, type) @@ -320,30 +320,30 @@ w_args_new = space.newtuple(args_w) return space.call(w_func, w_args_new, w_kwds) -@cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, external=False) +@cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, error=-1, header=None) def slot_tp_init(space, w_self, w_args, w_kwds): w_descr = space.lookup(w_self, '__init__') args = Arguments.frompacked(space, w_args, w_kwds) space.get_and_call_args(w_descr, w_self, args) return 0 -@cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) +@cpython_api([PyObject, PyObject, PyObject], PyObject, header=None) def slot_tp_call(space, w_self, w_args, w_kwds): return space.call(w_self, w_args, w_kwds) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_str(space, w_self): return space.str(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_nb_int(space, w_self): return space.int(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_iter(space, w_self): return space.iter(w_self) -@cpython_api([PyObject], PyObject, external=False) +@cpython_api([PyObject], PyObject, header=None) def slot_tp_iternext(space, w_self): return space.next(w_self) @@ -371,7 +371,7 @@ return @cpython_api([PyObject, PyObject, PyObject], rffi.INT_real, - error=-1, external=True) # XXX should not be exported + error=-1) # XXX should be header=None @func_renamer("cpyext_tp_setattro_%s" % (typedef.name,)) def slot_tp_setattro(space, w_self, w_name, w_value): if w_value is not None: @@ -385,8 +385,7 @@ if getattr_fn is None: return - @cpython_api([PyObject, PyObject], PyObject, - external=True) + @cpython_api([PyObject, PyObject], PyObject) @func_renamer("cpyext_tp_getattro_%s" % (typedef.name,)) def slot_tp_getattro(space, w_self, w_name): return space.call_function(getattr_fn, w_self, w_name) diff --git a/pypy/module/cpyext/stringobject.py b/pypy/module/cpyext/stringobject.py --- a/pypy/module/cpyext/stringobject.py +++ b/pypy/module/cpyext/stringobject.py @@ -103,7 +103,7 @@ track_reference(space, py_obj, w_obj) return w_obj -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def string_dealloc(space, py_obj): """Frees allocated PyStringObject resources. """ diff --git a/pypy/module/cpyext/structmember.py b/pypy/module/cpyext/structmember.py --- a/pypy/module/cpyext/structmember.py +++ b/pypy/module/cpyext/structmember.py @@ -31,8 +31,10 @@ (T_PYSSIZET, rffi.SSIZE_T, PyLong_AsSsize_t), ]) +_HEADER = 'pypy_structmember_decl.h' -@cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject) + +@cpython_api([PyObject, lltype.Ptr(PyMemberDef)], PyObject, header=_HEADER) def PyMember_GetOne(space, obj, w_member): addr = rffi.cast(ADDR, obj) addr += w_member.c_offset @@ -83,7 +85,8 @@ return w_result -@cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, error=-1) +@cpython_api([PyObject, lltype.Ptr(PyMemberDef), PyObject], rffi.INT_real, + error=-1, header=_HEADER) def PyMember_SetOne(space, obj, w_member, w_value): addr = rffi.cast(ADDR, obj) addr += w_member.c_offset diff --git a/pypy/module/cpyext/test/test_cpyext.py b/pypy/module/cpyext/test/test_cpyext.py --- a/pypy/module/cpyext/test/test_cpyext.py +++ b/pypy/module/cpyext/test/test_cpyext.py @@ -863,3 +863,15 @@ os.unlink('_imported_already') except OSError: pass + + def test_no_structmember(self): + """structmember.h should not be included by default.""" + mod = self.import_extension('foo', [ + ('bar', 'METH_NOARGS', + ''' + /* reuse a name that is #defined in structmember.h */ + int RO; + Py_RETURN_NONE; + ''' + ), + ]) diff --git a/pypy/module/cpyext/test/test_floatobject.py b/pypy/module/cpyext/test/test_floatobject.py --- a/pypy/module/cpyext/test/test_floatobject.py +++ b/pypy/module/cpyext/test/test_floatobject.py @@ -45,3 +45,35 @@ ]) assert module.from_string() == 1234.56 assert type(module.from_string()) is float + +class AppTestFloatMacros(AppTestCpythonExtensionBase): + def test_return_nan(self): + import math + + module = self.import_extension('foo', [ + ("return_nan", "METH_NOARGS", + "Py_RETURN_NAN;"), + ]) + assert math.isnan(module.return_nan()) + + def test_return_inf(self): + import math + + module = self.import_extension('foo', [ + ("return_inf", "METH_NOARGS", + "Py_RETURN_INF(10);"), + ]) + inf = module.return_inf() + assert inf > 0 + assert math.isinf(inf) + + def test_return_inf_negative(self): + import math + + module = self.import_extension('foo', [ + ("return_neginf", "METH_NOARGS", + "Py_RETURN_INF(-10);"), + ]) + neginf = module.return_neginf() + assert neginf < 0 + assert math.isinf(neginf) diff --git a/pypy/module/cpyext/test/test_intobject.py b/pypy/module/cpyext/test/test_intobject.py --- a/pypy/module/cpyext/test/test_intobject.py +++ b/pypy/module/cpyext/test/test_intobject.py @@ -99,6 +99,7 @@ """), ], prologue=""" + #include "structmember.h" typedef struct { PyObject_HEAD diff --git a/pypy/module/cpyext/test/test_translate.py b/pypy/module/cpyext/test/test_translate.py --- a/pypy/module/cpyext/test/test_translate.py +++ b/pypy/module/cpyext/test/test_translate.py @@ -19,7 +19,7 @@ @specialize.memo() def get_tp_function(space, typedef): - @cpython_api([], lltype.Signed, error=-1, external=False) + @cpython_api([], lltype.Signed, error=-1, header=None) def slot_tp_function(space): return typedef.value diff --git a/pypy/module/cpyext/typeobject.py b/pypy/module/cpyext/typeobject.py --- a/pypy/module/cpyext/typeobject.py +++ b/pypy/module/cpyext/typeobject.py @@ -183,7 +183,7 @@ if pto.c_tp_new: add_tp_new_wrapper(space, dict_w, pto) -@cpython_api([PyObject, PyObject, PyObject], PyObject, external=False) +@cpython_api([PyObject, PyObject, PyObject], PyObject, header=None) def tp_new_wrapper(space, self, w_args, w_kwds): tp_new = rffi.cast(PyTypeObjectPtr, self).c_tp_new @@ -311,7 +311,7 @@ dealloc=type_dealloc) -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def subtype_dealloc(space, obj): pto = obj.c_ob_type base = pto @@ -327,7 +327,7 @@ # hopefully this does not clash with the memory model assumed in # extension modules -@cpython_api([PyObject, Py_ssize_tP], lltype.Signed, external=False, +@cpython_api([PyObject, Py_ssize_tP], lltype.Signed, header=None, error=CANNOT_FAIL) def str_segcount(space, w_obj, ref): if ref: @@ -335,7 +335,7 @@ return 1 @cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, - external=False, error=-1) + header=None, error=-1) def str_getreadbuffer(space, w_str, segment, ref): from pypy.module.cpyext.stringobject import PyString_AsString if segment != 0: @@ -348,7 +348,7 @@ return space.len_w(w_str) @cpython_api([PyObject, Py_ssize_t, rffi.CCHARPP], lltype.Signed, - external=False, error=-1) + header=None, error=-1) def str_getcharbuffer(space, w_str, segment, ref): from pypy.module.cpyext.stringobject import PyString_AsString if segment != 0: @@ -361,7 +361,7 @@ return space.len_w(w_str) @cpython_api([PyObject, Py_ssize_t, rffi.VOIDPP], lltype.Signed, - external=False, error=-1) + header=None, error=-1) def buf_getreadbuffer(space, pyref, segment, ref): from pypy.module.cpyext.bufferobject import PyBufferObject if segment != 0: @@ -393,7 +393,7 @@ buf_getreadbuffer.api_func.get_wrapper(space)) pto.c_tp_as_buffer = c_buf -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def type_dealloc(space, obj): from pypy.module.cpyext.object import PyObject_dealloc obj_pto = rffi.cast(PyTypeObjectPtr, obj) diff --git a/pypy/module/cpyext/unicodeobject.py b/pypy/module/cpyext/unicodeobject.py --- a/pypy/module/cpyext/unicodeobject.py +++ b/pypy/module/cpyext/unicodeobject.py @@ -75,7 +75,7 @@ track_reference(space, py_obj, w_obj) return w_obj -@cpython_api([PyObject], lltype.Void, external=False) +@cpython_api([PyObject], lltype.Void, header=None) def unicode_dealloc(space, py_obj): py_unicode = rffi.cast(PyUnicodeObject, py_obj) if py_unicode.c_buffer: diff --git a/pypy/module/sys/interp_encoding.py b/pypy/module/sys/interp_encoding.py --- a/pypy/module/sys/interp_encoding.py +++ b/pypy/module/sys/interp_encoding.py @@ -34,7 +34,11 @@ elif sys.platform == "darwin": base_encoding = "utf-8" else: - base_encoding = None + # In CPython, the default base encoding is NULL. This is paired with a + # comment that says "If non-NULL, this is different than the default + # encoding for strings". Therefore, the default filesystem encoding is the + # default encoding for strings, which is ASCII. + base_encoding = "ascii" def _getfilesystemencoding(space): encoding = base_encoding diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/backend_tests.py @@ -1847,3 +1847,8 @@ thread.start_new_thread(f, ()) time.sleep(1.5) assert seen == ['init!', 'init done'] + 6 * [7] + + def test_sizeof_struct_directly(self): + # only works with the Python FFI instances + ffi = FFI(backend=self.Backend()) + assert ffi.sizeof("struct{int a;}") == ffi.sizeof("int") diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_ffi_backend.py @@ -420,3 +420,7 @@ ]: x = ffi.sizeof(name) assert 1 <= x <= 16 + + def test_ffi_def_extern(self): + ffi = FFI() + py.test.raises(ValueError, ffi.def_extern) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi0/test_verify.py @@ -92,8 +92,8 @@ assert lib.sin(1.23) == math.sin(1.23) def _Wconversion(cdef, source, **kargs): - if sys.platform == 'win32': - py.test.skip("needs GCC or Clang") + if sys.platform in ('win32', 'darwin'): + py.test.skip("needs GCC") ffi = FFI() ffi.cdef(cdef) py.test.raises(VerificationError, ffi.verify, source, **kargs) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_recompiler.py @@ -1714,3 +1714,33 @@ # a case where 'onerror' is not callable py.test.raises(TypeError, ffi.def_extern(name='bar', onerror=42), lambda x: x) + +def test_extern_python_stdcall(): + ffi = FFI() + ffi.cdef(""" + extern "Python" int __stdcall foo(int); + extern "Python" int WINAPI bar(int); + int (__stdcall * mycb1)(int); + int indirect_call(int); + """) + lib = verify(ffi, 'test_extern_python_stdcall', """ + #ifndef _MSC_VER + # define __stdcall + #endif + static int (__stdcall * mycb1)(int); + static int indirect_call(int x) { + return mycb1(x); + } + """) + # + @ffi.def_extern() + def foo(x): + return x + 42 + @ffi.def_extern() + def bar(x): + return x + 43 + assert lib.foo(100) == 142 + assert lib.bar(100) == 143 + lib.mycb1 = lib.foo + assert lib.mycb1(200) == 242 + assert lib.indirect_call(300) == 342 diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_verify1.py @@ -72,8 +72,8 @@ assert lib.sin(1.23) == math.sin(1.23) def _Wconversion(cdef, source, **kargs): - if sys.platform == 'win32': - py.test.skip("needs GCC or Clang") + if sys.platform in ('win32', 'darwin'): + py.test.skip("needs GCC") ffi = FFI() ffi.cdef(cdef) py.test.raises(VerificationError, ffi.verify, source, **kargs) @@ -2092,20 +2092,20 @@ old = sys.getdlopenflags() try: ffi1 = FFI() - ffi1.cdef("int foo_verify_dlopen_flags;") + ffi1.cdef("int foo_verify_dlopen_flags_1;") sys.setdlopenflags(ffi1.RTLD_GLOBAL | ffi1.RTLD_NOW) - lib1 = ffi1.verify("int foo_verify_dlopen_flags;") + lib1 = ffi1.verify("int foo_verify_dlopen_flags_1;") finally: sys.setdlopenflags(old) ffi2 = FFI() ffi2.cdef("int *getptr(void);") lib2 = ffi2.verify(""" - extern int foo_verify_dlopen_flags; - static int *getptr(void) { return &foo_verify_dlopen_flags; } + extern int foo_verify_dlopen_flags_1; + static int *getptr(void) { return &foo_verify_dlopen_flags_1; } """) p = lib2.getptr() - assert ffi1.addressof(lib1, 'foo_verify_dlopen_flags') == p + assert ffi1.addressof(lib1, 'foo_verify_dlopen_flags_1') == p def test_consider_not_implemented_function_type(): ffi = FFI() diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py @@ -49,7 +49,8 @@ import setuptools except ImportError: _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit