Author: Armin Rigo <ar...@tunes.org>
Branch:
Changeset: r2585:fc7471ccde10
Date: 2016-01-15 11:26 +0100
http://bitbucket.org/cffi/cffi/changeset/fc7471ccde10/
Log: hg merge static-callback-embedding
Embedding!
diff too long, truncating to 2000 out of 2798 lines
diff --git a/c/_cffi_backend.c b/c/_cffi_backend.c
--- a/c/_cffi_backend.c
+++ b/c/_cffi_backend.c
@@ -6354,7 +6354,7 @@
#endif
struct _cffi_externpy_s; /* forward declaration */
-static void _cffi_call_python(struct _cffi_externpy_s *, char *args);
+static void cffi_call_python(struct _cffi_externpy_s *, char *args);
static void *cffi_exports[] = {
NULL,
@@ -6387,7 +6387,7 @@
_cffi_to_c__Bool,
_prepare_pointer_call_argument,
convert_array_from_object,
- _cffi_call_python,
+ cffi_call_python,
};
static struct { const char *name; int value; } all_dlopen_flags[] = {
diff --git a/c/call_python.c b/c/call_python.c
--- a/c/call_python.c
+++ b/c/call_python.c
@@ -94,7 +94,7 @@
return NULL;
/* force _update_cache_to_call_python() to be called the next time
- the C function invokes _cffi_call_python, to update the cache */
+ the C function invokes cffi_call_python, to update the cache */
old1 = externpy->reserved1;
externpy->reserved1 = Py_None; /* a non-NULL value */
Py_INCREF(Py_None);
@@ -143,7 +143,15 @@
return 2; /* out of memory? */
}
-static void _cffi_call_python(struct _cffi_externpy_s *externpy, char *args)
+#if (defined(WITH_THREAD) && !defined(_MSC_VER) && \
+ !defined(__amd64__) && !defined(__x86_64__) && \
+ !defined(__i386__) && !defined(__i386))
+# define read_barrier() __sync_synchronize()
+#else
+# define read_barrier() (void)0
+#endif
+
+static void cffi_call_python(struct _cffi_externpy_s *externpy, char *args)
{
/* Invoked by the helpers generated from extern "Python" in the cdef.
@@ -164,6 +172,21 @@
at least 8 bytes in size.
*/
int err = 0;
+
+ /* This read barrier is needed for _embedding.h. It is paired
+ with the write_barrier() there. Without this barrier, we can
+ in theory see the following situation: the Python
+ initialization code already ran (in another thread), and the
+ '_cffi_call_python' function pointer directed execution here;
+ but any number of other data could still be seen as
+ uninitialized below. For example, 'externpy' would still
+ contain NULLs even though it was correctly set up, or
+ 'interpreter_lock' (the GIL inside CPython) would still be seen
+ as NULL, or 'autoInterpreterState' (used by
+ PyGILState_Ensure()) would be NULL or contain bogus fields.
+ */
+ read_barrier();
+
save_errno();
/* We need the infotuple here. We could always go through
diff --git a/c/cffi1_module.c b/c/cffi1_module.c
--- a/c/cffi1_module.c
+++ b/c/cffi1_module.c
@@ -3,7 +3,7 @@
#include "realize_c_type.c"
#define CFFI_VERSION_MIN 0x2601
-#define CFFI_VERSION_MAX 0x26FF
+#define CFFI_VERSION_MAX 0x27FF
typedef struct FFIObject_s FFIObject;
typedef struct LibObject_s LibObject;
@@ -214,5 +214,12 @@
(PyObject *)lib) < 0)
return NULL;
+#if PY_MAJOR_VERSION >= 3
+ /* add manually 'module_name' in sys.modules: it seems that
+ Py_InitModule() is not enough to do that */
+ if (PyDict_SetItemString(modules_dict, module_name, m) < 0)
+ return NULL;
+#endif
+
return m;
}
diff --git a/c/test_c.py b/c/test_c.py
--- a/c/test_c.py
+++ b/c/test_c.py
@@ -12,7 +12,7 @@
# ____________________________________________________________
import sys
-assert __version__ == "1.4.2", ("This test_c.py file is for testing a version"
+assert __version__ == "1.4.3", ("This test_c.py file is for testing a version"
" of cffi that differs from the one that we"
" get from 'import _cffi_backend'")
if sys.version_info < (3,):
diff --git a/cffi/__init__.py b/cffi/__init__.py
--- a/cffi/__init__.py
+++ b/cffi/__init__.py
@@ -4,8 +4,8 @@
from .api import FFI, CDefError, FFIError
from .ffiplatform import VerificationError, VerificationMissing
-__version__ = "1.4.2"
-__version_info__ = (1, 4, 2)
+__version__ = "1.4.3"
+__version_info__ = (1, 4, 3)
# The verifier module file names are based on the CRC32 of a string that
# contains the following version number. It may be older than __version__
diff --git a/cffi/_cffi_include.h b/cffi/_cffi_include.h
--- a/cffi/_cffi_include.h
+++ b/cffi/_cffi_include.h
@@ -146,8 +146,9 @@
((Py_ssize_t(*)(CTypeDescrObject *, PyObject *, char **))_cffi_exports[23])
#define _cffi_convert_array_from_object \
((int(*)(char *, CTypeDescrObject *, PyObject *))_cffi_exports[24])
+#define _CFFI_CPIDX 25
#define _cffi_call_python \
- ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[25])
+ ((void(*)(struct _cffi_externpy_s *, char *))_cffi_exports[_CFFI_CPIDX])
#define _CFFI_NUM_EXPORTS 26
typedef struct _ctypedescr CTypeDescrObject;
@@ -206,7 +207,8 @@
/********** end CPython-specific section **********/
#else
_CFFI_UNUSED_FN
-static void (*_cffi_call_python)(struct _cffi_externpy_s *, char *);
+static void (*_cffi_call_python_org)(struct _cffi_externpy_s *, char *);
+# define _cffi_call_python _cffi_call_python_org
#endif
diff --git a/cffi/_embedding.h b/cffi/_embedding.h
new file mode 100644
--- /dev/null
+++ b/cffi/_embedding.h
@@ -0,0 +1,517 @@
+
+/***** Support code for embedding *****/
+
+#if defined(_MSC_VER)
+# define CFFI_DLLEXPORT __declspec(dllexport)
+#elif defined(__GNUC__)
+# define CFFI_DLLEXPORT __attribute__((visibility("default")))
+#else
+# define CFFI_DLLEXPORT /* nothing */
+#endif
+
+
+/* There are two global variables of type _cffi_call_python_fnptr:
+
+ * _cffi_call_python, which we declare just below, is the one called
+ by ``extern "Python"`` implementations.
+
+ * _cffi_call_python_org, which on CPython is actually part of the
+ _cffi_exports[] array, is the function pointer copied from
+ _cffi_backend.
+
+ After initialization is complete, both are equal. However, the
+ first one remains equal to &_cffi_start_and_call_python until the
+ very end of initialization, when we are (or should be) sure that
+ concurrent threads also see a completely initialized world, and
+ only then is it changed.
+*/
+#undef _cffi_call_python
+typedef void (*_cffi_call_python_fnptr)(struct _cffi_externpy_s *, char *);
+static void _cffi_start_and_call_python(struct _cffi_externpy_s *, char *);
+static _cffi_call_python_fnptr _cffi_call_python =
&_cffi_start_and_call_python;
+
+
+#ifndef _MSC_VER
+ /* --- Assuming a GCC not infinitely old --- */
+# define cffi_compare_and_swap(l,o,n) __sync_bool_compare_and_swap(l,o,n)
+# define cffi_write_barrier() __sync_synchronize()
+# if !defined(__amd64__) && !defined(__x86_64__) && \
+ !defined(__i386__) && !defined(__i386)
+# define cffi_read_barrier() __sync_synchronize()
+# else
+# define cffi_read_barrier() (void)0
+# endif
+#else
+ /* --- Windows threads version --- */
+# include <Windows.h>
+# define cffi_compare_and_swap(l,o,n) \
+ (InterlockedCompareExchangePointer(l,n,o) ==
(o))
+# define cffi_write_barrier()
InterlockedCompareExchange(&_cffi_dummy,0,0)
+# define cffi_read_barrier() (void)0
+static volatile LONG _cffi_dummy;
+#endif
+
+#ifdef WITH_THREAD
+# ifndef _MSC_VER
+# include <pthread.h>
+ static pthread_mutex_t _cffi_embed_startup_lock;
+# else
+ static CRITICAL_SECTION _cffi_embed_startup_lock;
+# endif
+ static char _cffi_embed_startup_lock_ready = 0;
+#endif
+
+static void _cffi_acquire_reentrant_mutex(void)
+{
+ static void *volatile lock = NULL;
+
+ while (!cffi_compare_and_swap(&lock, NULL, (void *)1)) {
+ /* should ideally do a spin loop instruction here, but
+ hard to do it portably and doesn't really matter I
+ think: pthread_mutex_init() should be very fast, and
+ this is only run at start-up anyway. */
+ }
+
+#ifdef WITH_THREAD
+ if (!_cffi_embed_startup_lock_ready) {
+# ifndef _MSC_VER
+ pthread_mutexattr_t attr;
+ pthread_mutexattr_init(&attr);
+ pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE);
+ pthread_mutex_init(&_cffi_embed_startup_lock, &attr);
+# else
+ InitializeCriticalSection(&_cffi_embed_startup_lock);
+# endif
+ _cffi_embed_startup_lock_ready = 1;
+ }
+#endif
+
+ while (!cffi_compare_and_swap(&lock, (void *)1, NULL))
+ ;
+
+#ifndef _MSC_VER
+ pthread_mutex_lock(&_cffi_embed_startup_lock);
+#else
+ EnterCriticalSection(&_cffi_embed_startup_lock);
+#endif
+}
+
+static void _cffi_release_reentrant_mutex(void)
+{
+#ifndef _MSC_VER
+ pthread_mutex_unlock(&_cffi_embed_startup_lock);
+#else
+ LeaveCriticalSection(&_cffi_embed_startup_lock);
+#endif
+}
+
+
+/********** CPython-specific section **********/
+#ifndef PYPY_VERSION
+
+
+#define _cffi_call_python_org _cffi_exports[_CFFI_CPIDX]
+
+PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(void); /* forward */
+
+static void _cffi_py_initialize(void)
+{
+ /* XXX use initsigs=0, which "skips initialization registration of
+ signal handlers, which might be useful when Python is
+ embedded" according to the Python docs. But review and think
+ if it should be a user-controllable setting.
+
+ XXX we should also give a way to write errors to a buffer
+ instead of to stderr.
+
+ XXX if importing 'site' fails, CPython (any version) calls
+ exit(). Should we try to work around this behavior here?
+ */
+ Py_InitializeEx(0);
+}
+
+static int _cffi_initialize_python(void)
+{
+ /* This initializes Python, imports _cffi_backend, and then the
+ present .dll/.so is set up as a CPython C extension module.
+ */
+ int result;
+ PyGILState_STATE state;
+ PyObject *pycode=NULL, *global_dict=NULL, *x;
+
+#if PY_MAJOR_VERSION >= 3
+ /* see comments in _cffi_carefully_make_gil() about the
+ Python2/Python3 difference
+ */
+#else
+ /* Acquire the GIL. We have no threadstate here. If Python is
+ already initialized, it is possible that there is already one
+ existing for this thread, but it is not made current now.
+ */
+ PyEval_AcquireLock();
+
+ _cffi_py_initialize();
+
+ /* The Py_InitializeEx() sometimes made a threadstate for us, but
+ not always. Indeed Py_InitializeEx() could be called and do
+ nothing. So do we have a threadstate, or not? We don't know,
+ but we can replace it with NULL in all cases.
+ */
+ (void)PyThreadState_Swap(NULL);
+
+ /* Now we can release the GIL and re-acquire immediately using the
+ logic of PyGILState(), which handles making or installing the
+ correct threadstate.
+ */
+ PyEval_ReleaseLock();
+#endif
+ state = PyGILState_Ensure();
+
+ /* Call the initxxx() function from the present module. It will
+ create and initialize us as a CPython extension module, instead
+ of letting the startup Python code do it---it might reimport
+ the same .dll/.so and get maybe confused on some platforms.
+ It might also have troubles locating the .dll/.so again for all
+ I know.
+ */
+ (void)_CFFI_PYTHON_STARTUP_FUNC();
+ if (PyErr_Occurred())
+ goto error;
+
+ /* Now run the Python code provided to ffi.embedding_init_code().
+ */
+ pycode = Py_CompileString(_CFFI_PYTHON_STARTUP_CODE,
+ "<init code for '" _CFFI_MODULE_NAME "'>",
+ Py_file_input);
+ if (pycode == NULL)
+ goto error;
+ global_dict = PyDict_New();
+ if (global_dict == NULL)
+ goto error;
+ if (PyDict_SetItemString(global_dict, "__builtins__",
+ PyThreadState_GET()->interp->builtins) < 0)
+ goto error;
+ x = PyEval_EvalCode(
+#if PY_MAJOR_VERSION < 3
+ (PyCodeObject *)
+#endif
+ pycode, global_dict, global_dict);
+ if (x == NULL)
+ goto error;
+ Py_DECREF(x);
+
+ /* Done! Now if we've been called from
+ _cffi_start_and_call_python() in an ``extern "Python"``, we can
+ only hope that the Python code did correctly set up the
+ corresponding @ffi.def_extern() function. Otherwise, the
+ general logic of ``extern "Python"`` functions (inside the
+ _cffi_backend module) will find that the reference is still
+ missing and print an error.
+ */
+ result = 0;
+ done:
+ Py_XDECREF(pycode);
+ Py_XDECREF(global_dict);
+ PyGILState_Release(state);
+ return result;
+
+ error:;
+ {
+ /* Print as much information as potentially useful.
+ Debugging load-time failures with embedding is not fun
+ */
+ PyObject *exception, *v, *tb, *f, *modules, *mod;
+ PyErr_Fetch(&exception, &v, &tb);
+ if (exception != NULL) {
+ PyErr_NormalizeException(&exception, &v, &tb);
+ PyErr_Display(exception, v, tb);
+ }
+ Py_XDECREF(exception);
+ Py_XDECREF(v);
+ Py_XDECREF(tb);
+
+ f = PySys_GetObject((char *)"stderr");
+ if (f != NULL && f != Py_None) {
+ PyFile_WriteString("\nFrom: " _CFFI_MODULE_NAME
+ "\ncompiled with cffi version: 1.4.3"
+ "\n_cffi_backend module: ", f);
+ modules = PyImport_GetModuleDict();
+ mod = PyDict_GetItemString(modules, "_cffi_backend");
+ if (mod == NULL) {
+ PyFile_WriteString("not loaded", f);
+ }
+ else {
+ v = PyObject_GetAttrString(mod, "__file__");
+ PyFile_WriteObject(v, f, 0);
+ Py_XDECREF(v);
+ }
+ PyFile_WriteString("\nsys.path: ", f);
+ PyFile_WriteObject(PySys_GetObject((char *)"path"), f, 0);
+ PyFile_WriteString("\n\n", f);
+ }
+ }
+ result = -1;
+ goto done;
+}
+
+PyAPI_DATA(char *) _PyParser_TokenNames[]; /* from CPython */
+
+static int _cffi_carefully_make_gil(void)
+{
+ /* This does the basic initialization of Python. It can be called
+ completely concurrently from unrelated threads. It assumes
+ that we don't hold the GIL before (if it exists), and we don't
+ hold it afterwards.
+
+ What it really does is completely different in Python 2 and
+ Python 3.
+
+ Python 2
+ ========
+
+ Initialize the GIL, without initializing the rest of Python,
+ by calling PyEval_InitThreads().
+
+ PyEval_InitThreads() must not be called concurrently at all.
+ So we use a global variable as a simple spin lock. This global
+ variable must be from 'libpythonX.Y.so', not from this
+ cffi-based extension module, because it must be shared from
+ different cffi-based extension modules. We choose
+ _PyParser_TokenNames[0] as a completely arbitrary pointer value
+ that is never written to. The default is to point to the
+ string "ENDMARKER". We change it temporarily to point to the
+ next character in that string. (Yes, I know it's REALLY
+ obscure.)
+
+ Python 3
+ ========
+
+ In Python 3, PyEval_InitThreads() cannot be called before
+ Py_InitializeEx() any more. So this function calls
+ Py_InitializeEx() first. It uses the same obscure logic to
+ make sure we never call it concurrently.
+
+ Arguably, this is less good on the spinlock, because
+ Py_InitializeEx() takes much longer to run than
+ PyEval_InitThreads(). But I didn't find a way around it.
+ */
+
+#ifdef WITH_THREAD
+ char *volatile *lock = (char *volatile *)_PyParser_TokenNames;
+ char *old_value;
+
+ while (1) { /* spin loop */
+ old_value = *lock;
+ if (old_value[0] == 'E') {
+ assert(old_value[1] == 'N');
+ if (cffi_compare_and_swap(lock, old_value, old_value + 1))
+ break;
+ }
+ else {
+ assert(old_value[0] == 'N');
+ /* should ideally do a spin loop instruction here, but
+ hard to do it portably and doesn't really matter I
+ think: PyEval_InitThreads() should be very fast, and
+ this is only run at start-up anyway. */
+ }
+ }
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+ /* Python 3: call Py_InitializeEx() */
+ {
+ PyGILState_STATE state = PyGILState_UNLOCKED;
+ if (!Py_IsInitialized())
+ _cffi_py_initialize();
+ else
+ state = PyGILState_Ensure();
+
+ PyEval_InitThreads();
+ PyGILState_Release(state);
+ }
+#else
+ /* Python 2: call PyEval_InitThreads() */
+# ifdef WITH_THREAD
+ if (!PyEval_ThreadsInitialized()) {
+ PyEval_InitThreads(); /* makes the GIL */
+ PyEval_ReleaseLock(); /* then release it */
+ }
+ /* else: there is already a GIL, but we still needed to do the
+ spinlock dance to make sure that we see it as fully ready */
+# endif
+#endif
+
+#ifdef WITH_THREAD
+ /* release the lock */
+ while (!cffi_compare_and_swap(lock, old_value + 1, old_value))
+ ;
+#endif
+
+ return 0;
+}
+
+/********** end CPython-specific section **********/
+
+
+#else
+
+
+/********** PyPy-specific section **********/
+
+PyMODINIT_FUNC _CFFI_PYTHON_STARTUP_FUNC(const void *[]); /* forward */
+
+static struct _cffi_pypy_init_s {
+ const char *name;
+ void (*func)(const void *[]);
+ const char *code;
+} _cffi_pypy_init = {
+ _CFFI_MODULE_NAME,
+ _CFFI_PYTHON_STARTUP_FUNC,
+ _CFFI_PYTHON_STARTUP_CODE,
+};
+
+extern int pypy_carefully_make_gil(const char *);
+extern int pypy_init_embedded_cffi_module(int, struct _cffi_pypy_init_s *);
+
+static int _cffi_carefully_make_gil(void)
+{
+ return pypy_carefully_make_gil(_CFFI_MODULE_NAME);
+}
+
+static int _cffi_initialize_python(void)
+{
+ return pypy_init_embedded_cffi_module(0xB011, &_cffi_pypy_init);
+}
+
+/********** end PyPy-specific section **********/
+
+
+#endif
+
+
+#ifdef __GNUC__
+__attribute__((noinline))
+#endif
+static _cffi_call_python_fnptr _cffi_start_python(void)
+{
+ /* Delicate logic to initialize Python. This function can be
+ called multiple times concurrently, e.g. when the process calls
+ its first ``extern "Python"`` functions in multiple threads at
+ once. It can also be called recursively, in which case we must
+ ignore it. We also have to consider what occurs if several
+ different cffi-based extensions reach this code in parallel
+ threads---it is a different copy of the code, then, and we
+ can't have any shared global variable unless it comes from
+ 'libpythonX.Y.so'.
+
+ Idea:
+
+ * _cffi_carefully_make_gil(): "carefully" call
+ PyEval_InitThreads() (possibly with Py_InitializeEx() first).
+
+ * then we use a (local) custom lock to make sure that a call to this
+ cffi-based extension will wait if another call to the *same*
+ extension is running the initialization in another thread.
+ It is reentrant, so that a recursive call will not block, but
+ only one from a different thread.
+
+ * then we grab the GIL and (Python 2) we call Py_InitializeEx().
+ At this point, concurrent calls to Py_InitializeEx() are not
+ possible: we have the GIL.
+
+ * do the rest of the specific initialization, which may
+ temporarily release the GIL but not the custom lock.
+ Only release the custom lock when we are done.
+ */
+ static char called = 0;
+
+ if (_cffi_carefully_make_gil() != 0)
+ return NULL;
+
+ _cffi_acquire_reentrant_mutex();
+
+ /* Here the GIL exists, but we don't have it. We're only protected
+ from concurrency by the reentrant mutex. */
+
+ /* This file only initializes the embedded module once, the first
+ time this is called, even if there are subinterpreters. */
+ if (!called) {
+ called = 1; /* invoke _cffi_initialize_python() only once,
+ but don't set '_cffi_call_python' right now,
+ otherwise concurrent threads won't call
+ this function at all (we need them to wait) */
+ if (_cffi_initialize_python() == 0) {
+ /* now initialization is finished. Switch to the fast-path. */
+
+ /* We would like nobody to see the new value of
+ '_cffi_call_python' without also seeing the rest of the
+ data initialized. However, this is not possible. But
+ the new value of '_cffi_call_python' is the function
+ 'cffi_call_python()' from _cffi_backend. So: */
+ cffi_write_barrier();
+ /* ^^^ we put a write barrier here, and a corresponding
+ read barrier at the start of cffi_call_python(). This
+ ensures that after that read barrier, we see everything
+ done here before the write barrier.
+ */
+
+ assert(_cffi_call_python_org != NULL);
+ _cffi_call_python = (_cffi_call_python_fnptr)_cffi_call_python_org;
+ }
+ else {
+ /* initialization failed. Reset this to NULL, even if it was
+ already set to some other value. Future calls to
+ _cffi_start_python() are still forced to occur, and will
+ always return NULL from now on. */
+ _cffi_call_python_org = NULL;
+ }
+ }
+
+ _cffi_release_reentrant_mutex();
+
+ return (_cffi_call_python_fnptr)_cffi_call_python_org;
+}
+
+static
+void _cffi_start_and_call_python(struct _cffi_externpy_s *externpy, char *args)
+{
+ _cffi_call_python_fnptr fnptr;
+ int current_err = errno;
+#ifdef _MSC_VER
+ int current_lasterr = GetLastError();
+#endif
+ fnptr = _cffi_start_python();
+ if (fnptr == NULL) {
+ fprintf(stderr, "function %s() called, but initialization code "
+ "failed. Returning 0.\n", externpy->name);
+ memset(args, 0, externpy->size_of_result);
+ }
+#ifdef _MSC_VER
+ SetLastError(current_lasterr);
+#endif
+ errno = current_err;
+
+ if (fnptr != NULL)
+ fnptr(externpy, args);
+}
+
+
+/* The cffi_start_python() function makes sure Python is initialized
+ and our cffi module is set up. It can be called manually from the
+ user C code. The same effect is obtained automatically from any
+ dll-exported ``extern "Python"`` function. This function returns
+ -1 if initialization failed, 0 if all is OK. */
+_CFFI_UNUSED_FN
+static int cffi_start_python(void)
+{
+ if (_cffi_call_python == &_cffi_start_and_call_python) {
+ if (_cffi_start_python() == NULL)
+ return -1;
+ }
+ cffi_read_barrier();
+ return 0;
+}
+
+#undef cffi_compare_and_swap
+#undef cffi_write_barrier
+#undef cffi_read_barrier
diff --git a/cffi/api.py b/cffi/api.py
--- a/cffi/api.py
+++ b/cffi/api.py
@@ -74,6 +74,7 @@
self._windows_unicode = None
self._init_once_cache = {}
self._cdef_version = None
+ self._embedding = None
if hasattr(backend, 'set_ffi'):
backend.set_ffi(self)
for name in backend.__dict__:
@@ -101,13 +102,21 @@
If 'packed' is specified as True, all structs declared inside this
cdef are packed, i.e. laid out without any field alignment at all.
"""
+ self._cdef(csource, override=override, packed=packed)
+
+ def embedding_api(self, csource, packed=False):
+ self._cdef(csource, packed=packed, dllexport=True)
+ if self._embedding is None:
+ self._embedding = ''
+
+ def _cdef(self, csource, override=False, **options):
if not isinstance(csource, str): # unicode, on Python 2
if not isinstance(csource, basestring):
raise TypeError("cdef() argument must be a string")
csource = csource.encode('ascii')
with self._lock:
self._cdef_version = object()
- self._parser.parse(csource, override=override, packed=packed)
+ self._parser.parse(csource, override=override, **options)
self._cdefsources.append(csource)
if override:
for cache in self._function_caches:
@@ -533,6 +542,25 @@
('_UNICODE', '1')]
kwds['define_macros'] = defmacros
+ def _apply_embedding_fix(self, kwds):
+ # must include an argument like "-lpython2.7" for the compiler
+ if '__pypy__' in sys.builtin_module_names:
+ pythonlib = "pypy-c"
+ else:
+ if sys.platform == "win32":
+ template = "python%d%d"
+ if sys.flags.debug:
+ template = template + '_d'
+ else:
+ template = "python%d.%d"
+ pythonlib = (template %
+ (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff))
+ if hasattr(sys, 'abiflags'):
+ pythonlib += sys.abiflags
+ libraries = kwds.get('libraries', [])
+ if pythonlib not in libraries:
+ kwds['libraries'] = libraries + [pythonlib]
+
def set_source(self, module_name, source, source_extension='.c', **kwds):
if hasattr(self, '_assigned_source'):
raise ValueError("set_source() cannot be called several times "
@@ -592,14 +620,23 @@
recompile(self, module_name, source,
c_file=filename, call_c_compiler=False, **kwds)
- def compile(self, tmpdir='.', verbose=0):
+ def compile(self, tmpdir='.', verbose=0, target=None):
+ """The 'target' argument gives the final file name of the
+ compiled DLL. Use '*' to force distutils' choice, suitable for
+ regular CPython C API modules. Use a file name ending in '.*'
+ to ask for the system's default extension for dynamic libraries
+ (.so/.dll).
+
+ The default is '*' when building a non-embedded C API extension,
+ and (module_name + '.*') when building an embedded library.
+ """
from .recompiler import recompile
#
if not hasattr(self, '_assigned_source'):
raise ValueError("set_source() must be called before compile()")
module_name, source, source_extension, kwds = self._assigned_source
return recompile(self, module_name, source, tmpdir=tmpdir,
- source_extension=source_extension,
+ target=target, source_extension=source_extension,
compiler_verbose=verbose, **kwds)
def init_once(self, func, tag):
@@ -626,6 +663,32 @@
self._init_once_cache[tag] = (True, result)
return result
+ def embedding_init_code(self, pysource):
+ if self._embedding:
+ raise ValueError("embedding_init_code() can only be called once")
+ # fix 'pysource' before it gets dumped into the C file:
+ # - remove empty lines at the beginning, so it starts at "line 1"
+ # - dedent, if all non-empty lines are indented
+ # - check for SyntaxErrors
+ import re
+ match = re.match(r'\s*\n', pysource)
+ if match:
+ pysource = pysource[match.end():]
+ lines = pysource.splitlines() or ['']
+ prefix = re.match(r'\s*', lines[0]).group()
+ for i in range(1, len(lines)):
+ line = lines[i]
+ if line.rstrip():
+ while not line.startswith(prefix):
+ prefix = prefix[:-1]
+ i = len(prefix)
+ lines = [line[i:]+'\n' for line in lines]
+ pysource = ''.join(lines)
+ #
+ compile(pysource, "cffi_init", "exec")
+ #
+ self._embedding = pysource
+
def _load_backend_lib(backend, name, flags):
if name is None:
diff --git a/cffi/cparser.py b/cffi/cparser.py
--- a/cffi/cparser.py
+++ b/cffi/cparser.py
@@ -220,8 +220,7 @@
self._included_declarations = set()
self._anonymous_counter = 0
self._structnode2type = weakref.WeakKeyDictionary()
- self._override = False
- self._packed = False
+ self._options = None
self._int_constants = {}
self._recomplete = []
self._uses_new_feature = None
@@ -281,16 +280,15 @@
msg = 'parse error\n%s' % (msg,)
raise api.CDefError(msg)
- def parse(self, csource, override=False, packed=False):
- prev_override = self._override
- prev_packed = self._packed
+ def parse(self, csource, override=False, packed=False, dllexport=False):
+ prev_options = self._options
try:
- self._override = override
- self._packed = packed
+ self._options = {'override': override,
+ 'packed': packed,
+ 'dllexport': dllexport}
self._internal_parse(csource)
finally:
- self._override = prev_override
- self._packed = prev_packed
+ self._options = prev_options
def _internal_parse(self, csource):
ast, macros, csource = self._parse(csource)
@@ -376,10 +374,13 @@
def _declare_function(self, tp, quals, decl):
tp = self._get_type_pointer(tp, quals)
- if self._inside_extern_python:
- self._declare('extern_python ' + decl.name, tp)
+ if self._options['dllexport']:
+ tag = 'dllexport_python '
+ elif self._inside_extern_python:
+ tag = 'extern_python '
else:
- self._declare('function ' + decl.name, tp)
+ tag = 'function '
+ self._declare(tag + decl.name, tp)
def _parse_decl(self, decl):
node = decl.type
@@ -449,7 +450,7 @@
prevobj, prevquals = self._declarations[name]
if prevobj is obj and prevquals == quals:
return
- if not self._override:
+ if not self._options['override']:
raise api.FFIError(
"multiple declarations of %s (for interactive usage, "
"try cdef(xx, override=True))" % (name,))
@@ -728,7 +729,7 @@
if isinstance(tp, model.StructType) and tp.partial:
raise NotImplementedError("%s: using both bitfields and '...;'"
% (tp,))
- tp.packed = self._packed
+ tp.packed = self._options['packed']
if tp.completed: # must be re-completed: it is not opaque any more
tp.completed = 0
self._recomplete.append(tp)
diff --git a/cffi/ffiplatform.py b/cffi/ffiplatform.py
--- a/cffi/ffiplatform.py
+++ b/cffi/ffiplatform.py
@@ -21,12 +21,14 @@
allsources.append(os.path.normpath(src))
return Extension(name=modname, sources=allsources, **kwds)
-def compile(tmpdir, ext, compiler_verbose=0):
+def compile(tmpdir, ext, compiler_verbose=0, target_extension=None,
+ embedding=False):
"""Compile a C extension module using distutils."""
saved_environ = os.environ.copy()
try:
- outputfilename = _build(tmpdir, ext, compiler_verbose)
+ outputfilename = _build(tmpdir, ext, compiler_verbose,
+ target_extension, embedding)
outputfilename = os.path.abspath(outputfilename)
finally:
# workaround for a distutils bugs where some env vars can
@@ -36,7 +38,32 @@
os.environ[key] = value
return outputfilename
-def _build(tmpdir, ext, compiler_verbose=0):
+def _save_val(name):
+ import distutils.sysconfig
+ config_vars = distutils.sysconfig.get_config_vars()
+ return config_vars.get(name, Ellipsis)
+
+def _restore_val(name, value):
+ import distutils.sysconfig
+ config_vars = distutils.sysconfig.get_config_vars()
+ config_vars[name] = value
+ if value is Ellipsis:
+ del config_vars[name]
+
+def _win32_hack_for_embedding():
+ from distutils.msvc9compiler import MSVCCompiler
+ if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'):
+ MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \
+ MSVCCompiler._remove_visual_c_ref
+ MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file:
manifest_file
+
+def _win32_unhack_for_embedding():
+ from distutils.msvc9compiler import MSVCCompiler
+ MSVCCompiler._remove_visual_c_ref = \
+ MSVCCompiler._remove_visual_c_ref_CFFI_BAK
+
+def _build(tmpdir, ext, compiler_verbose=0, target_extension=None,
+ embedding=False):
# XXX compact but horrible :-(
from distutils.core import Distribution
import distutils.errors, distutils.log
@@ -49,18 +76,29 @@
options['build_temp'] = ('ffiplatform', tmpdir)
#
try:
+ if sys.platform == 'win32' and embedding:
+ _win32_hack_for_embedding()
old_level = distutils.log.set_threshold(0) or 0
+ old_SO = _save_val('SO')
+ old_EXT_SUFFIX = _save_val('EXT_SUFFIX')
try:
+ if target_extension is not None:
+ _restore_val('SO', target_extension)
+ _restore_val('EXT_SUFFIX', target_extension)
distutils.log.set_verbosity(compiler_verbose)
dist.run_command('build_ext')
+ cmd_obj = dist.get_command_obj('build_ext')
+ [soname] = cmd_obj.get_outputs()
finally:
distutils.log.set_threshold(old_level)
+ _restore_val('SO', old_SO)
+ _restore_val('EXT_SUFFIX', old_EXT_SUFFIX)
+ if sys.platform == 'win32' and embedding:
+ _win32_unhack_for_embedding()
except (distutils.errors.CompileError,
distutils.errors.LinkError) as e:
raise VerificationError('%s: %s' % (e.__class__.__name__, e))
#
- cmd_obj = dist.get_command_obj('build_ext')
- [soname] = cmd_obj.get_outputs()
return soname
try:
diff --git a/cffi/recompiler.py b/cffi/recompiler.py
--- a/cffi/recompiler.py
+++ b/cffi/recompiler.py
@@ -3,6 +3,7 @@
from .cffi_opcode import *
VERSION = "0x2601"
+VERSION_EMBEDDED = "0x2701"
class GlobalExpr:
@@ -281,6 +282,29 @@
lines[i:i+1] = self._rel_readlines('parse_c_type.h')
prnt(''.join(lines))
#
+ # if we have ffi._embedding != None, we give it here as a macro
+ # and include an extra file
+ base_module_name = self.module_name.split('.')[-1]
+ if self.ffi._embedding is not None:
+ prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,))
+ prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' %
+ (self._string_literal(self.ffi._embedding),))
+ prnt('#ifdef PYPY_VERSION')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % (
+ base_module_name,))
+ prnt('#elif PY_MAJOR_VERSION >= 3')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC PyInit_%s' % (
+ base_module_name,))
+ prnt('#else')
+ prnt('# define _CFFI_PYTHON_STARTUP_FUNC init%s' % (
+ base_module_name,))
+ prnt('#endif')
+ lines = self._rel_readlines('_embedding.h')
+ prnt(''.join(lines))
+ version = VERSION_EMBEDDED
+ else:
+ version = VERSION
+ #
# then paste the C source given by the user, verbatim.
prnt('/************************************************************/')
prnt()
@@ -365,17 +389,16 @@
prnt()
#
# the init function
- base_module_name = self.module_name.split('.')[-1]
prnt('#ifdef PYPY_VERSION')
prnt('PyMODINIT_FUNC')
prnt('_cffi_pypyinit_%s(const void *p[])' % (base_module_name,))
prnt('{')
if self._num_externpy:
prnt(' if (((intptr_t)p[0]) >= 0x0A03) {')
- prnt(' _cffi_call_python = '
+ prnt(' _cffi_call_python_org = '
'(void(*)(struct _cffi_externpy_s *, char *))p[1];')
prnt(' }')
- prnt(' p[0] = (const void *)%s;' % VERSION)
+ prnt(' p[0] = (const void *)%s;' % version)
prnt(' p[1] = &_cffi_type_context;')
prnt('}')
# on Windows, distutils insists on putting init_cffi_xyz in
@@ -394,14 +417,14 @@
prnt('PyInit_%s(void)' % (base_module_name,))
prnt('{')
prnt(' return _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, VERSION))
+ self.module_name, version))
prnt('}')
prnt('#else')
prnt('PyMODINIT_FUNC')
prnt('init%s(void)' % (base_module_name,))
prnt('{')
prnt(' _cffi_init("%s", %s, &_cffi_type_context);' % (
- self.module_name, VERSION))
+ self.module_name, version))
prnt('}')
prnt('#endif')
@@ -1123,7 +1146,10 @@
assert isinstance(tp, model.FunctionPtrType)
self._do_collect_type(tp)
- def _generate_cpy_extern_python_decl(self, tp, name):
+ def _generate_cpy_dllexport_python_collecttype(self, tp, name):
+ self._generate_cpy_extern_python_collecttype(tp, name)
+
+ def _generate_cpy_extern_python_decl(self, tp, name, dllexport=False):
prnt = self._prnt
if isinstance(tp.result, model.VoidType):
size_of_result = '0'
@@ -1156,7 +1182,11 @@
size_of_a = 'sizeof(%s) > %d ? sizeof(%s) : %d' % (
tp.result.get_c_name(''), size_of_a,
tp.result.get_c_name(''), size_of_a)
- prnt('static %s' % tp.result.get_c_name(name_and_arguments))
+ if dllexport:
+ tag = 'CFFI_DLLEXPORT'
+ else:
+ tag = 'static'
+ prnt('%s %s' % (tag, tp.result.get_c_name(name_and_arguments)))
prnt('{')
prnt(' char a[%s];' % size_of_a)
prnt(' char *p = a;')
@@ -1174,6 +1204,9 @@
prnt()
self._num_externpy += 1
+ def _generate_cpy_dllexport_python_decl(self, tp, name):
+ self._generate_cpy_extern_python_decl(tp, name, dllexport=True)
+
def _generate_cpy_extern_python_ctx(self, tp, name):
if self.target_is_python:
raise ffiplatform.VerificationError(
@@ -1185,6 +1218,21 @@
self._lsts["global"].append(
GlobalExpr(name, '&_cffi_externpy__%s' % name, type_op, name))
+ def _generate_cpy_dllexport_python_ctx(self, tp, name):
+ self._generate_cpy_extern_python_ctx(tp, name)
+
+ def _string_literal(self, s):
+ def _char_repr(c):
+ # escape with a '\' the characters '\', '"' or (for trigraphs) '?'
+ if c in '\\"?': return '\\' + c
+ if ' ' <= c < '\x7F': return c
+ if c == '\n': return '\\n'
+ return '\\%03o' % ord(c)
+ lines = []
+ for line in s.splitlines(True):
+ lines.append('"%s"' % ''.join([_char_repr(c) for c in line]))
+ return ' \\\n'.join(lines)
+
# ----------
# emitting the opcodes for individual types
@@ -1311,12 +1359,15 @@
def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True,
c_file=None, source_extension='.c', extradir=None,
- compiler_verbose=1, **kwds):
+ compiler_verbose=1, target=None, **kwds):
if not isinstance(module_name, str):
module_name = module_name.encode('ascii')
if ffi._windows_unicode:
ffi._apply_windows_unicode(kwds)
if preamble is not None:
+ embedding = (ffi._embedding is not None)
+ if embedding:
+ ffi._apply_embedding_fix(kwds)
if c_file is None:
c_file, parts = _modname_to_file(tmpdir, module_name,
source_extension)
@@ -1325,13 +1376,40 @@
ext_c_file = os.path.join(*parts)
else:
ext_c_file = c_file
- ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds)
+ #
+ if target is None:
+ if embedding:
+ target = '%s.*' % module_name
+ else:
+ target = '*'
+ if target == '*':
+ target_module_name = module_name
+ target_extension = None # use default
+ else:
+ if target.endswith('.*'):
+ target = target[:-2]
+ if sys.platform == 'win32':
+ target += '.dll'
+ else:
+ target += '.so'
+ # split along the first '.' (not the last one, otherwise the
+ # preceeding dots are interpreted as splitting package names)
+ index = target.find('.')
+ if index < 0:
+ raise ValueError("target argument %r should be a file name "
+ "containing a '.'" % (target,))
+ target_module_name = target[:index]
+ target_extension = target[index:]
+ #
+ ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds)
updated = make_c_source(ffi, module_name, preamble, c_file)
if call_c_compiler:
cwd = os.getcwd()
try:
os.chdir(tmpdir)
- outputfilename = ffiplatform.compile('.', ext,
compiler_verbose)
+ outputfilename = ffiplatform.compile('.', ext,
compiler_verbose,
+ target_extension,
+ embedding=embedding)
finally:
os.chdir(cwd)
return outputfilename
diff --git a/demo/embedding.py b/demo/embedding.py
new file mode 100644
--- /dev/null
+++ b/demo/embedding.py
@@ -0,0 +1,21 @@
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add(int, int);
+""")
+
+ffi.embedding_init_code("""
+ from _embedding_cffi import ffi
+ print("preparing") # printed once
+
+ @ffi.def_extern()
+ def add(x, y):
+ print("adding %d and %d" % (x, y))
+ return x + y
+""")
+
+ffi.set_source("_embedding_cffi", "")
+
+ffi.compile(verbose=True)
diff --git a/demo/embedding_test.c b/demo/embedding_test.c
new file mode 100644
--- /dev/null
+++ b/demo/embedding_test.c
@@ -0,0 +1,19 @@
+/* Link this program with libembedding_test.so.
+ E.g. with gcc:
+
+ gcc -o embedding_test embedding_test.c _embedding_cffi*.so
+*/
+
+#include <stdio.h>
+
+extern int add(int x, int y);
+
+
+int main(void)
+{
+ int res = add(40, 2);
+ printf("result: %d\n", res);
+ res = add(100, -5);
+ printf("result: %d\n", res);
+ return 0;
+}
diff --git a/doc/source/cdef.rst b/doc/source/cdef.rst
--- a/doc/source/cdef.rst
+++ b/doc/source/cdef.rst
@@ -138,6 +138,8 @@
for ``lib.__class__`` before version 1.4.
+.. _cdef:
+
ffi.cdef(): declaring types and functions
-----------------------------------------
diff --git a/doc/source/conf.py b/doc/source/conf.py
--- a/doc/source/conf.py
+++ b/doc/source/conf.py
@@ -47,7 +47,7 @@
# The short X.Y version.
version = '1.4'
# The full version, including alpha/beta/rc tags.
-release = '1.4.2'
+release = '1.4.3'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
diff --git a/doc/source/embedding.rst b/doc/source/embedding.rst
new file mode 100644
--- /dev/null
+++ b/doc/source/embedding.rst
@@ -0,0 +1,369 @@
+================================
+Using CFFI for embedding
+================================
+
+.. contents::
+
+You can use CFFI to generate a ``.so/.dll`` which exports the API of
+your choice to any C application that wants to link with this
+``.so/.dll``.
+
+The general idea is as follows:
+
+* You write and execute a Python script, which produces a ``.so/.dll``
+ file with the API of your choice. The script also gives some Python
+ code to be "frozen" inside the ``.so``.
+
+* At runtime, the C application loads this ``.so/.dll`` without having
+ to know that it was produced by Python and CFFI.
+
+* The first time a C function is called, Python is initialized and
+ the frozen Python code is executed.
+
+* The frozen Python code attaches Python functions that implement the
+ C functions of your API, which are then used for all subsequent C
+ function calls.
+
+One of the goals of this approach is to be entirely independent from
+the CPython C API: no ``Py_Initialize()`` nor ``PyRun_SimpleString()``
+nor even ``PyObject``. It works identically on CPython and PyPy.
+
+.. note:: PyPy release 4.0.1 contains CFFI 1.4 only.
+
+This is entirely *new in version 1.5.*
+
+
+Usage
+-----
+
+.. __: overview.html#embedding
+
+See the `paragraph in the overview page`__ for a quick introduction.
+In this section, we explain every step in more details. We will use
+here this slightly expanded example:
+
+.. code-block:: c
+
+ /* file plugin.h */
+ typedef struct { int x, y; } point_t;
+ extern int do_stuff(point_t *);
+
+.. code-block:: python
+
+ # file plugin_build.py
+ import cffi
+ ffi = cffi.FFI()
+
+ with open('plugin.h') as f:
+ ffi.embedding_api(f.read())
+
+ ffi.set_source("my_plugin", '''
+ #include "plugin.h"
+ ''')
+
+ ffi.embedding_init_code("""
+ from my_plugin import ffi
+
+ @ffi.def_extern()
+ def do_stuff(p):
+ print("adding %d and %d" % (p.x, p.y))
+ return p.x + p.y
+ """)
+
+ ffi.compile(target="plugin-1.5.*", verbose=True)
+
+Running the code above produces a *DLL*, i,e, a dynamically-loadable
+library. It is a file with the extension ``.dll`` on Windows or
+``.so`` on other platforms. As usual, it is produced by generating
+some intermediate ``.c`` code and then calling the regular
+platform-specific C compiler.
+
+Here are some details about the methods used above:
+
+* **ffi.embedding_api(source):** parses the given C source, which
+ declares functions that you want to be exported by the DLL. It can
+ also declare types, constants and global variables that are part of
+ the C-level API of your DLL.
+
+ The functions that are found in ``source`` will be automatically
+ defined in the ``.c`` file: they will contain code that initializes
+ the Python interpreter the first time any of them is called,
+ followed by code to call the attached Python function (with
+ ``@ffi.def_extern()``, see next point).
+
+ The global variables, on the other hand, are not automatically
+ produced. You have to write their definition explicitly in
+ ``ffi.set_source()``, as regular C code (see the point after next).
+
+* **ffi.embedding_init_code(python_code):** this gives
+ initialization-time Python source code. This code is copied
+ ("frozen") inside the DLL. At runtime, the code is executed when
+ the DLL is first initialized, just after Python itself is
+ initialized. This newly initialized Python interpreter has got an
+ extra "built-in" module that can be loaded magically without
+ accessing any files, with a line like "``from my_plugin import ffi,
+ lib``". The name ``my_plugin`` comes from the first argument to
+ ``ffi.set_source()``. This module represents "the caller's C world"
+ from the point of view of Python.
+
+ The initialization-time Python code can import other modules or
+ packages as usual. You may have typical Python issues like needing
+ to set up ``sys.path`` somehow manually first.
+
+ For every function declared within ``ffi.embedding_api()``, the
+ initialization-time Python code or one of the modules it imports
+ should use the decorator ``@ffi.def_extern()`` to attach a
+ corresponding Python function to it.
+
+ If the initialization-time Python code fails with an exception, then
+ you get a traceback printed to stderr, along with more information
+ to help you identify problems like wrong ``sys.path``. If some
+ function remains unattached at the time where the C code tries to
+ call it, an error message is also printed to stderr and the function
+ returns zero/null.
+
+ Note that the CFFI module never calls ``exit()``, but CPython itself
+ contains code that calls ``exit()``, for example if importing
+ ``site`` fails. This may be worked around in the future.
+
+* **ffi.set_source(c_module_name, c_code):** set the name of the
+ module from Python's point of view. It also gives more C code which
+ will be included in the generated C code. In trivial examples it
+ can be an empty string. It is where you would ``#include`` some
+ other files, define global variables, and so on. The macro
+ ``CFFI_DLLEXPORT`` is available to this C code: it expands to the
+ platform-specific way of saying "the following declaration should be
+ exported from the DLL". For example, you would put "``extern int
+ my_glob;``" in ``ffi.embedding_api()`` and "``CFFI_DLLEXPORT int
+ my_glob = 42;``" in ``ffi.set_source()``.
+
+ Currently, any *type* declared in ``ffi.embedding_api()`` must also
+ be present in the ``c_code``. This is automatic if this code
+ contains a line like ``#include "plugin.h"`` in the example above.
+
+* **ffi.compile([target=...] [, verbose=True]):** make the C code and
+ compile it. By default, it produces a file called
+ ``c_module_name.dll`` or ``c_module_name.so``, but the default can
+ be changed with the optional ``target`` keyword argument. You can
+ use ``target="foo.*"`` with a literal ``*`` to ask for a file called
+ ``foo.dll`` on Windows or ``foo.so`` elsewhere. One reason for
+ specifying an alternate ``target`` is to include characters not
+ usually allowed in Python module names, like "``plugin-1.5.*``".
+
+ For more complicated cases, you can call instead
+ ``ffi.emit_c_code("foo.c")`` and compile the resulting ``foo.c``
+ file using other means. CFFI's compilation logic is based on the
+ standard library ``distutils`` package, which is really developed
+ and tested for the purpose of making CPython extension modules, not
+ other DLLs.
+
+
+More reading
+------------
+
+If you're reading this page about embedding and you are not familiar
+with CFFI already, here are a few pointers to what you could read
+next:
+
+* For the ``@ffi.def_extern()`` functions, integer C types are passed
+ simply as Python integers; and simple pointers-to-struct and basic
+ arrays are all straightforward enough. However, sooner or later you
+ will need to read about this topic in more details here__.
+
+* ``@ffi.def_extern()``: see `documentation here,`__ notably on what
+ happens if the Python function raises an exception.
+
+* To create Python objects attached to C data, one common solution is
+ to use ``ffi.new_handle()``. See documentation here__.
+
+* In embedding mode, the major direction is C code that calls Python
+ functions. This is the opposite of the regular extending mode of
+ CFFI, in which the major direction is Python code calling C. That's
+ why the page `Using the ffi/lib objects`_ talks first about the
+ latter, and why the direction "C code that calls Python" is
+ generally referred to as "callbacks" in that page. If you also
+ need to have your Python code call C code, read more about
+ `Embedding and Extending`_ below.
+
+* ``ffi.embedding_api(source)``: follows the same syntax as
+ ``ffi.cdef()``, `documented here.`__ You can use the "``...``"
+ syntax as well, although in practice it may be less useful than it
+ is for ``cdef()``. On the other hand, it is expected that often the
+ C sources that you need to give to ``ffi.embedding_api()`` would be
+ exactly the same as the content of some ``.h`` file that you want to
+ give to users of your DLL. That's why the example above does this::
+
+ with open('foo.h') as f:
+ ffi.embedding(f.read())
+
+ Note that a drawback of this approach is that ``ffi.embedding()``
+ doesn't support ``#ifdef`` directives. You may have to use a more
+ convoluted expression like::
+
+ with open('foo.h') as f:
+ lines = [line for line in f if not line.startswith('#')]
+ ffi.embedding(''.join(lines))
+
+ As in the example above, you can also use the same ``foo.h`` from
+ ``ffi.set_source()``::
+
+ ffi.set_source('module_name', '#include "foo.h"')
+
+
+.. __: using.html#working
+.. __: using.html#def-extern
+.. __: using.html#ffi-new_handle
+.. __: cdef.html#cdef
+
+.. _`Using the ffi/lib objects`: using.html
+
+
+Troubleshooting
+---------------
+
+The error message
+
+ cffi extension module 'c_module_name' has unknown version 0x2701
+
+means that the running Python interpreter located a CFFI version older
+than 1.5. CFFI 1.5 or newer must be installed in the running Python.
+
+
+Using multiple CFFI-made DLLs
+-----------------------------
+
+Multiple CFFI-made DLLs can be used by the same process.
+
+Note that all CFFI-made DLLs in a process share a single Python
+interpreter. The effect is the same as the one you get by trying to
+build a large Python application by assembling a lot of unrelated
+packages. Some of these might be libraries that monkey-patch some
+functions from the standard library, for example, which might be
+unexpected from other parts.
+
+
+Multithreading
+--------------
+
+Multithreading should work transparently, based on Python's standard
+Global Interpreter Lock.
+
+If two threads both try to call a C function when Python is not yet
+initialized, then locking occurs. One thread proceeds with
+initialization and blocks the other thread. The other thread will be
+allowed to continue only when the execution of the initialization-time
+Python code is done.
+
+If the two threads call two *different* CFFI-made DLLs, the Python
+initialization itself will still be serialized, but the two pieces of
+initialization-time Python code will not. The idea is that there is a
+priori no reason for one DLL to wait for initialization of the other
+DLL to be complete.
+
+After initialization, Python's standard Global Interpreter Lock kicks
+in. The end result is that when one CPU progresses on executing
+Python code, no other CPU can progress on executing more Python code
+from another thread of the same process. At regular intervals, the
+lock switches to a different thread, so that no single thread should
+appear to block indefinitely.
+
+
+Testing
+-------
+
+For testing purposes, a CFFI-made DLL can be imported in a running
+Python interpreter instead of being loaded like a C shared library.
+
+You might have some issues with the file name: for example, on
+Windows, Python expects the file to be called ``c_module_name.pyd``,
+but the CFFI-made DLL is called ``target.dll`` instead. The base name
+``target`` is the one specified in ``ffi.compile()``, and on Windows
+the extension is ``.dll`` instead of ``.pyd``. You have to rename or
+copy the file, or on POSIX use a symlink.
+
+The module then works like a regular CFFI extension module. It is
+imported with "``from c_module_name import ffi, lib``" and exposes on
+the ``lib`` object all C functions. You can test it by calling these
+C functions. The initialization-time Python code frozen inside the
+DLL is executed the first time such a call is done.
+
+
+Embedding and Extending
+-----------------------
+
+The embedding mode is not incompatible with the non-embedding mode of
+CFFI.
+
+You can use *both* ``ffi.embedding_api()`` and ``ffi.cdef()`` in the
+same build script. You put in the former the declarations you want to
+be exported by the DLL; you put in the latter only the C functions and
+types that you want to share between C and Python, but not export from
+the DLL.
+
+As an example of that, consider the case where you would like to have
+a DLL-exported C function written in C directly, maybe to handle some
+cases before calling Python functions. To do that, you must *not* put
+the function's signature in ``ffi.embedding_api()``. (Note that this
+requires more hacks if you use ``ffi.embedding(f.read())``.) You must
+only write the custom function definition in ``ffi.set_source()``, and
+prefix it with the macro CFFI_DLLEXPORT:
+
+.. code-block:: c
+
+ CFFI_DLLEXPORT int myfunc(int a, int b)
+ {
+ /* implementation here */
+ }
+
+This function can, if it wants, invoke Python functions using the
+general mechanism of "callbacks"---called this way because it is a
+call from C to Python, although in this case it is not calling
+anything back:
+
+.. code-block:: python
+
+ ffi.cdef("""
+ extern "Python" int mycb(int);
+ """)
+
+ ffi.set_source("my_plugin", """
+
+ static int mycb(int); /* the callback: forward declaration, to make
+ it accessible from the C code that follows
*/
+
+ CFFI_DLLEXPORT int myfunc(int a, int b)
+ {
+ int product = a * b; /* some custom C code */
+ return mycb(product);
+ }
+ """)
+
+and then the Python initialization code needs to contain the lines:
+
+.. code-block:: python
+
+ @ffi.def_extern()
+ def mycb(x):
+ print "hi, I'm called with x =", x
+ return x * 10
+
+This ``@ffi.def_extern`` is attaching a Python function to the C
+callback ``mycb()``, which in this case is not exported from the DLL.
+Nevertheless, the automatic initialization of Python occurs when
+``mycb()`` is called, if it happens to be the first function called
+from C. More precisely, it does not happen when ``myfunc()`` is
+called: this is just a C function, with no extra code magically
+inserted around it. It only happens when ``myfunc()`` calls
+``mycb()``.
+
+As the above explanation hints, this is how ``ffi.embedding_api()``
+actually implements function calls that directly invoke Python code;
+here, we have merely decomposed it explicitly, in order to add some
+custom C code in the middle.
+
+In case you need to force, from C code, Python to be initialized
+before the first ``@ffi.def_extern()`` is called, you can do so by
+calling the C function ``cffi_start_python()`` with no argument. It
+returns an integer, 0 or -1, to tell if the initialization succeeded
+or not. Currently there is no way to prevent a failing initialization
+from also dumping a traceback and more information to stderr.
diff --git a/doc/source/index.rst b/doc/source/index.rst
--- a/doc/source/index.rst
+++ b/doc/source/index.rst
@@ -18,6 +18,7 @@
overview
using
cdef
+ embedding
Goals
diff --git a/doc/source/installation.rst b/doc/source/installation.rst
--- a/doc/source/installation.rst
+++ b/doc/source/installation.rst
@@ -51,11 +51,11 @@
Download and Installation:
-* http://pypi.python.org/packages/source/c/cffi/cffi-1.4.2.tar.gz
+* http://pypi.python.org/packages/source/c/cffi/cffi-1.4.3.tar.gz
- - MD5: 81357fe5042d00650b85b728cc181df2
+ - MD5: ...
- - SHA: 76cff6f1ff5bfb2b9c6c8e2cfa8bf90b5c944394
+ - SHA: ...
* Or grab the most current version from the `Bitbucket page`_:
``hg clone https://bitbucket.org/cffi/cffi``
diff --git a/doc/source/overview.rst b/doc/source/overview.rst
--- a/doc/source/overview.rst
+++ b/doc/source/overview.rst
@@ -287,6 +287,54 @@
distributed in precompiled form like any other extension module.*
+.. _embedding:
+
+Embedding
+---------
+
+*New in version 1.5.*
+
+CFFI can be used for embedding__: creating a standard
+dynamically-linked library (``.dll`` under Windows, ``.so`` elsewhere)
+which can be used from a C application.
+
+.. code-block:: python
+
+ import cffi
+ ffi = cffi.FFI()
+
+ ffi.embedding_api("""
+ int do_stuff(int, int);
+ """)
+
+ ffi.set_source("my_plugin", "")
+
+ ffi.embedding_init_code("""
+ from my_plugin import ffi
+
+ @ffi.def_extern()
+ def do_stuff(x, y):
+ print("adding %d and %d" % (x, y))
+ return x + y
+ """)
+
+ ffi.compile(target="plugin-1.5.*", verbose=True)
+
+This simple example creates ``plugin-1.5.dll`` or ``plugin-1.5.so`` as
+a DLL with a single exported function, ``do_stuff()``. You execute
+the script above once, with the interpreter you want to have
+internally used; it can be CPython 2.x or 3.x or PyPy. This DLL can
+then be used "as usual" from an application; the application doesn't
+need to know that it is talking with a library made with Python and
+CFFI. At runtime, when the application calls ``int do_stuff(int,
+int)``, the Python interpreter is automatically initialized and ``def
+do_stuff(x, y):`` gets called. `See the details in the documentation
+about embedding.`__
+
+.. __: embedding.html
+.. __: embedding.html
+
+
What actually happened?
-----------------------
diff --git a/doc/source/using.rst b/doc/source/using.rst
--- a/doc/source/using.rst
+++ b/doc/source/using.rst
@@ -423,6 +423,7 @@
with ``int foo();`` really means ``int foo(void);``.)
+.. _extern-python:
.. _`extern "Python"`:
Extern "Python" (new-style callbacks)
@@ -603,6 +604,7 @@
}
""")
+
Extern "Python": reference
~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -629,6 +631,8 @@
return a default value. This can be controlled with ``error`` and
``onerror``, described below.
+.. _def-extern:
+
The ``@ffi.def_extern()`` decorator takes these optional arguments:
* ``name``: the name of the function as written in the cdef. By default
@@ -1066,12 +1070,13 @@
points in time, and using it in a ``with`` statement.
+.. _ffi-new_handle:
.. _`ffi.new_handle()`:
**ffi.new_handle(python_object)**: return a non-NULL cdata of type
``void *`` that contains an opaque reference to ``python_object``. You
can pass it around to C functions or store it into C structures. Later,
-you can use **ffi.from_handle(p)** to retrive the original
+you can use **ffi.from_handle(p)** to retrieve the original
``python_object`` from a value with the same ``void *`` pointer.
*Calling ffi.from_handle(p) is invalid and will likely crash if
the cdata object returned by new_handle() is not kept alive!*
diff --git a/setup.py b/setup.py
--- a/setup.py
+++ b/setup.py
@@ -144,9 +144,10 @@
`Mailing list <https://groups.google.com/forum/#!forum/python-cffi>`_
""",
- version='1.4.2',
+ version='1.4.3',
packages=['cffi'] if cpython else [],
- package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h']}
+ package_data={'cffi': ['_cffi_include.h', 'parse_c_type.h',
+ '_embedding.h']}
if cpython else {},
zip_safe=False,
diff --git a/testing/cffi0/test_version.py b/testing/cffi0/test_version.py
--- a/testing/cffi0/test_version.py
+++ b/testing/cffi0/test_version.py
@@ -53,3 +53,10 @@
content = open(p).read()
#v = BACKEND_VERSIONS.get(v, v)
assert (('assert __version__ == "%s"' % v) in content)
+
+def test_embedding_h():
+ parent = os.path.dirname(os.path.dirname(cffi.__file__))
+ v = cffi.__version__
+ p = os.path.join(parent, 'cffi', '_embedding.h')
+ content = open(p).read()
+ assert ('cffi version: %s"' % (v,)) in content
diff --git a/testing/cffi1/test_zdist.py b/testing/cffi1/test_zdist.py
--- a/testing/cffi1/test_zdist.py
+++ b/testing/cffi1/test_zdist.py
@@ -59,11 +59,16 @@
if (name.endswith('.so') or name.endswith('.pyd') or
name.endswith('.dylib')):
found_so = os.path.join(curdir, name)
- # foo.cpython-34m.so => foo
- name = name.split('.')[0]
- # foo_d.so => foo (Python 2 debug builds)
+ # foo.so => foo
+ parts = name.split('.')
+ del parts[-1]
+ if len(parts) > 1 and parts[-1] != 'bar':
+ # foo.cpython-34m.so => foo, but foo.bar.so => foo.bar
+ del parts[-1]
+ name = '.'.join(parts)
+ # foo_d => foo (Python 2 debug builds)
if name.endswith('_d') and hasattr(sys, 'gettotalrefcount'):
- name = name.rsplit('_', 1)[0]
+ name = name[:-2]
name += '.SO'
if name.startswith('pycparser') and name.endswith('.egg'):
continue # no clue why this shows up sometimes and not
others
@@ -208,6 +213,58 @@
'Release': '?'}})
@chdir_to_tmp
+ def test_api_compile_explicit_target_1(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target="foo.bar.*")
+ if sys.platform != 'win32':
+ sofile = self.check_produced_files({
+ 'foo.bar.SO': None,
+ 'mod_name_in_package': {'mymod.c': None,
+ 'mymod.o': None}})
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'foo.bar.SO': None,
+ 'mod_name_in_package': {'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
+ def test_api_compile_explicit_target_2(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target=os.path.join("mod_name_in_package",
"foo.bar.*"))
+ if sys.platform != 'win32':
+ sofile = self.check_produced_files({
+ 'mod_name_in_package': {'foo.bar.SO': None,
+ 'mymod.c': None,
+ 'mymod.o': None}})
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'mod_name_in_package': {'foo.bar.SO': None,
+ 'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
+ def test_api_compile_explicit_target_3(self):
+ ffi = cffi.FFI()
+ ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
+ x = ffi.compile(target="foo.bar.baz")
+ if sys.platform != 'win32':
+ self.check_produced_files({
+ 'foo.bar.baz': None,
+ 'mod_name_in_package': {'mymod.c': None,
+ 'mymod.o': None}})
+ sofile = os.path.join(str(self.udir), 'foo.bar.baz')
+ assert os.path.isabs(x) and os.path.samefile(x, sofile)
+ else:
+ self.check_produced_files({
+ 'foo.bar.baz': None,
+ 'mod_name_in_package': {'mymod.c': None},
+ 'Release': '?'})
+
+ @chdir_to_tmp
def test_api_distutils_extension_1(self):
ffi = cffi.FFI()
ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/")
diff --git a/testing/embedding/__init__.py b/testing/embedding/__init__.py
new file mode 100644
diff --git a/testing/embedding/add1-test.c b/testing/embedding/add1-test.c
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add1-test.c
@@ -0,0 +1,13 @@
+#include <stdio.h>
+
+extern int add1(int, int);
+
+
+int main(void)
+{
+ int x, y;
+ x = add1(40, 2);
+ y = add1(100, -5);
+ printf("got: %d %d\n", x, y);
+ return 0;
+}
diff --git a/testing/embedding/add1.py b/testing/embedding/add1.py
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add1.py
@@ -0,0 +1,33 @@
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add1(int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ import sys, time
+ sys.stdout.write("preparing")
+ for i in range(3):
+ sys.stdout.flush()
+ time.sleep(0.02)
+ sys.stdout.write(".")
+ sys.stdout.write("\n")
+
+ from _add1_cffi import ffi
+
+ int(ord("A")) # check that built-ins are there
+
+ @ffi.def_extern()
+ def add1(x, y):
+ sys.stdout.write("adding %d and %d\n" % (x, y))
+ sys.stdout.flush()
+ return x + y
+""")
+
+ffi.set_source("_add1_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/testing/embedding/add2-test.c b/testing/embedding/add2-test.c
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add2-test.c
@@ -0,0 +1,14 @@
+#include <stdio.h>
+
+extern int add1(int, int);
+extern int add2(int, int, int);
+
+
+int main(void)
+{
+ int x, y;
+ x = add1(40, 2);
+ y = add2(100, -5, -20);
+ printf("got: %d %d\n", x, y);
+ return 0;
+}
diff --git a/testing/embedding/add2.py b/testing/embedding/add2.py
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add2.py
@@ -0,0 +1,29 @@
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add2(int, int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ import sys
+ sys.stdout.write("prepADD2\n")
+
+ assert '_add2_cffi' in sys.modules
+ m = sys.modules['_add2_cffi']
+ import _add2_cffi
+ ffi = _add2_cffi.ffi
+
+ @ffi.def_extern()
+ def add2(x, y, z):
+ sys.stdout.write("adding %d and %d and %d\n" % (x, y, z))
+ sys.stdout.flush()
+ return x + y + z
+""")
+
+ffi.set_source("_add2_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/testing/embedding/add3.py b/testing/embedding/add3.py
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add3.py
@@ -0,0 +1,24 @@
+import cffi
+
+ffi = cffi.FFI()
+
+ffi.embedding_api("""
+ int add3(int, int, int, int);
+""")
+
+ffi.embedding_init_code(r"""
+ from _add3_cffi import ffi
+ import sys
+
+ @ffi.def_extern()
+ def add3(x, y, z, t):
+ sys.stdout.write("adding %d, %d, %d, %d\n" % (x, y, z, t))
+ sys.stdout.flush()
+ return x + y + z + t
+""")
+
+ffi.set_source("_add3_cffi", """
+""")
+
+fn = ffi.compile(verbose=True)
+print('FILENAME: %s' % (fn,))
diff --git a/testing/embedding/add_recursive-test.c
b/testing/embedding/add_recursive-test.c
new file mode 100644
--- /dev/null
+++ b/testing/embedding/add_recursive-test.c
@@ -0,0 +1,27 @@
+#include <stdio.h>
+
+#ifdef _MSC_VER
+# define DLLIMPORT __declspec(dllimport)
+#else
+# define DLLIMPORT extern
+#endif
+
+DLLIMPORT int add_rec(int, int);
+DLLIMPORT int (*my_callback)(int);
+
+static int some_callback(int x)
+{
+ printf("some_callback(%d)\n", x);
+ fflush(stdout);
+ return add_rec(x, 9);
+}
+
+int main(void)
+{
+ int x, y;
+ my_callback = some_callback;
+ x = add_rec(40, 2);
+ y = add_rec(100, -5);
_______________________________________________
pypy-commit mailing list
pypy-commit@python.org
https://mail.python.org/mailman/listinfo/pypy-commit
