Author: Armin Rigo <ar...@tunes.org> Branch: cffi-static-callback-embedding Changeset: r81792:e5825665a3ae Date: 2016-01-15 14:15 +0100 http://bitbucket.org/pypy/pypy/changeset/e5825665a3ae/
Log: update to cffi/4ce79f707838 diff --git a/lib_pypy/cffi.egg-info/PKG-INFO b/lib_pypy/cffi.egg-info/PKG-INFO --- a/lib_pypy/cffi.egg-info/PKG-INFO +++ b/lib_pypy/cffi.egg-info/PKG-INFO @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cffi -Version: 1.4.3 +Version: 1.5.0 Summary: Foreign Function Interface for Python calling C code. Home-page: http://cffi.readthedocs.org Author: Armin Rigo, Maciej Fijalkowski diff --git a/lib_pypy/cffi/__init__.py b/lib_pypy/cffi/__init__.py --- a/lib_pypy/cffi/__init__.py +++ b/lib_pypy/cffi/__init__.py @@ -4,8 +4,8 @@ from .api import FFI, CDefError, FFIError from .ffiplatform import VerificationError, VerificationMissing -__version__ = "1.4.3" -__version_info__ = (1, 4, 3) +__version__ = "1.5.0" +__version_info__ = (1, 5, 0) # The verifier module file names are based on the CRC32 of a string that # contains the following version number. It may be older than __version__ diff --git a/lib_pypy/cffi/api.py b/lib_pypy/cffi/api.py --- a/lib_pypy/cffi/api.py +++ b/lib_pypy/cffi/api.py @@ -74,7 +74,7 @@ self._windows_unicode = None self._init_once_cache = {} self._cdef_version = None - self._embedding_init_code = None + self._embedding = None if hasattr(backend, 'set_ffi'): backend.set_ffi(self) for name in backend.__dict__: @@ -94,7 +94,7 @@ self.NULL = self.cast(self.BVoidP, 0) self.CData, self.CType = backend._get_types() - def cdef(self, csource, override=False, packed=False, dllexport=False): + def cdef(self, csource, override=False, packed=False): """Parse the given C source. This registers all declared functions, types, and global variables. The functions and global variables can then be accessed via either 'ffi.dlopen()' or 'ffi.verify()'. @@ -102,14 +102,21 @@ If 'packed' is specified as True, all structs declared inside this cdef are packed, i.e. laid out without any field alignment at all. """ + self._cdef(csource, override=override, packed=packed) + + def embedding_api(self, csource, packed=False): + self._cdef(csource, packed=packed, dllexport=True) + if self._embedding is None: + self._embedding = '' + + def _cdef(self, csource, override=False, **options): if not isinstance(csource, str): # unicode, on Python 2 if not isinstance(csource, basestring): raise TypeError("cdef() argument must be a string") csource = csource.encode('ascii') with self._lock: self._cdef_version = object() - self._parser.parse(csource, override=override, packed=packed, - dllexport=dllexport) + self._parser.parse(csource, override=override, **options) self._cdefsources.append(csource) if override: for cache in self._function_caches: @@ -535,6 +542,25 @@ ('_UNICODE', '1')] kwds['define_macros'] = defmacros + def _apply_embedding_fix(self, kwds): + # must include an argument like "-lpython2.7" for the compiler + if '__pypy__' in sys.builtin_module_names: + pythonlib = "pypy-c" + else: + if sys.platform == "win32": + template = "python%d%d" + if sys.flags.debug: + template = template + '_d' + else: + template = "python%d.%d" + pythonlib = (template % + (sys.hexversion >> 24, (sys.hexversion >> 16) & 0xff)) + if hasattr(sys, 'abiflags'): + pythonlib += sys.abiflags + libraries = kwds.get('libraries', []) + if pythonlib not in libraries: + kwds['libraries'] = libraries + [pythonlib] + def set_source(self, module_name, source, source_extension='.c', **kwds): if hasattr(self, '_assigned_source'): raise ValueError("set_source() cannot be called several times " @@ -594,14 +620,23 @@ recompile(self, module_name, source, c_file=filename, call_c_compiler=False, **kwds) - def compile(self, tmpdir='.', verbose=0): + def compile(self, tmpdir='.', verbose=0, target=None): + """The 'target' argument gives the final file name of the + compiled DLL. Use '*' to force distutils' choice, suitable for + regular CPython C API modules. Use a file name ending in '.*' + to ask for the system's default extension for dynamic libraries + (.so/.dll). + + The default is '*' when building a non-embedded C API extension, + and (module_name + '.*') when building an embedded library. + """ from .recompiler import recompile # if not hasattr(self, '_assigned_source'): raise ValueError("set_source() must be called before compile()") module_name, source, source_extension, kwds = self._assigned_source return recompile(self, module_name, source, tmpdir=tmpdir, - source_extension=source_extension, + target=target, source_extension=source_extension, compiler_verbose=verbose, **kwds) def init_once(self, func, tag): @@ -629,17 +664,30 @@ return result def embedding_init_code(self, pysource): - if self._embedding_init_code is not None: + if self._embedding: raise ValueError("embedding_init_code() can only be called once") - # check for SyntaxErrors, at least, and automatically add a - # "if 1:" line in front of the code if the whole pysource is - # indented - try: - compile(pysource, "cffi_init", "exec") - except IndentationError: - pysource = 'if 1:\n' + pysource - compile(pysource, "cffi_init", "exec") - self._embedding_init_code = pysource + # fix 'pysource' before it gets dumped into the C file: + # - remove empty lines at the beginning, so it starts at "line 1" + # - dedent, if all non-empty lines are indented + # - check for SyntaxErrors + import re + match = re.match(r'\s*\n', pysource) + if match: + pysource = pysource[match.end():] + lines = pysource.splitlines() or [''] + prefix = re.match(r'\s*', lines[0]).group() + for i in range(1, len(lines)): + line = lines[i] + if line.rstrip(): + while not line.startswith(prefix): + prefix = prefix[:-1] + i = len(prefix) + lines = [line[i:]+'\n' for line in lines] + pysource = ''.join(lines) + # + compile(pysource, "cffi_init", "exec") + # + self._embedding = pysource def _load_backend_lib(backend, name, flags): diff --git a/lib_pypy/cffi/cparser.py b/lib_pypy/cffi/cparser.py --- a/lib_pypy/cffi/cparser.py +++ b/lib_pypy/cffi/cparser.py @@ -374,11 +374,10 @@ def _declare_function(self, tp, quals, decl): tp = self._get_type_pointer(tp, quals) - if self._inside_extern_python: - if self._options['dllexport']: - tag = 'dllexport_python ' - else: - tag = 'extern_python ' + if self._options['dllexport']: + tag = 'dllexport_python ' + elif self._inside_extern_python: + tag = 'extern_python ' else: tag = 'function ' self._declare(tag + decl.name, tp) diff --git a/lib_pypy/cffi/ffiplatform.py b/lib_pypy/cffi/ffiplatform.py --- a/lib_pypy/cffi/ffiplatform.py +++ b/lib_pypy/cffi/ffiplatform.py @@ -21,12 +21,14 @@ allsources.append(os.path.normpath(src)) return Extension(name=modname, sources=allsources, **kwds) -def compile(tmpdir, ext, compiler_verbose=0): +def compile(tmpdir, ext, compiler_verbose=0, target_extension=None, + embedding=False): """Compile a C extension module using distutils.""" saved_environ = os.environ.copy() try: - outputfilename = _build(tmpdir, ext, compiler_verbose) + outputfilename = _build(tmpdir, ext, compiler_verbose, + target_extension, embedding) outputfilename = os.path.abspath(outputfilename) finally: # workaround for a distutils bugs where some env vars can @@ -36,7 +38,32 @@ os.environ[key] = value return outputfilename -def _build(tmpdir, ext, compiler_verbose=0): +def _save_val(name): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + return config_vars.get(name, Ellipsis) + +def _restore_val(name, value): + import distutils.sysconfig + config_vars = distutils.sysconfig.get_config_vars() + config_vars[name] = value + if value is Ellipsis: + del config_vars[name] + +def _win32_hack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + if not hasattr(MSVCCompiler, '_remove_visual_c_ref_CFFI_BAK'): + MSVCCompiler._remove_visual_c_ref_CFFI_BAK = \ + MSVCCompiler._remove_visual_c_ref + MSVCCompiler._remove_visual_c_ref = lambda self,manifest_file: manifest_file + +def _win32_unhack_for_embedding(): + from distutils.msvc9compiler import MSVCCompiler + MSVCCompiler._remove_visual_c_ref = \ + MSVCCompiler._remove_visual_c_ref_CFFI_BAK + +def _build(tmpdir, ext, compiler_verbose=0, target_extension=None, + embedding=False): # XXX compact but horrible :-( from distutils.core import Distribution import distutils.errors, distutils.log @@ -49,18 +76,29 @@ options['build_temp'] = ('ffiplatform', tmpdir) # try: + if sys.platform == 'win32' and embedding: + _win32_hack_for_embedding() old_level = distutils.log.set_threshold(0) or 0 + old_SO = _save_val('SO') + old_EXT_SUFFIX = _save_val('EXT_SUFFIX') try: + if target_extension is not None: + _restore_val('SO', target_extension) + _restore_val('EXT_SUFFIX', target_extension) distutils.log.set_verbosity(compiler_verbose) dist.run_command('build_ext') + cmd_obj = dist.get_command_obj('build_ext') + [soname] = cmd_obj.get_outputs() finally: distutils.log.set_threshold(old_level) + _restore_val('SO', old_SO) + _restore_val('EXT_SUFFIX', old_EXT_SUFFIX) + if sys.platform == 'win32' and embedding: + _win32_unhack_for_embedding() except (distutils.errors.CompileError, distutils.errors.LinkError) as e: raise VerificationError('%s: %s' % (e.__class__.__name__, e)) # - cmd_obj = dist.get_command_obj('build_ext') - [soname] = cmd_obj.get_outputs() return soname try: diff --git a/lib_pypy/cffi/recompiler.py b/lib_pypy/cffi/recompiler.py --- a/lib_pypy/cffi/recompiler.py +++ b/lib_pypy/cffi/recompiler.py @@ -282,13 +282,13 @@ lines[i:i+1] = self._rel_readlines('parse_c_type.h') prnt(''.join(lines)) # - # if we have ffi._embedding_init_code, we give it here as a macro + # if we have ffi._embedding != None, we give it here as a macro # and include an extra file base_module_name = self.module_name.split('.')[-1] - if self.ffi._embedding_init_code is not None: + if self.ffi._embedding is not None: prnt('#define _CFFI_MODULE_NAME "%s"' % (self.module_name,)) prnt('#define _CFFI_PYTHON_STARTUP_CODE %s' % - (self._string_literal(self.ffi._embedding_init_code),)) + (self._string_literal(self.ffi._embedding),)) prnt('#ifdef PYPY_VERSION') prnt('# define _CFFI_PYTHON_STARTUP_FUNC _cffi_pypyinit_%s' % ( base_module_name,)) @@ -1359,12 +1359,15 @@ def recompile(ffi, module_name, preamble, tmpdir='.', call_c_compiler=True, c_file=None, source_extension='.c', extradir=None, - compiler_verbose=1, **kwds): + compiler_verbose=1, target=None, **kwds): if not isinstance(module_name, str): module_name = module_name.encode('ascii') if ffi._windows_unicode: ffi._apply_windows_unicode(kwds) if preamble is not None: + embedding = (ffi._embedding is not None) + if embedding: + ffi._apply_embedding_fix(kwds) if c_file is None: c_file, parts = _modname_to_file(tmpdir, module_name, source_extension) @@ -1373,13 +1376,40 @@ ext_c_file = os.path.join(*parts) else: ext_c_file = c_file - ext = ffiplatform.get_extension(ext_c_file, module_name, **kwds) + # + if target is None: + if embedding: + target = '%s.*' % module_name + else: + target = '*' + if target == '*': + target_module_name = module_name + target_extension = None # use default + else: + if target.endswith('.*'): + target = target[:-2] + if sys.platform == 'win32': + target += '.dll' + else: + target += '.so' + # split along the first '.' (not the last one, otherwise the + # preceeding dots are interpreted as splitting package names) + index = target.find('.') + if index < 0: + raise ValueError("target argument %r should be a file name " + "containing a '.'" % (target,)) + target_module_name = target[:index] + target_extension = target[index:] + # + ext = ffiplatform.get_extension(ext_c_file, target_module_name, **kwds) updated = make_c_source(ffi, module_name, preamble, c_file) if call_c_compiler: cwd = os.getcwd() try: os.chdir(tmpdir) - outputfilename = ffiplatform.compile('.', ext, compiler_verbose) + outputfilename = ffiplatform.compile('.', ext, compiler_verbose, + target_extension, + embedding=embedding) finally: os.chdir(cwd) return outputfilename diff --git a/pypy/module/_cffi_backend/__init__.py b/pypy/module/_cffi_backend/__init__.py --- a/pypy/module/_cffi_backend/__init__.py +++ b/pypy/module/_cffi_backend/__init__.py @@ -3,7 +3,7 @@ from rpython.rlib import rdynload, clibffi, entrypoint from rpython.rtyper.lltypesystem import rffi -VERSION = "1.4.3" +VERSION = "1.5.0" FFI_DEFAULT_ABI = clibffi.FFI_DEFAULT_ABI try: diff --git a/pypy/module/_cffi_backend/test/_backend_test_c.py b/pypy/module/_cffi_backend/test/_backend_test_c.py --- a/pypy/module/_cffi_backend/test/_backend_test_c.py +++ b/pypy/module/_cffi_backend/test/_backend_test_c.py @@ -1,7 +1,7 @@ # ____________________________________________________________ import sys -assert __version__ == "1.4.3", ("This test_c.py file is for testing a version" +assert __version__ == "1.5.0", ("This test_c.py file is for testing a version" " of cffi that differs from the one that we" " get from 'import _cffi_backend'") if sys.version_info < (3,): diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_new_ffi_1.py @@ -1719,3 +1719,10 @@ exec("from _test_import_from_lib.lib import *", d) assert (set(key for key in d if not key.startswith('_')) == set(['myfunc', 'MYFOO'])) + # + # also test "import *" on the module itself, which should be + # equivalent to "import ffi, lib" + d = {} + exec("from _test_import_from_lib import *", d) + assert (sorted([x for x in d.keys() if not x.startswith('__')]) == + ['ffi', 'lib']) diff --git a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py --- a/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py +++ b/pypy/module/test_lib_pypy/cffi_tests/cffi1/test_zdist.py @@ -60,11 +60,16 @@ if (name.endswith('.so') or name.endswith('.pyd') or name.endswith('.dylib')): found_so = os.path.join(curdir, name) - # foo.cpython-34m.so => foo - name = name.split('.')[0] - # foo_d.so => foo (Python 2 debug builds) + # foo.so => foo + parts = name.split('.') + del parts[-1] + if len(parts) > 1 and parts[-1] != 'bar': + # foo.cpython-34m.so => foo, but foo.bar.so => foo.bar + del parts[-1] + name = '.'.join(parts) + # foo_d => foo (Python 2 debug builds) if name.endswith('_d') and hasattr(sys, 'gettotalrefcount'): - name = name.rsplit('_', 1)[0] + name = name[:-2] name += '.SO' if name.startswith('pycparser') and name.endswith('.egg'): continue # no clue why this shows up sometimes and not others @@ -209,6 +214,58 @@ 'Release': '?'}}) @chdir_to_tmp + def test_api_compile_explicit_target_1(self): + ffi = cffi.FFI() + ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/") + x = ffi.compile(target="foo.bar.*") + if sys.platform != 'win32': + sofile = self.check_produced_files({ + 'foo.bar.SO': None, + 'mod_name_in_package': {'mymod.c': None, + 'mymod.o': None}}) + assert os.path.isabs(x) and os.path.samefile(x, sofile) + else: + self.check_produced_files({ + 'foo.bar.SO': None, + 'mod_name_in_package': {'mymod.c': None}, + 'Release': '?'}) + + @chdir_to_tmp + def test_api_compile_explicit_target_2(self): + ffi = cffi.FFI() + ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/") + x = ffi.compile(target=os.path.join("mod_name_in_package", "foo.bar.*")) + if sys.platform != 'win32': + sofile = self.check_produced_files({ + 'mod_name_in_package': {'foo.bar.SO': None, + 'mymod.c': None, + 'mymod.o': None}}) + assert os.path.isabs(x) and os.path.samefile(x, sofile) + else: + self.check_produced_files({ + 'mod_name_in_package': {'foo.bar.SO': None, + 'mymod.c': None}, + 'Release': '?'}) + + @chdir_to_tmp + def test_api_compile_explicit_target_3(self): + ffi = cffi.FFI() + ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/") + x = ffi.compile(target="foo.bar.baz") + if sys.platform != 'win32': + self.check_produced_files({ + 'foo.bar.baz': None, + 'mod_name_in_package': {'mymod.c': None, + 'mymod.o': None}}) + sofile = os.path.join(str(self.udir), 'foo.bar.baz') + assert os.path.isabs(x) and os.path.samefile(x, sofile) + else: + self.check_produced_files({ + 'foo.bar.baz': None, + 'mod_name_in_package': {'mymod.c': None}, + 'Release': '?'}) + + @chdir_to_tmp def test_api_distutils_extension_1(self): ffi = cffi.FFI() ffi.set_source("mod_name_in_package.mymod", "/*code would be here*/") _______________________________________________ pypy-commit mailing list pypy-commit@python.org https://mail.python.org/mailman/listinfo/pypy-commit