Author: Ronan Lamy <[email protected]>
Branch:
Changeset: r87478:298b37893194
Date: 2016-09-30 18:53 +0100
http://bitbucket.org/pypy/pypy/changeset/298b37893194/
Log: Merge branch 'test-cpyext'
diff --git a/pypy/module/cpyext/test/conftest.py
b/pypy/module/cpyext/test/conftest.py
--- a/pypy/module/cpyext/test/conftest.py
+++ b/pypy/module/cpyext/test/conftest.py
@@ -2,6 +2,8 @@
import pytest
def pytest_configure(config):
+ if config.option.runappdirect:
+ return
from pypy.tool.pytest.objspace import gettestobjspace
# For some reason (probably a ll2ctypes cache issue on linux64)
# it's necessary to run "import time" at least once before any
diff --git a/pypy/module/cpyext/test/test_cpyext.py
b/pypy/module/cpyext/test/test_cpyext.py
--- a/pypy/module/cpyext/test/test_cpyext.py
+++ b/pypy/module/cpyext/test/test_cpyext.py
@@ -5,7 +5,7 @@
import py, pytest
from pypy import pypydir
-from pypy.interpreter import gateway
+from pypy.interpreter.gateway import unwrap_spec, interp2app
from rpython.rtyper.lltypesystem import lltype, ll2ctypes
from rpython.translator.gensupp import uniquemodulename
from rpython.tool.udir import udir
@@ -18,7 +18,8 @@
from .support import c_compile
-only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
+HERE = py.path.local(pypydir) / 'module' / 'cpyext' / 'test'
+only_pypy ="config.option.runappdirect and '__pypy__' not in
sys.builtin_module_names"
@api.cpython_api([], api.PyObject)
def PyPy_Crash1(space):
@@ -42,19 +43,6 @@
files.append(filename)
return files
-def create_so(modname, include_dirs, source_strings=None, source_files=None,
- compile_extra=None, link_extra=None, libraries=None):
- dirname = (udir/uniquemodulename('module')).ensure(dir=1)
- if source_strings:
- assert not source_files
- files = convert_sources_to_files(source_strings, dirname)
- source_files = files
- soname = c_compile(source_files, outputfilename=str(dirname/modname),
- compile_extra=compile_extra, link_extra=link_extra,
- include_dirs=include_dirs,
- libraries=libraries)
- return soname
-
class SystemCompilationInfo(object):
"""Bundles all the generic information required to compile extensions.
@@ -68,6 +56,88 @@
self.extra_libs = extra_libs
self.ext = ext
+ def compile_extension_module(self, name, include_dirs=None,
+ source_files=None, source_strings=None):
+ """
+ Build an extension module and return the filename of the resulting
+ native code file.
+
+ name is the name of the module, possibly including dots if it is a
+ module inside a package.
+
+ Any extra keyword arguments are passed on to ExternalCompilationInfo to
+ build the module (so specify your source with one of those).
+ """
+ include_dirs = include_dirs or []
+ modname = name.split('.')[-1]
+ dirname = (udir/uniquemodulename('module')).ensure(dir=1)
+ if source_strings:
+ assert not source_files
+ files = convert_sources_to_files(source_strings, dirname)
+ source_files = files
+ soname = c_compile(source_files, outputfilename=str(dirname/modname),
+ compile_extra=self.compile_extra,
+ link_extra=self.link_extra,
+ include_dirs=self.include_extra + include_dirs,
+ libraries=self.extra_libs)
+ pydname = soname.new(purebasename=modname, ext=self.ext)
+ soname.rename(pydname)
+ return str(pydname)
+
+ def import_module(self, name, init=None, body='', filename=None,
+ include_dirs=None, PY_SSIZE_T_CLEAN=False):
+ """
+ init specifies the overall template of the module.
+
+ if init is None, the module source will be loaded from a file in this
+ test directory, give a name given by the filename parameter.
+
+ if filename is None, the module name will be used to construct the
+ filename.
+ """
+ if init is not None:
+ code = make_source(name, init, body, PY_SSIZE_T_CLEAN)
+ kwds = dict(source_strings=[code])
+ else:
+ assert not PY_SSIZE_T_CLEAN
+ if filename is None:
+ filename = name
+ filename = HERE / (filename + ".c")
+ kwds = dict(source_files=[filename])
+ mod = self.compile_extension_module(
+ name, include_dirs=include_dirs, **kwds)
+ return self.load_module(mod, name)
+
+ def import_extension(self, modname, functions, prologue="",
+ include_dirs=None, more_init="", PY_SSIZE_T_CLEAN=False):
+ body = prologue + make_methods(functions, modname)
+ init = """Py_InitModule("%s", methods);""" % (modname,)
+ if more_init:
+ init += more_init
+ return self.import_module(
+ name=modname, init=init, body=body, include_dirs=include_dirs,
+ PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
+
+
+class ExtensionCompiler(SystemCompilationInfo):
+ """Extension compiler for appdirect mode"""
+ def load_module(space, mod, name):
+ import imp
+ return imp.load_dynamic(name, mod)
+
+class SpaceCompiler(SystemCompilationInfo):
+ """Extension compiler for regular (untranslated PyPy) mode"""
+ def __init__(self, space, *args, **kwargs):
+ self.space = space
+ SystemCompilationInfo.__init__(self, *args, **kwargs)
+
+ def load_module(self, mod, name):
+ space = self.space
+ api.load_extension_module(space, mod, name)
+ return space.getitem(
+ space.sys.get('modules'), space.wrap(name))
+
+
def get_cpyext_info(space):
from pypy.module.imp.importing import get_so_extension
state = space.fromcache(State)
@@ -88,7 +158,7 @@
link_extra = ["-g"]
else:
compile_extra = link_extra = None
- return SystemCompilationInfo(
+ return SpaceCompiler(space,
include_extra=api.include_dirs,
compile_extra=compile_extra,
link_extra=link_extra,
@@ -96,30 +166,6 @@
ext=get_so_extension(space))
-def compile_extension_module(sys_info, modname, include_dirs=[],
- source_files=None, source_strings=None):
- """
- Build an extension module and return the filename of the resulting native
- code file.
-
- modname is the name of the module, possibly including dots if it is a
module
- inside a package.
-
- Any extra keyword arguments are passed on to ExternalCompilationInfo to
- build the module (so specify your source with one of those).
- """
- modname = modname.split('.')[-1]
- soname = create_so(modname,
- include_dirs=sys_info.include_extra + include_dirs,
- source_files=source_files,
- source_strings=source_strings,
- compile_extra=sys_info.compile_extra,
- link_extra=sys_info.link_extra,
- libraries=sys_info.extra_libs)
- pydname = soname.new(purebasename=modname, ext=sys_info.ext)
- soname.rename(pydname)
- return str(pydname)
-
def get_so_suffix():
from imp import get_suffixes, C_EXTENSION
for suffix, mode, typ in get_suffixes():
@@ -142,12 +188,60 @@
"-O0", "-g", "-Werror=implicit-function-declaration", "-fPIC"]
link_extra = None
ext = get_so_suffix()
- return SystemCompilationInfo(
+ return ExtensionCompiler(
include_extra=[get_python_inc()],
compile_extra=compile_extra,
link_extra=link_extra,
ext=get_so_suffix())
+def make_methods(functions, modname):
+ methods_table = []
+ codes = []
+ for funcname, flags, code in functions:
+ cfuncname = "%s_%s" % (modname, funcname)
+ methods_table.append(
+ "{\"%s\", %s, %s}," % (funcname, cfuncname, flags))
+ func_code = """
+ static PyObject* %s(PyObject* self, PyObject* args)
+ {
+ %s
+ }
+ """ % (cfuncname, code)
+ codes.append(func_code)
+
+ body = "\n".join(codes) + """
+ static PyMethodDef methods[] = {
+ %s
+ { NULL }
+ };
+ """ % ('\n'.join(methods_table),)
+ return body
+
+def make_source(name, init, body, PY_SSIZE_T_CLEAN):
+ code = """
+ %(PY_SSIZE_T_CLEAN)s
+ #include <Python.h>
+ /* fix for cpython 2.7 Python.h if running tests with -A
+ since pypy compiles with -fvisibility-hidden */
+ #undef PyMODINIT_FUNC
+ #ifdef __GNUC__
+ # define RPY_EXPORTED extern __attribute__((visibility("default")))
+ #else
+ # define RPY_EXPORTED extern __declspec(dllexport)
+ #endif
+ #define PyMODINIT_FUNC RPY_EXPORTED void
+
+ %(body)s
+
+ PyMODINIT_FUNC
+ init%(name)s(void) {
+ %(init)s
+ }
+ """ % dict(name=name, init=init, body=body,
+ PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
+ if PY_SSIZE_T_CLEAN else '')
+ return code
+
def freeze_refcnts(self):
rawrefcount._dont_free_any_more()
@@ -159,22 +253,6 @@
#state.print_refcounts()
self.frozen_ll2callocations = set(ll2ctypes.ALLOCATED.values())
-class FakeSpace(object):
- """Like TinyObjSpace, but different"""
- def __init__(self, config):
- self.config = config
-
- def passthrough(self, arg):
- return arg
- listview = passthrough
- str_w = passthrough
-
- def unwrap(self, args):
- try:
- return args.str_w(None)
- except:
- return args
-
class LeakCheckingTest(object):
"""Base class for all cpyext tests."""
spaceconfig = dict(usemodules=['cpyext', 'thread', '_rawffi', 'array',
@@ -265,9 +343,12 @@
cls.w_libc = cls.space.wrap(get_libc_name())
def setup_method(self, meth):
- freeze_refcnts(self)
+ if not self.runappdirect:
+ freeze_refcnts(self)
def teardown_method(self, meth):
+ if self.runappdirect:
+ return
self.cleanup_references(self.space)
# XXX: like AppTestCpythonExtensionBase.teardown_method:
# find out how to disable check_and_print_leaks() if the
@@ -293,21 +374,42 @@
skip("Windows Python >= 2.6 only")
assert isinstance(sys.dllhandle, int)
+
+def _unwrap_include_dirs(space, w_include_dirs):
+ if w_include_dirs is None:
+ return None
+ else:
+ return [space.str_w(s) for s in space.listview(w_include_dirs)]
+
+def debug_collect(space):
+ rawrefcount._collect()
+
class AppTestCpythonExtensionBase(LeakCheckingTest):
def setup_class(cls):
space = cls.space
- space.getbuiltinmodule("cpyext")
- # 'import os' to warm up reference counts
- w_import = space.builtin.getdictvalue(space, '__import__')
- space.call_function(w_import, space.wrap("os"))
- #state = cls.space.fromcache(RefcountState) ZZZ
- #state.non_heaptypes_w[:] = []
+ cls.w_here = space.wrap(str(HERE))
if not cls.runappdirect:
cls.w_runappdirect = space.wrap(cls.runappdirect)
+ space.getbuiltinmodule("cpyext")
+ # 'import os' to warm up reference counts
+ w_import = space.builtin.getdictvalue(space, '__import__')
+ space.call_function(w_import, space.wrap("os"))
+ #state = cls.space.fromcache(RefcountState) ZZZ
+ #state.non_heaptypes_w[:] = []
+ cls.w_debug_collect = space.wrap(interp2app(debug_collect))
+
+ def record_imported_module(self, name):
+ """
+ Record a module imported in a test so that it can be cleaned up in
+ teardown before the check for leaks is done.
+
+ name gives the name of the module in the space's sys.modules.
+ """
+ self.imported_module_names.append(name)
def setup_method(self, func):
- @gateway.unwrap_spec(name=str)
+ @unwrap_spec(name=str)
def compile_module(space, name,
w_source_files=None,
w_source_strings=None):
@@ -322,172 +424,68 @@
source_strings = space.listview_bytes(w_source_strings)
else:
source_strings = None
- pydname = compile_extension_module(
- self.sys_info, name,
+ pydname = self.sys_info.compile_extension_module(
+ name,
source_files=source_files,
source_strings=source_strings)
+
+ # hackish, but tests calling compile_module() always end up
+ # importing the result
+ self.record_imported_module(name)
+
return space.wrap(pydname)
- @gateway.unwrap_spec(name=str, init='str_or_None', body=str,
- load_it=bool, filename='str_or_None',
- PY_SSIZE_T_CLEAN=bool)
- def import_module(space, name, init=None, body='', load_it=True,
+ @unwrap_spec(name=str, init='str_or_None', body=str,
+ filename='str_or_None', PY_SSIZE_T_CLEAN=bool)
+ def import_module(space, name, init=None, body='',
filename=None, w_include_dirs=None,
PY_SSIZE_T_CLEAN=False):
- """
- init specifies the overall template of the module.
+ include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+ w_result = self.sys_info.import_module(
+ name, init, body, filename, include_dirs, PY_SSIZE_T_CLEAN)
+ self.record_imported_module(name)
+ return w_result
- if init is None, the module source will be loaded from a file in
this
- test direcory, give a name given by the filename parameter.
- if filename is None, the module name will be used to construct the
- filename.
- """
- if w_include_dirs is None:
- include_dirs = []
- else:
- include_dirs = [space.str_w(s) for s in
space.listview(w_include_dirs)]
- if init is not None:
- code = """
- %(PY_SSIZE_T_CLEAN)s
- #include <Python.h>
- /* fix for cpython 2.7 Python.h if running tests with -A
- since pypy compiles with -fvisibility-hidden */
- #undef PyMODINIT_FUNC
- #ifdef __GNUC__
- # define RPY_EXPORTED extern
__attribute__((visibility("default")))
- #else
- # define RPY_EXPORTED extern __declspec(dllexport)
- #endif
- #define PyMODINIT_FUNC RPY_EXPORTED void
+ @unwrap_spec(mod=str, name=str)
+ def load_module(space, mod, name):
+ return self.sys_info.load_module(mod, name)
- %(body)s
-
- PyMODINIT_FUNC
- init%(name)s(void) {
- %(init)s
- }
- """ % dict(name=name, init=init, body=body,
- PY_SSIZE_T_CLEAN='#define PY_SSIZE_T_CLEAN'
- if PY_SSIZE_T_CLEAN else '')
- kwds = dict(source_strings=[code])
- else:
- assert not PY_SSIZE_T_CLEAN
- if filename is None:
- filename = name
- filename = py.path.local(pypydir) / 'module' \
- / 'cpyext'/ 'test' / (filename + ".c")
- kwds = dict(source_files=[filename])
- mod = compile_extension_module(self.sys_info, name,
- include_dirs=include_dirs, **kwds)
-
- if load_it:
- if self.runappdirect:
- import imp
- return imp.load_dynamic(name, mod)
- else:
- api.load_extension_module(space, mod, name)
- self.imported_module_names.append(name)
- return space.getitem(
- space.sys.get('modules'),
- space.wrap(name))
- else:
- path = os.path.dirname(mod)
- if self.runappdirect:
- return path
- else:
- return space.wrap(path)
-
- @gateway.unwrap_spec(mod=str, name=str)
- def reimport_module(space, mod, name):
- if self.runappdirect:
- import imp
- return imp.load_dynamic(name, mod)
- else:
- api.load_extension_module(space, mod, name)
- return space.getitem(
- space.sys.get('modules'),
- space.wrap(name))
-
- @gateway.unwrap_spec(modname=str, prologue=str,
+ @unwrap_spec(modname=str, prologue=str,
more_init=str, PY_SSIZE_T_CLEAN=bool)
def import_extension(space, modname, w_functions, prologue="",
w_include_dirs=None, more_init="",
PY_SSIZE_T_CLEAN=False):
functions = space.unwrap(w_functions)
- methods_table = []
- codes = []
- for funcname, flags, code in functions:
- cfuncname = "%s_%s" % (modname, funcname)
- methods_table.append("{\"%s\", %s, %s}," %
- (funcname, cfuncname, flags))
- func_code = """
- static PyObject* %s(PyObject* self, PyObject* args)
- {
- %s
- }
- """ % (cfuncname, code)
- codes.append(func_code)
-
- body = prologue + "\n".join(codes) + """
- static PyMethodDef methods[] = {
- %s
- { NULL }
- };
- """ % ('\n'.join(methods_table),)
- init = """Py_InitModule("%s", methods);""" % (modname,)
- if more_init:
- init += more_init
- return import_module(space, name=modname, init=init, body=body,
- w_include_dirs=w_include_dirs,
- PY_SSIZE_T_CLEAN=PY_SSIZE_T_CLEAN)
-
- @gateway.unwrap_spec(name=str)
- def record_imported_module(name):
- """
- Record a module imported in a test so that it can be cleaned up in
- teardown before the check for leaks is done.
-
- name gives the name of the module in the space's sys.modules.
- """
- self.imported_module_names.append(name)
-
- def debug_collect(space):
- rawrefcount._collect()
+ include_dirs = _unwrap_include_dirs(space, w_include_dirs)
+ w_result = self.sys_info.import_extension(
+ modname, functions, prologue, include_dirs, more_init,
+ PY_SSIZE_T_CLEAN)
+ self.record_imported_module(modname)
+ return w_result
# A list of modules which the test caused to be imported (in
# self.space). These will be cleaned up automatically in teardown.
self.imported_module_names = []
if self.runappdirect:
- fake = FakeSpace(self.space.config)
- def interp2app(func):
- def run(*args, **kwargs):
- for k in kwargs.keys():
- if k not in func.unwrap_spec and not
k.startswith('w_'):
- v = kwargs.pop(k)
- kwargs['w_' + k] = v
- return func(fake, *args, **kwargs)
- return run
- def wrap(func):
- return func
self.sys_info = get_sys_info_app()
+ self.compile_module = self.sys_info.compile_extension_module
+ self.load_module = self.sys_info.load_module
+ self.import_module = self.sys_info.import_module
+ self.import_extension = self.sys_info.import_extension
else:
- interp2app = gateway.interp2app
wrap = self.space.wrap
self.sys_info = get_cpyext_info(self.space)
- self.w_compile_module = wrap(interp2app(compile_module))
- self.w_import_module = wrap(interp2app(import_module))
- self.w_reimport_module = wrap(interp2app(reimport_module))
- self.w_import_extension = wrap(interp2app(import_extension))
- self.w_record_imported_module =
wrap(interp2app(record_imported_module))
- self.w_here = wrap(str(py.path.local(pypydir)) +
'/module/cpyext/test/')
- self.w_debug_collect = wrap(interp2app(debug_collect))
+ self.w_compile_module = wrap(interp2app(compile_module))
+ self.w_load_module = wrap(interp2app(load_module))
+ self.w_import_module = wrap(interp2app(import_module))
+ self.w_import_extension = wrap(interp2app(import_extension))
- # create the file lock before we count allocations
- self.space.call_method(self.space.sys.get("stdout"), "flush")
+ # create the file lock before we count allocations
+ self.space.call_method(self.space.sys.get("stdout"), "flush")
- freeze_refcnts(self)
- #self.check_and_print_leaks()
+ freeze_refcnts(self)
+ #self.check_and_print_leaks()
def unimport_module(self, name):
"""
@@ -498,6 +496,8 @@
self.space.delitem(w_modules, w_name)
def teardown_method(self, func):
+ if self.runappdirect:
+ return
for name in self.imported_module_names:
self.unimport_module(name)
self.cleanup_references(self.space)
@@ -632,19 +632,15 @@
If `cherry.date` is an extension module which imports `apple.banana`,
the latter is added to `sys.modules` for the `"apple.banana"` key.
"""
- if self.runappdirect:
- skip('record_imported_module not supported in runappdirect mode')
+ import sys, types, os
# Build the extensions.
banana = self.compile_module(
- "apple.banana", source_files=[self.here + 'banana.c'])
- self.record_imported_module("apple.banana")
+ "apple.banana", source_files=[os.path.join(self.here, 'banana.c')])
date = self.compile_module(
- "cherry.date", source_files=[self.here + 'date.c'])
- self.record_imported_module("cherry.date")
+ "cherry.date", source_files=[os.path.join(self.here, 'date.c')])
# Set up some package state so that the extensions can actually be
# imported.
- import sys, types, os
cherry = sys.modules['cherry'] = types.ModuleType('cherry')
cherry.__path__ = [os.path.dirname(date)]
@@ -652,7 +648,6 @@
apple.__path__ = [os.path.dirname(banana)]
import cherry.date
- import apple.banana
assert sys.modules['apple.banana'].__name__ == 'apple.banana'
assert sys.modules['cherry.date'].__name__ == 'cherry.date'
@@ -989,7 +984,7 @@
f.write('not again!\n')
f.close()
m1 = sys.modules['foo']
- m2 = self.reimport_module(m1.__file__, name='foo')
+ m2 = self.load_module(m1.__file__, name='foo')
assert m1 is m2
assert m1 is sys.modules['foo']
diff --git a/pypy/module/cpyext/test/test_import.py
b/pypy/module/cpyext/test/test_import.py
--- a/pypy/module/cpyext/test/test_import.py
+++ b/pypy/module/cpyext/test/test_import.py
@@ -1,6 +1,6 @@
from pypy.module.cpyext.test.test_api import BaseApiTest
from pypy.module.cpyext.test.test_cpyext import AppTestCpythonExtensionBase
-from rpython.rtyper.lltypesystem import rffi, lltype
+from rpython.rtyper.lltypesystem import rffi
class TestImport(BaseApiTest):
def test_import(self, space, api):
@@ -39,9 +39,9 @@
class AppTestImportLogic(AppTestCpythonExtensionBase):
def test_import_logic(self):
- path = self.import_module(name='test_import_module', load_it=False)
- import sys
- sys.path.append(path)
+ import sys, os
+ path = self.compile_module('test_import_module',
+ source_files=[os.path.join(self.here, 'test_import_module.c')])
+ sys.path.append(os.path.dirname(path))
import test_import_module
assert test_import_module.TEST is None
-
_______________________________________________
pypy-commit mailing list
[email protected]
https://mail.python.org/mailman/listinfo/pypy-commit
