https://github.com/python/cpython/commit/1e4e59bb3714ba7c6b6297f1a74e231b056f004c commit: 1e4e59bb3714ba7c6b6297f1a74e231b056f004c branch: main author: Itamar Oren <[email protected]> committer: encukou <[email protected]> date: 2025-11-14T10:43:25+01:00 summary:
gh-116146: Add C-API to create module from spec and initfunc (GH-139196) Co-authored-by: Kumar Aditya <[email protected]> Co-authored-by: Petr Viktorin <[email protected]> Co-authored-by: Victor Stinner <[email protected]> files: A Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst M Doc/c-api/import.rst M Doc/whatsnew/3.15.rst M Include/cpython/import.h M Lib/test/test_embed.py M Programs/_testembed.c M Python/import.c diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst index 8eabc0406b11ce..24e673d3d1394f 100644 --- a/Doc/c-api/import.rst +++ b/Doc/c-api/import.rst @@ -333,3 +333,24 @@ Importing Modules strings instead of Python :class:`str` objects. .. versionadded:: 3.14 + +.. c:function:: PyObject* PyImport_CreateModuleFromInitfunc(PyObject *spec, PyObject* (*initfunc)(void)) + + This function is a building block that enables embedders to implement + the :py:meth:`~importlib.abc.Loader.create_module` step of custom + static extension importers (e.g. of statically-linked extensions). + + *spec* must be a :class:`~importlib.machinery.ModuleSpec` object. + + *initfunc* must be an :ref:`initialization function <extension-export-hook>`, + the same as for :c:func:`PyImport_AppendInittab`. + + On success, create and return a module object. + This module will not be initialized; call :c:func:`!PyModule_Exec` + to initialize it. + (Custom importers should do this in their + :py:meth:`~importlib.abc.Loader.exec_module` method.) + + On error, return NULL with an exception set. + + .. versionadded:: next diff --git a/Doc/whatsnew/3.15.rst b/Doc/whatsnew/3.15.rst index 31594a2e70bd4c..9393b65ed8e906 100644 --- a/Doc/whatsnew/3.15.rst +++ b/Doc/whatsnew/3.15.rst @@ -1080,6 +1080,10 @@ New features thread state. (Contributed by Victor Stinner in :gh:`139653`.) +* Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating + a module from a *spec* and *initfunc*. + (Contributed by Itamar Oren in :gh:`116146`.) + Changed C APIs -------------- diff --git a/Include/cpython/import.h b/Include/cpython/import.h index 0ce0b1ee6cce2a..149a20af8b9cbb 100644 --- a/Include/cpython/import.h +++ b/Include/cpython/import.h @@ -10,6 +10,13 @@ struct _inittab { PyAPI_DATA(struct _inittab *) PyImport_Inittab; PyAPI_FUNC(int) PyImport_ExtendInittab(struct _inittab *newtab); +// Custom importers may use this API to initialize statically linked +// extension modules directly from a spec and init function, +// without needing to go through inittab +PyAPI_FUNC(PyObject *) PyImport_CreateModuleFromInitfunc( + PyObject *spec, + PyObject *(*initfunc)(void)); + struct _frozen { const char *name; /* ASCII encoded string */ const unsigned char *code; diff --git a/Lib/test/test_embed.py b/Lib/test/test_embed.py index 1933f691a78be5..1078796eae84e2 100644 --- a/Lib/test/test_embed.py +++ b/Lib/test/test_embed.py @@ -239,6 +239,31 @@ def test_repeated_init_and_inittab(self): lines = "\n".join(lines) + "\n" self.assertEqual(out, lines) + def test_create_module_from_initfunc(self): + out, err = self.run_embedded_interpreter("test_create_module_from_initfunc") + if support.Py_GIL_DISABLED: + # the test imports a singlephase init extension, so it emits a warning + # under the free-threaded build + expected_runtime_warning = ( + "RuntimeWarning: The global interpreter lock (GIL)" + " has been enabled to load module 'embedded_ext'" + ) + filtered_err_lines = [ + line + for line in err.strip().splitlines() + if expected_runtime_warning not in line + ] + self.assertEqual(filtered_err_lines, []) + else: + self.assertEqual(err, "") + self.assertEqual(out, + "<module 'my_test_extension' (static-extension)>\n" + "my_test_extension.executed='yes'\n" + "my_test_extension.exec_slot_ran='yes'\n" + "<module 'embedded_ext' (static-extension)>\n" + "embedded_ext.executed='yes'\n" + ) + def test_forced_io_encoding(self): # Checks forced configuration of embedded interpreter IO streams env = dict(os.environ, PYTHONIOENCODING="utf-8:surrogateescape") diff --git a/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst new file mode 100644 index 00000000000000..be8043e26ddda8 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-11-08-10-51-50.gh-issue-116146.pCmx6L.rst @@ -0,0 +1,2 @@ +Add a new :c:func:`PyImport_CreateModuleFromInitfunc` C-API for creating a +module from a *spec* and *initfunc*. Patch by Itamar Oren. diff --git a/Programs/_testembed.c b/Programs/_testembed.c index d3600fecbe2775..27224e508bdd3e 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -166,6 +166,8 @@ static PyModuleDef embedded_ext = { static PyObject* PyInit_embedded_ext(void) { + // keep this as a single-phase initialization module; + // see test_create_module_from_initfunc return PyModule_Create(&embedded_ext); } @@ -1894,8 +1896,16 @@ static int test_initconfig_exit(void) } +int +extension_module_exec(PyObject *mod) +{ + return PyModule_AddStringConstant(mod, "exec_slot_ran", "yes"); +} + + static PyModuleDef_Slot extension_slots[] = { {Py_mod_gil, Py_MOD_GIL_NOT_USED}, + {Py_mod_exec, extension_module_exec}, {0, NULL} }; @@ -2213,6 +2223,106 @@ static int test_repeated_init_and_inittab(void) return 0; } +static PyObject* +create_module(PyObject* self, PyObject* spec) +{ + PyObject *name = PyObject_GetAttrString(spec, "name"); + if (!name) { + return NULL; + } + if (PyUnicode_EqualToUTF8(name, "my_test_extension")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, init_my_test_extension); + } + if (PyUnicode_EqualToUTF8(name, "embedded_ext")) { + Py_DECREF(name); + return PyImport_CreateModuleFromInitfunc(spec, PyInit_embedded_ext); + } + PyErr_Format(PyExc_LookupError, "static module %R not found", name); + Py_DECREF(name); + return NULL; +} + +static PyObject* +exec_module(PyObject* self, PyObject* mod) +{ + if (PyModule_Exec(mod) < 0) { + return NULL; + } + Py_RETURN_NONE; +} + +static PyMethodDef create_static_module_methods[] = { + {"create_module", create_module, METH_O, NULL}, + {"exec_module", exec_module, METH_O, NULL}, + {} +}; + +static struct PyModuleDef create_static_module_def = { + PyModuleDef_HEAD_INIT, + .m_name = "create_static_module", + .m_size = 0, + .m_methods = create_static_module_methods, + .m_slots = extension_slots, +}; + +PyMODINIT_FUNC PyInit_create_static_module(void) { + return PyModuleDef_Init(&create_static_module_def); +} + +static int +test_create_module_from_initfunc(void) +{ + wchar_t* argv[] = { + PROGRAM_NAME, + L"-c", + // Multi-phase initialization + L"import my_test_extension;" + L"print(my_test_extension);" + L"print(f'{my_test_extension.executed=}');" + L"print(f'{my_test_extension.exec_slot_ran=}');" + // Single-phase initialization + L"import embedded_ext;" + L"print(embedded_ext);" + L"print(f'{embedded_ext.executed=}');" + }; + PyConfig config; + if (PyImport_AppendInittab("create_static_module", + &PyInit_create_static_module) != 0) { + fprintf(stderr, "PyImport_AppendInittab() failed\n"); + return 1; + } + PyConfig_InitPythonConfig(&config); + config.isolated = 1; + config_set_argv(&config, Py_ARRAY_LENGTH(argv), argv); + init_from_config_clear(&config); + int result = PyRun_SimpleString( + "import sys\n" + "from importlib.util import spec_from_loader\n" + "import create_static_module\n" + "class StaticExtensionImporter:\n" + " _ORIGIN = \"static-extension\"\n" + " @classmethod\n" + " def find_spec(cls, fullname, path, target=None):\n" + " if fullname in {'my_test_extension', 'embedded_ext'}:\n" + " return spec_from_loader(fullname, cls, origin=cls._ORIGIN)\n" + " return None\n" + " @staticmethod\n" + " def create_module(spec):\n" + " return create_static_module.create_module(spec)\n" + " @staticmethod\n" + " def exec_module(module):\n" + " create_static_module.exec_module(module)\n" + " module.executed = 'yes'\n" + "sys.meta_path.append(StaticExtensionImporter)\n" + ); + if (result < 0) { + fprintf(stderr, "PyRun_SimpleString() failed\n"); + return 1; + } + return Py_RunMain(); +} + static void wrap_allocator(PyMemAllocatorEx *allocator); static void unwrap_allocator(PyMemAllocatorEx *allocator); @@ -2396,6 +2506,7 @@ static struct TestCase TestCases[] = { #endif {"test_get_incomplete_frame", test_get_incomplete_frame}, {"test_gilstate_after_finalization", test_gilstate_after_finalization}, + {"test_create_module_from_initfunc", test_create_module_from_initfunc}, {NULL, NULL} }; diff --git a/Python/import.c b/Python/import.c index b05b40448d02ac..9ab2d3b3552235 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2364,8 +2364,23 @@ is_builtin(PyObject *name) return 0; } +static PyModInitFunction +lookup_inittab_initfunc(const struct _Py_ext_module_loader_info* info) +{ + for (struct _inittab *p = INITTAB; p->name != NULL; p++) { + if (_PyUnicode_EqualToASCIIString(info->name, p->name)) { + return (PyModInitFunction)p->initfunc; + } + } + // not found + return NULL; +} + static PyObject* -create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) +create_builtin( + PyThreadState *tstate, PyObject *name, + PyObject *spec, + PyModInitFunction initfunc) { struct _Py_ext_module_loader_info info; if (_Py_ext_module_loader_info_init_for_builtin(&info, name) < 0) { @@ -2396,25 +2411,15 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) _extensions_cache_delete(info.path, info.name); } - struct _inittab *found = NULL; - for (struct _inittab *p = INITTAB; p->name != NULL; p++) { - if (_PyUnicode_EqualToASCIIString(info.name, p->name)) { - found = p; - break; - } - } - if (found == NULL) { - // not found - mod = Py_NewRef(Py_None); - goto finally; - } - - PyModInitFunction p0 = (PyModInitFunction)found->initfunc; + PyModInitFunction p0 = initfunc; if (p0 == NULL) { - /* Cannot re-init internal module ("sys" or "builtins") */ - assert(is_core_module(tstate->interp, info.name, info.path)); - mod = import_add_module(tstate, info.name); - goto finally; + p0 = lookup_inittab_initfunc(&info); + if (p0 == NULL) { + /* Cannot re-init internal module ("sys" or "builtins") */ + assert(is_core_module(tstate->interp, info.name, info.path)); + mod = import_add_module(tstate, info.name); + goto finally; + } } #ifdef Py_GIL_DISABLED @@ -2440,6 +2445,33 @@ create_builtin(PyThreadState *tstate, PyObject *name, PyObject *spec) return mod; } +PyObject* +PyImport_CreateModuleFromInitfunc( + PyObject *spec, PyObject *(*initfunc)(void)) +{ + if (initfunc == NULL) { + PyErr_BadInternalCall(); + return NULL; + } + + PyThreadState *tstate = _PyThreadState_GET(); + + PyObject *name = PyObject_GetAttr(spec, &_Py_ID(name)); + if (name == NULL) { + return NULL; + } + + if (!PyUnicode_Check(name)) { + PyErr_Format(PyExc_TypeError, + "spec name must be string, not %T", name); + Py_DECREF(name); + return NULL; + } + + PyObject *mod = create_builtin(tstate, name, spec, initfunc); + Py_DECREF(name); + return mod; +} /*****************************/ /* the builtin modules table */ @@ -3209,7 +3241,7 @@ bootstrap_imp(PyThreadState *tstate) } // Create the _imp module from its definition. - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_CLEAR(name); Py_DECREF(spec); if (mod == NULL) { @@ -4369,7 +4401,7 @@ _imp_create_builtin(PyObject *module, PyObject *spec) return NULL; } - PyObject *mod = create_builtin(tstate, name, spec); + PyObject *mod = create_builtin(tstate, name, spec, NULL); Py_DECREF(name); return mod; } _______________________________________________ Python-checkins mailing list -- [email protected] To unsubscribe send an email to [email protected] https://mail.python.org/mailman3//lists/python-checkins.python.org Member address: [email protected]
