This is an automated email from the ASF dual-hosted git repository.
astitcher pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/qpid-proton.git
The following commit(s) were added to refs/heads/main by this push:
new a2e9eb634 PROTON-2816/PROTON-2817: Support building python extension
with unbundled proton
a2e9eb634 is described below
commit a2e9eb63444bbda70f8870e28d5ef93b68c0bbe5
Author: Andrew Stitcher <[email protected]>
AuthorDate: Fri Jan 12 16:03:38 2024 -0500
PROTON-2816/PROTON-2817: Support building python extension with unbundled
proton
There are 3 cases:
* Bundled proton library - this should be the default and is what you
should use with pip etc.
* System proton library - this is what you should use in a distro
package.
* Use the in tree library built earlier in the build process - this is
what you usually want for developer debugging, and should be the
version actually tested by the in tree tests.
This change also add control of the ability to package and test in
isolated virtual python environments. Using these is the default, but
under some circumstances you do not want the build process to construct
a virtual environment by pulling packages from the internet. So we now
have an option to turn this behaviour off and rely on the hosts python
environment.
---
python/CMakeLists.txt | 87 +++++++++++++++++++++--------
python/MANIFEST.in | 1 +
python/{setup.py => ext_build_devtree.py} | 24 +++++++-
python/{setup.py => ext_build_unbundled.py} | 25 ++++++++-
python/setup.py | 6 +-
5 files changed, 113 insertions(+), 30 deletions(-)
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
index d67a96948..4f3ab4fa3 100644
--- a/python/CMakeLists.txt
+++ b/python/CMakeLists.txt
@@ -53,6 +53,8 @@ set(py_dist_files
README.rst
MANIFEST.in
ext_build.py
+ ext_build_devtree.py
+ ext_build_unbundled.py
cproton.h
cproton_ext.c
cproton.py
@@ -128,7 +130,6 @@ add_custom_command(OUTPUT .timestamp.copied_pysrc
COMMAND ${CMAKE_COMMAND} -E copy
${PN_C_SOURCE_DIR}/core/frame_generators.h src/core
COMMAND ${CMAKE_COMMAND} -E copy
${PN_C_SOURCE_DIR}/core/frame_consumers.c src/core
COMMAND ${CMAKE_COMMAND} -E copy
${PN_C_SOURCE_DIR}/core/frame_consumers.h src/core
- COMMAND ${CMAKE_COMMAND} -E copy
${PROJECT_SOURCE_DIR}/VERSION.txt .
COMMAND ${CMAKE_COMMAND} -E touch .timestamp.copied_pysrc
DEPENDS generated_c_files ${py_cgen} ${py_csrc} ${py_cinc}
${PROJECT_SOURCE_DIR}/VERSION.txt)
@@ -139,28 +140,55 @@ foreach(file IN LISTS py_dist_files pysrc)
list(APPEND pysrc_files "${CMAKE_CURRENT_BINARY_DIR}/${file}")
endforeach()
-add_custom_target(pysrc_copied ALL DEPENDS .timestamp.copied_pysrc
${pysrc_files})
+add_custom_command(OUTPUT VERSION.txt
+ COMMAND ${CMAKE_COMMAND} -E copy
${PROJECT_SOURCE_DIR}/VERSION.txt .
+ DEPENDS ${PROJECT_SOURCE_DIR}/VERSION.txt)
+
+add_custom_target(pysrc_copied DEPENDS ${pysrc_files} VERSION.txt)
+add_custom_target(pypkg_src_copied ALL DEPENDS pysrc_copied
.timestamp.copied_pysrc)
add_custom_command(OUTPUT ./tox.ini
COMMAND ${CMAKE_COMMAND} -E copy
"${CMAKE_CURRENT_SOURCE_DIR}/tox.ini" tox.ini
DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tox.ini")
+option(ENABLE_PYTHON_ISOLATED "Enable python building/testing with isolated
environments." ON)
+option(BUILD_PYTHON_UNBUNDLED_PKG "Build Python package without bundling
qpid-proton-core library" Off)
+
# Make python source and binary packages if we have prerequisites
+check_python_module("build" BUILD_MODULE_FOUND)
+
+if (ENABLE_PYTHON_ISOLATED)
+ set (pypkgbuildoption "")
+else ()
+ set (pypkgbuildoption "-n")
+endif ()
+
check_python_module("setuptools" SETUPTOOLS_MODULE_FOUND)
check_python_module("wheel" WHEEL_MODULE_FOUND)
-check_python_module("build" BUILD_MODULE_FOUND)
check_python_module("cffi" CFFI_MODULE_FOUND)
-if (BUILD_MODULE_FOUND AND SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND AND
CFFI_MODULE_FOUND)
- add_custom_command(OUTPUT .timestamp.dist
- DEPENDS pysrc_copied
- COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
- COMMAND ${Python_EXECUTABLE} -m build -n
- COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+
+if (BUILD_MODULE_FOUND AND
+ (ENABLE_PYTHON_ISOLATED OR (SETUPTOOLS_MODULE_FOUND AND WHEEL_MODULE_FOUND
AND CFFI_MODULE_FOUND)))
+ if (BUILD_PYTHON_UNBUNDLED_PKG)
+ add_custom_command(OUTPUT .timestamp.dist
+ DEPENDS pypkg_src_copied
+ COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+ COMMAND ${CMAKE_COMMAND} -E env
+ QPID_PYTHON_UNBUNDLING=unbundled
+ ${Python_EXECUTABLE} -m build ${pypkgbuildoption}
+ COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+ else ()
+ add_custom_command(OUTPUT .timestamp.dist
+ DEPENDS pypkg_src_copied
+ COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.dist
+ COMMAND ${Python_EXECUTABLE} -m build ${pypkgbuildoption}
+ COMMAND ${CMAKE_COMMAND} -E touch .timestamp.dist)
+ endif ()
add_custom_target(pydist ALL DEPENDS .timestamp.dist)
endif ()
if (BUILD_TESTING)
- add_custom_target(pytest_cffi ALL DEPENDS .timestamp.cproton_ffi)
+ add_custom_target(pytest_cffi ALL DEPENDS .timestamp.test_env)
# python test: python/tests/proton-test
set (py_src "${CMAKE_CURRENT_SOURCE_DIR}")
@@ -179,8 +207,7 @@ if (BUILD_TESTING)
set (python_coverage_options -m coverage run --parallel-mode)
endif(CMAKE_BUILD_TYPE MATCHES "Coverage")
- option(ENABLE_PYTHON_TEST_VENV "Enable python testing within a separate
virtual environment." ON)
- if (ENABLE_PYTHON_TEST_VENV)
+ if (ENABLE_PYTHON_ISOLATED)
# Create Python virtual environment to run tests
set(pytest_venv "${py_bin}/pytest_env")
# Have to use a conditional here as you can't use generator expressions in
OUTPUT or BYPRODUCTS
@@ -193,27 +220,41 @@ if (BUILD_TESTING)
set(pytest_executable "${pytest_bin}/python${CMAKE_EXECUTABLE_SUFFIX}")
add_custom_command(
- OUTPUT ${pytest_venv}/env.txt
+ OUTPUT .timestamp.test_env
COMMAND ${Python_EXECUTABLE} -m venv ${pytest_venv}
COMMAND ${pytest_executable} -m pip install --disable-pip-version-check
cffi setuptools
+ COMMAND ${CMAKE_COMMAND} -E env
+ "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
+ "QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
+ "QPID_PYTHON_UNBUNDLING=devtree"
+ ${pytest_executable} -m pip install -e .
COMMAND ${pytest_executable} -m pip freeze > ${pytest_venv}/env.txt
+ COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
BYPRODUCTS ${pytest_executable}
)
-
else()
set(pytest_executable "${Python_EXECUTABLE}")
set(pytest_venv "${py_bin}")
- FILE(TOUCH "${pytest_venv}/env.txt")
+ add_custom_command(
+ OUTPUT .timestamp.test_env
+ COMMAND ${CMAKE_COMMAND} -E remove -f ${pytest_venv}/env.txt
+ COMMAND ${CMAKE_COMMAND} -E env
+ "CMAKE_BINARY_DIR=${CMAKE_BINARY_DIR}"
+ "QPID_PROTON_CORE_TARGET_DIR=$<TARGET_FILE_DIR:qpid-proton-core>"
+ ${pytest_executable} ext_build_devtree.py
+ COMMAND ${CMAKE_COMMAND} -E touch .timestamp.test_env
+ DEPENDS pysrc_copied qpid-proton-core
+ )
endif()
- # Create c code for cffi extension
- add_custom_command(
- OUTPUT .timestamp.cproton_ffi
- COMMAND ${CMAKE_COMMAND} -E remove -f .timestamp.cproton_ffi
- COMMAND ${pytest_executable} ext_build.py
- COMMAND ${CMAKE_COMMAND} -E touch .timestamp.cproton_ffi
- DEPENDS ${pytest_venv}/env.txt pysrc_copied
- )
+ # If we are on windows copy the qpid-proton-core dll to the test directory
so we can find it easily
+ if (WIN32)
+ add_custom_command(
+ OUTPUT .timestamp.test_env
+ APPEND
+ COMMAND ${CMAKE_COMMAND} -E copy "$<TARGET_FILE:qpid-proton-core>" .
+ )
+ endif()
pn_add_test(
INTERPRETED
diff --git a/python/MANIFEST.in b/python/MANIFEST.in
index 605fc85c5..2ca528cca 100644
--- a/python/MANIFEST.in
+++ b/python/MANIFEST.in
@@ -18,6 +18,7 @@
include VERSION.txt
include ext_build.py
+include ext_build_unbundled.py
include cproton.h
include cproton_ext.c
include cproton.py
diff --git a/python/setup.py b/python/ext_build_devtree.py
similarity index 56%
copy from python/setup.py
copy to python/ext_build_devtree.py
index 51976ace9..8a0a66be2 100644
--- a/python/setup.py
+++ b/python/ext_build_devtree.py
@@ -15,9 +15,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+
+import os
-from setuptools import setup
+from cffi import FFI
-setup(
- cffi_modules='ext_build.py:ffibuilder'
+bld_tree_top = os.environ.get("CMAKE_BINARY_DIR")
+bld_clibdir = os.environ.get("QPID_PROTON_CORE_TARGET_DIR")
+cdefs = open('cproton.h').read()
+c_code = open('cproton_ext.c').read()
+extra_link_args = [f"-Wl,-rpath,{bld_clibdir}"] if os.name == 'posix' else None
+ffibuilder = FFI()
+ffibuilder.cdef(cdefs)
+ffibuilder.set_source(
+ "cproton_ffi",
+ c_code,
+ include_dirs=[f"{bld_tree_top}/c/include"],
+ library_dirs=[f"{bld_clibdir}"],
+ libraries=["qpid-proton-core"],
+ extra_link_args=extra_link_args,
)
+
+if __name__ == "__main__":
+ ffibuilder.compile(verbose=True)
diff --git a/python/setup.py b/python/ext_build_unbundled.py
similarity index 59%
copy from python/setup.py
copy to python/ext_build_unbundled.py
index 51976ace9..d1ea27d57 100644
--- a/python/setup.py
+++ b/python/ext_build_unbundled.py
@@ -15,9 +15,28 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+
+import cffi.pkgconfig
+
+from cffi import FFI
-from setuptools import setup
+ffibuilder = FFI()
-setup(
- cffi_modules='ext_build.py:ffibuilder'
+# cdef() expects a single string declaring the C types, functions and
+# globals needed to use the shared object. It must be in valid C syntax
+# with cffi extensions
+cdefs = open('cproton.h').read()
+ffibuilder.cdef(cdefs)
+
+cffi.pkgconfig.flags_from_pkgconfig(['libqpid-proton-core'])
+
+c_code = open('cproton_ext.c').read()
+ffibuilder.set_source_pkgconfig(
+ "cproton_ffi",
+ ['libqpid-proton-core'],
+ c_code,
)
+
+if __name__ == "__main__":
+ ffibuilder.compile(verbose=True)
diff --git a/python/setup.py b/python/setup.py
index 51976ace9..76d4db5df 100644
--- a/python/setup.py
+++ b/python/setup.py
@@ -16,8 +16,12 @@
# specific language governing permissions and limitations
# under the License.
+import os
from setuptools import setup
+unbundling = os.environ.get("QPID_PYTHON_UNBUNDLING")
+unbundling = f'_{unbundling}' if unbundling else ''
+
setup(
- cffi_modules='ext_build.py:ffibuilder'
+ cffi_modules=f'ext_build{unbundling}.py:ffibuilder'
)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
