absurdfarce commented on code in PR #1271:
URL:
https://github.com/apache/cassandra-python-driver/pull/1271#discussion_r2829318427
##########
Jenkinsfile:
##########
@@ -34,7 +34,7 @@ slack = new Slack()
DEFAULT_CASSANDRA = ['3.11', '4.0', '4.1', '5.0']
DEFAULT_DSE = ['dse-5.1.35', 'dse-6.8.30', 'dse-6.9.0']
DEFAULT_HCD = ['hcd-1.0.0']
-DEFAULT_RUNTIME = ['3.9.23', '3.10.18', '3.11.13', '3.12.11', '3.13.5']
+DEFAULT_RUNTIME = ['3.10.18', '3.11.13', '3.12.11', '3.13.5']
Review Comment:
Bumped minimum version to 3.10 since we won't be officially supporting 3.9
with Python driver 3.30.0. We'll need to add 3.14 separately but I need to get
that version installed on the Jenkins runners first.
##########
fix-jenkinsfile-libev.py:
##########
@@ -0,0 +1,9 @@
+import sys
+import toml
+
+pyproject = toml.load(sys.argv[1])
+base = pyproject["tool"]["cassandra-driver"]
+base["libev-includes"] = [sys.argv[2]]
+base["libev-libs"] = [sys.argv[3]]
+
+print(toml.dumps(pyproject))
Review Comment:
Helper script to update pyproject.toml to point at the correct location for
libev includes/libs. Called in Jenkinsfile below.
##########
Jenkinsfile:
##########
@@ -165,15 +165,18 @@ def getMatrixBuilds(buildContext) {
def initializeEnvironment() {
sh label: 'Initialize the environment', script: '''#!/bin/bash -lex
- pyenv global ${PYTHON_VERSION}
- sudo apt-get install socat
- pip install --upgrade pip
- pip install -U setuptools
- # install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023
+ # One of the integration tests relies on socat so let's install that here
+ sudo apt-get install -y socat moreutils
+
+ pyenv shell ${PYTHON_VERSION}
+ python -m venv jenkins-venv
+ . ./jenkins-venv/bin/activate
+ pip install --upgrade pip setuptools wheel
+
+ # Install a version of pyyaml<6.0 compatible with ccm-3.1.5 as of Aug 2023
# this works around the python-3.10+ compatibility problem as described in
DSP-23524
- pip install wheel
- pip install "Cython<3.0" "pyyaml<6.0" --no-build-isolation
Review Comment:
Cython is now handled by the build backend so there's no need to install it
in the venv here
##########
Jenkinsfile:
##########
@@ -244,39 +240,65 @@ ENVIRONMENT_EOF
}
sh label: 'Display Python and environment information', script:
'''#!/bin/bash -le
+ . ./jenkins-venv/bin/activate
+
# Load CCM environment variables
set -o allexport
. ${HOME}/environment.txt
set +o allexport
python --version
pip --version
- pip freeze
printenv | sort
'''
}
-def installDriverAndCompileExtensions() {
- if (env.CYTHON_ENABLED == 'True') {
- sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace
- '''
- } else {
- sh label: 'Install the driver and compile with C extensions without
Cython', script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace --no-cython
- '''
- }
+def installDriver() {
+ sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
+ # Update libev includes and libs to point to the right spot for this
install
+ pyenv shell ${PYTHON_VERSION}
+ python -m venv libev-venv
+ . ./libev-venv/bin/activate
+ pip install toml
+ python fix-jenkinsfile-libev.py ./pyproject.toml "/usr/include"
"/usr/lib/x86_64-linux-gnu" | sponge ./pyproject.toml
+ deactivate
+
+ ls /usr/include/ev.h
+ ls /usr/lib/x86_64-linux-gnu/libev*
+
+ # Now that we've made relevant mods to our local pyproject.toml we're
ready to build the driver
+ . ./jenkins-venv/bin/activate
+
+ # Load CCM environment variables
+ set -o allexport
+ . ${HOME}/environment.txt
+ set +o allexport
+
+ cat ./pyproject.toml
+ pip install --verbose --editable .
Review Comment:
We need a local install of the built driver; without this pytest will
discover (and apparently prefer) the local versions of the corresponding Python
packages so unless the built native libs are also available in the same
directory structure we'll get lots of complaints about libev being unavailable.
##########
pyproject.toml:
##########
@@ -51,6 +51,6 @@ include = ['cassandra', 'cassandra.io',
'cassandra.cqlengine', 'cassandra.graph'
build-murmur3-extension = true
build-libev-extension = true
build-cython-extensions = true
-libev-includes = []
-libev-libs = []
+libev-includes = ["/usr/include/libev", "/usr/local/include",
"/opt/local/include", "/usr/include"]
+libev-libs = ["/usr/local/lib", "/opt/local/lib", "/usr/lib64"]
Review Comment:
We used to have these defaults embedded in setup.py which we would apply
when the appropriate conditions were met. In the spirit of making the build
more explicit it seemed better to just put these values directly in
pyproject.toml as defaults.
##########
Jenkinsfile:
##########
@@ -244,39 +240,65 @@ ENVIRONMENT_EOF
}
sh label: 'Display Python and environment information', script:
'''#!/bin/bash -le
+ . ./jenkins-venv/bin/activate
+
# Load CCM environment variables
set -o allexport
. ${HOME}/environment.txt
set +o allexport
python --version
pip --version
- pip freeze
printenv | sort
'''
}
-def installDriverAndCompileExtensions() {
- if (env.CYTHON_ENABLED == 'True') {
- sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace
- '''
- } else {
- sh label: 'Install the driver and compile with C extensions without
Cython', script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace --no-cython
- '''
- }
+def installDriver() {
+ sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
+ # Update libev includes and libs to point to the right spot for this
install
+ pyenv shell ${PYTHON_VERSION}
+ python -m venv libev-venv
+ . ./libev-venv/bin/activate
+ pip install toml
+ python fix-jenkinsfile-libev.py ./pyproject.toml "/usr/include"
"/usr/lib/x86_64-linux-gnu" | sponge ./pyproject.toml
+ deactivate
+
+ ls /usr/include/ev.h
+ ls /usr/lib/x86_64-linux-gnu/libev*
+
+ # Now that we've made relevant mods to our local pyproject.toml we're
ready to build the driver
+ . ./jenkins-venv/bin/activate
+
+ # Load CCM environment variables
+ set -o allexport
+ . ${HOME}/environment.txt
+ set +o allexport
+
+ cat ./pyproject.toml
+ pip install --verbose --editable .
+
+ # After install display a list of packages in the venv for auditing
+ pip list
+ '''
}
+
def executeStandardTests() {
try {
sh label: 'Execute unit tests', script: '''#!/bin/bash -lex
+ . ./jenkins-venv/bin/activate
+
# Load CCM environment variables
set -o allexport
. ${HOME}/environment.txt
set +o allexport
+ LIBEV_MOD_PATH=`find . -name "libevwrapper.cpython*"`
+ ldd $LIBEV_MOD_PATH
+
+ python -c 'import cassandra.io.libevwrapper'
+
Review Comment:
Some leftover debug code... this can probably be removed.
##########
Jenkinsfile:
##########
@@ -244,39 +240,65 @@ ENVIRONMENT_EOF
}
sh label: 'Display Python and environment information', script:
'''#!/bin/bash -le
+ . ./jenkins-venv/bin/activate
+
# Load CCM environment variables
set -o allexport
. ${HOME}/environment.txt
set +o allexport
python --version
pip --version
- pip freeze
printenv | sort
'''
}
-def installDriverAndCompileExtensions() {
- if (env.CYTHON_ENABLED == 'True') {
- sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace
- '''
- } else {
- sh label: 'Install the driver and compile with C extensions without
Cython', script: '''#!/bin/bash -lex
- python setup.py build_ext --inplace --no-cython
- '''
- }
+def installDriver() {
+ sh label: 'Install the driver and compile with C extensions with Cython',
script: '''#!/bin/bash -lex
+ # Update libev includes and libs to point to the right spot for this
install
+ pyenv shell ${PYTHON_VERSION}
+ python -m venv libev-venv
+ . ./libev-venv/bin/activate
+ pip install toml
+ python fix-jenkinsfile-libev.py ./pyproject.toml "/usr/include"
"/usr/lib/x86_64-linux-gnu" | sponge ./pyproject.toml
Review Comment:
The heart of the matter. These paths match the install location for the
package installs used on the Jenkins runners.
Note that this is done in a separate venv to avoid contaminating the actual
build venv with unrelated packages.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
