This is an automated email from the ASF dual-hosted git repository.
paleolimbot pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow-nanoarrow.git
The following commit(s) were added to refs/heads/main by this push:
new 4a6292dd ci(python): Add Python to release verification with
TEST_WITH_MEMCHECK=1 (#446)
4a6292dd is described below
commit 4a6292ddab6de0e177db09e5f0cfc0f4274515f2
Author: Dewey Dunnington <[email protected]>
AuthorDate: Tue Apr 30 13:53:50 2024 -0300
ci(python): Add Python to release verification with TEST_WITH_MEMCHECK=1
(#446)
This can be run locally with:
```shell
export NANOARROW_PLATFORM=ubuntu
# On M1 you might need:
# export NANOARROW_ARCH=arm64
docker compose run --rm -e TEST_WITH_MEMCHECK=1 -e TEST_DEFAULT=0 -e
TEST_PYTHON=1 -e VERBOSE=1 verify
```
---
.github/workflows/build-and-test-ipc.yaml | 2 +-
.github/workflows/build-and-test.yaml | 2 +-
.github/workflows/meson-build.yaml | 2 +-
.github/workflows/verify.yaml | 13 ++++---------
ci/scripts/build-arrow-cpp-minimal.sh | 3 ++-
dev/release/verify-release-candidate.sh | 24 ++++++++++++++++++++++++
python/src/nanoarrow/iterator.py | 3 ++-
python/tests/test_iterator.py | 3 +++
valgrind.supp | 29 +++++++++++++++++++++++++++++
9 files changed, 67 insertions(+), 14 deletions(-)
diff --git a/.github/workflows/build-and-test-ipc.yaml
b/.github/workflows/build-and-test-ipc.yaml
index 9f823a1a..c8865f6b 100644
--- a/.github/workflows/build-and-test-ipc.yaml
+++ b/.github/workflows/build-and-test-ipc.yaml
@@ -73,7 +73,7 @@ jobs:
with:
path: arrow
# Bump the number at the end of this line to force a new Arrow C++
build
- key: arrow-${{ runner.os }}-10
+ key: arrow-${{ runner.os }}-${{ runner.arch }}-1
- name: Build Arrow C++
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
diff --git a/.github/workflows/build-and-test.yaml
b/.github/workflows/build-and-test.yaml
index 0191b97c..143a53ca 100644
--- a/.github/workflows/build-and-test.yaml
+++ b/.github/workflows/build-and-test.yaml
@@ -63,7 +63,7 @@ jobs:
with:
path: arrow
# Bump the number at the end of this line to force a new Arrow C++
build
- key: arrow-${{ runner.os }}-10
+ key: arrow-${{ runner.os }}-${{ runner.arch }}-1
- name: Build Arrow C++
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
diff --git a/.github/workflows/meson-build.yaml
b/.github/workflows/meson-build.yaml
index 5f06b707..926c18ea 100644
--- a/.github/workflows/meson-build.yaml
+++ b/.github/workflows/meson-build.yaml
@@ -53,7 +53,7 @@ jobs:
with:
path: arrow
# Bump the number at the end of this line to force a new Arrow C++
build
- key: arrow-meson-build-10
+ key: arrow-${{ runner.os }}-${{ runner.arch }}-1
- name: Build Arrow C++
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
diff --git a/.github/workflows/verify.yaml b/.github/workflows/verify.yaml
index b0943905..bac4a48a 100644
--- a/.github/workflows/verify.yaml
+++ b/.github/workflows/verify.yaml
@@ -78,22 +78,17 @@ jobs:
- name: Cache Arrow C++ Build
id: cache-arrow-build
- uses: actions/cache@v3
+ uses: actions/cache@v4
with:
path: arrow
- key: arrow-${{ runner.os }}-6
+ # Bump the number at the end of this line to force a new Arrow C++
build
+ key: arrow-${{ runner.os }}-${{ runner.arch }}-1
- name: Build Arrow C++
if: steps.cache-arrow-build.outputs.cache-hit != 'true'
shell: bash
run: |
- curl
https://dlcdn.apache.org/arrow/arrow-14.0.2/apache-arrow-14.0.2.tar.gz | \
- tar -zxf -
- mkdir arrow-build && cd arrow-build
- cmake ../apache-arrow-14.0.2/cpp -DCMAKE_INSTALL_PREFIX=../arrow
- cmake --build .
- cmake --install . --prefix=../arrow ${{
matrix.config.extra_cmake_install }}
- cd ..
+ src/ci/scripts/build-arrow-cpp-minimal.sh 15.0.2 arrow
- name: Set CMake options (Windows)
if: matrix.config.label == 'windows'
diff --git a/ci/scripts/build-arrow-cpp-minimal.sh
b/ci/scripts/build-arrow-cpp-minimal.sh
index 89ae2ae5..ff0238be 100755
--- a/ci/scripts/build-arrow-cpp-minimal.sh
+++ b/ci/scripts/build-arrow-cpp-minimal.sh
@@ -49,12 +49,13 @@ curl -L
"https://www.apache.org/dyn/closer.lua?action=download&filename=arrow/ar
tar -zxf -
mkdir build && cd build
cmake ../apache-arrow-${ARROW_CPP_VERSION}/cpp \
+ -DCMAKE_BUILD_TYPE=Debug \
-DARROW_JEMALLOC=OFF \
-DARROW_SIMD_LEVEL=NONE \
-DARROW_FILESYSTEM=OFF \
-DCMAKE_INSTALL_PREFIX="${ARROW_CPP_INSTALL_DIR}"
cmake --build . --parallel $(nproc)
-cmake --install . --prefix="${ARROW_CPP_INSTALL_DIR}"
+cmake --install . --prefix="${ARROW_CPP_INSTALL_DIR}" --config=Debug
popd
diff --git a/dev/release/verify-release-candidate.sh
b/dev/release/verify-release-candidate.sh
index 61baa219..2c4c8187 100755
--- a/dev/release/verify-release-candidate.sh
+++ b/dev/release/verify-release-candidate.sh
@@ -354,6 +354,30 @@ test_python() {
show_info "Testing wheel"
python -m pytest -vv
+ if [ ${TEST_WITH_MEMCHECK} -gt 0 ]; then
+ show_info "Setup environment for Python/valgrind"
+
+ # To have more control over exactly which dependencies are available,
+ # set up a separate venv to run. Essentially, we check for leaks in
+ # all self-contained tests but not tests involving pyarrow or numpy.
+ VALGRIND_PYTHON_VENV="${NANOARROW_TMPDIR}/python/valgrind_venv"
+ python -m venv "${VALGRIND_PYTHON_VENV}"
+ source "${VALGRIND_PYTHON_VENV}/bin/activate"
+ python -m pip install --upgrade pip
+ python -m pip install pytest
+
+ # Ensure we have debug symbols for the build we pass to valgrind
+ export NANOARROW_DEBUG_EXTENSION=1
+ pip install -e .
+
+ show_info "Run Python tests with valgrind"
+ valgrind --tool=memcheck --leak-check=full --error-exitcode=1 \
+ --suppressions="${NANOARROW_SOURCE_DIR}/valgrind.supp" \
+ python -m pytest -vv
+
+ deactivate
+ fi
+
popd
}
diff --git a/python/src/nanoarrow/iterator.py b/python/src/nanoarrow/iterator.py
index 08ec67f9..6a535094 100644
--- a/python/src/nanoarrow/iterator.py
+++ b/python/src/nanoarrow/iterator.py
@@ -528,7 +528,8 @@ def _get_tzinfo(tz_string, strategy=None):
pass
raise RuntimeError(
- "zoneinfo (Python 3.9+) or dateutil is required to resolve timezone"
+ "zoneinfo (Python 3.9+, with tzdata on Windows) or "
+ "dateutil is required to resolve timezone"
)
diff --git a/python/tests/test_iterator.py b/python/tests/test_iterator.py
index d43813fd..826f9be2 100644
--- a/python/tests/test_iterator.py
+++ b/python/tests/test_iterator.py
@@ -17,6 +17,7 @@
import datetime
import decimal
+import os
import pytest
from nanoarrow.iterator import (
@@ -461,6 +462,8 @@ def test_get_tzinfo():
pytest.importorskip("zoneinfo")
pytest.importorskip("dateutil")
+ if os.name == "nt":
+ pytest.importorskip("tzdata")
tz_zoneinfo = _get_tzinfo("America/Halifax", strategy=["zoneinfo"])
tz_dateutil = _get_tzinfo("America/Halifax", strategy=["dateutil"])
diff --git a/valgrind.supp b/valgrind.supp
index 510778b4..c26db940 100644
--- a/valgrind.supp
+++ b/valgrind.supp
@@ -28,3 +28,32 @@
...
fun:__tls_get_addr
}
+
+{
+ <Python>:Handle PyMalloc confusing valgrind (possibly leaked)
+ Memcheck:Leak
+ fun:malloc
+ ...
+ fun:_PyObject_GC_Resize
+}
+
+{
+ <Python>:Handle PyMalloc confusing valgrind (possibly leaked)
+ Memcheck:Leak
+ fun:realloc
+ fun:_PyObject_GC_Resize
+}
+
+{
+ <Python>:Handle PyMalloc confusing valgrind (possibly leaked)
+ Memcheck:Leak
+ fun:malloc
+ fun:_PyObject_GC_New
+}
+
+{
+ <Python>:Handle PyMalloc confusing valgrind (possibly leaked)
+ Memcheck:Leak
+ fun:malloc
+ fun:_PyObject_GC_NewVar
+}
