This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 3d5ca3eabc9 Update Numpy and Pandas Deps (#40704)
3d5ca3eabc9 is described below
commit 3d5ca3eabc9fcb4d46bd9c6b4d913b7758364d55
Author: Jack McCluskey <[email protected]>
AuthorDate: Fri Oct 23 09:38:27 2026 -0400
Update Numpy and Pandas Deps (#40704)
* Update Numpy and Pandas Deps
* Add context for chosen versions
---
.../beam_PostCommit_Python_Dependency.json | 2 +-
sdks/python/pyproject.toml | 2 +-
sdks/python/setup.py | 15 +++++-----
sdks/python/test-suites/tox/py311/build.gradle | 33 ++++++----------------
sdks/python/tox.ini | 16 +++++------
5 files changed, 26 insertions(+), 42 deletions(-)
diff --git a/.github/trigger_files/beam_PostCommit_Python_Dependency.json
b/.github/trigger_files/beam_PostCommit_Python_Dependency.json
index 0c0e8ca6a12..1aa32de42b2 100644
--- a/.github/trigger_files/beam_PostCommit_Python_Dependency.json
+++ b/.github/trigger_files/beam_PostCommit_Python_Dependency.json
@@ -1,4 +1,4 @@
{
"comment": "Modify this file in a trivial way to cause this test suite to
run",
- "modification": 8
+ "modification": 6
}
diff --git a/sdks/python/pyproject.toml b/sdks/python/pyproject.toml
index 703047cb6ef..9a00fe03ea0 100644
--- a/sdks/python/pyproject.toml
+++ b/sdks/python/pyproject.toml
@@ -31,7 +31,7 @@ requires = [
# Avoid https://github.com/pypa/virtualenv/issues/2006
"distlib==0.4.4",
# Numpy headers
- "numpy>=1.14.3,<2.5.0", # Update setup.py as well.
+ "numpy>=1.23.2,<2.6.0", # Update setup.py as well.
# having cython here will create wheels that are platform dependent.
"cython>=3.2.6,<4",
## deps for generating external transform wrappers:
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index 10bb4b95191..69323197468 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -156,12 +156,11 @@ else:
'pyarrow>=14.0.1,<26.0.0',
]
-# Exclude pandas<=1.4.2 since it doesn't work with numpy 1.24.x.
-# Exclude 1.5.0 and 1.5.1 because of
-# https://github.com/pandas-dev/pandas/issues/45725
# must update the below "docs" and "test" for extras_require
dataframe_dependency = [
- 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.4',
+ # pandas 2.2.2 is the first version that works with numpy 2.0.0+
+ # TODO(https://github.com/apache/beam/issues/40709): support pandas 3+
+ 'pandas>=2.2.2,<2.4',
]
milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']
@@ -435,7 +434,9 @@ if __name__ == '__main__':
'jsonpickle>=3.0.4,<5.0.0',
# numpy can have breaking changes in minor versions.
# Use a strict upper bound.
- 'numpy>=1.14.3,<2.5.0', # Update pyproject.toml as well.
+ # Increment the lower bound when the minimum Python version is
+ # updated.
+ 'numpy>=1.23.2,<2.6.0', # Update pyproject.toml as well.
'objsize>=0.6.1,<0.8.0',
'packaging>=22.0',
'pillow>=12.1.1,<14',
@@ -490,7 +491,7 @@ if __name__ == '__main__':
'docstring-parser>=0.15,<1.0',
'docutils>=0.18.1',
'markdown',
- 'pandas<2.4.0',
+ 'pandas<2.6.0',
'openai',
'virtualenv-clone>=0.5,<1.0',
],
@@ -501,7 +502,7 @@ if __name__ == '__main__':
'jinja2>=3.0,<3.2',
'joblib>=1.0.1',
'mock>=1.0.1,<6.0.0',
- 'pandas<2.4.0',
+ 'pandas<2.6.0',
'parameterized>=0.7.1,<0.10.0',
'pydot>=1.2.0,<2',
'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
diff --git a/sdks/python/test-suites/tox/py311/build.gradle
b/sdks/python/test-suites/tox/py311/build.gradle
index 3228469cd6b..fe1202cc0d3 100644
--- a/sdks/python/test-suites/tox/py311/build.gradle
+++ b/sdks/python/test-suites/tox/py311/build.gradle
@@ -84,30 +84,15 @@ toxTask "testPy311pyarrow-25", "py311-pyarrow-25",
"${posargs}"
test.dependsOn "testPy311pyarrow-25"
postCommitPyDep.dependsOn "testPy311pyarrow-25"
-// Create a test task for each supported minor version of pandas
-toxTask "testPy311pandas-14", "py311-pandas-14", "${posargs}"
-test.dependsOn "testPy311pandas-14"
-postCommitPyDep.dependsOn "testPy311pandas-14"
-
-toxTask "testPy311pandas-15", "py311-pandas-15", "${posargs}"
-test.dependsOn "testPy311pandas-15"
-postCommitPyDep.dependsOn "testPy311pandas-15"
-
-toxTask "testPy311pandas-20", "py311-pandas-20", "${posargs}"
-test.dependsOn "testPy311pandas-20"
-postCommitPyDep.dependsOn "testPy311pandas-20"
-
-// TODO(https://github.com/apache/beam/issues/31192): Add below suites
-// after dependency compat tests suite switches to Python 3.9 or we add
-// Python 2.2 support.
-
-// toxTask "testPy311pandas-21", "py311-pandas-21", "${posargs}"
-// test.dependsOn "testPy311pandas-21"
-// postCommitPyDep.dependsOn "testPy311pandas-21"
-
-// toxTask "testPy311pandas-22", "py311-pandas-22", "${posargs}"
-// test.dependsOn "testPy311pandas-22"
-// postCommitPyDep.dependsOn "testPy311pandas-22"
+// Create a test task for each supported minor version of pandas.
+// Keep in sync with [testenv:py311-pandas-...] in tox.ini.
+toxTask "testPy311pandas-22", "py311-pandas-22", "${posargs}"
+test.dependsOn "testPy311pandas-22"
+postCommitPyDep.dependsOn "testPy311pandas-22"
+
+toxTask "testPy311pandas-23", "py311-pandas-23", "${posargs}"
+test.dependsOn "testPy311pandas-23"
+postCommitPyDep.dependsOn "testPy311pandas-23"
// TODO(https://github.com/apache/beam/issues/30908): Revise what are we
testing
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index b579f91fd2e..4d0b1903b13 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -365,16 +365,14 @@ commands =
/bin/sh -c 'pytest -o junit_suite_name={envname}
--junitxml=pytest_{envname}.xml -n 6 -m uses_pyarrow {posargs}; ret=$?; [ $ret
= 5 ] && exit 0 || exit $ret'
-[testenv:py311-pandas-{14,15,20}]
+[testenv:py311-pandas-{22,23}]
+# Keep in line with dataframe_dependency in setup.py. pandas 2.2.2 is the first
+# release compatible with numpy 2.
deps =
- 14: pandas>=1.4.3,<1.5.0
- 14: numpy>=1.14.3,<1.27.0
- # Exclude 1.5.0 and 1.5.1 because of
https://github.com/pandas-dev/pandas/issues/45725
- 15: pandas>=1.5.2,<1.6.0
- 15: numpy>=1.14.3,<1.27.0
- 20: pandas>=2.0.0,<2.1.0
- 20: pyarrow>=7
- 20: numpy>=1.14.3,<1.27.0
+ # Lower bound of the supported range; also keeps numpy 1.x covered.
+ 22: pandas>=2.2.2,<2.3.0
+ 22: numpy>=1.23.2,<2.0.0
+ 23: pandas>=2.3.0,<2.4.0
commands =
# Log pandas and numpy version for debugging
/bin/sh -c "pip freeze | grep -E '(pandas|numpy)'"
