This is an automated email from the ASF dual-hosted git repository.
shunping pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 9aed894f5d4 Update pandas version upper bound to support python 3.14
(#39056)
9aed894f5d4 is described below
commit 9aed894f5d454f5121fcbfb8d11d4a2f934a11f7
Author: Shunping Huang <[email protected]>
AuthorDate: Mon Jun 22 16:08:55 2026 -0400
Update pandas version upper bound to support python 3.14 (#39056)
* Update pandas version upper bound to support python 3.14
* Fix the failed tests causing by updating pandas
* Skip the test as the pandas bug is unfixed at 2.3.3
* Refactor get_dummies NaN column lookup to use global static constant
---
sdks/python/apache_beam/dataframe/frames.py | 7 ++++++-
sdks/python/apache_beam/dataframe/frames_test.py | 4 +++-
sdks/python/setup.py | 6 +++---
3 files changed, 12 insertions(+), 5 deletions(-)
diff --git a/sdks/python/apache_beam/dataframe/frames.py
b/sdks/python/apache_beam/dataframe/frames.py
index b839d5504ce..310791d2b58 100644
--- a/sdks/python/apache_beam/dataframe/frames.py
+++ b/sdks/python/apache_beam/dataframe/frames.py
@@ -63,6 +63,10 @@ __all__ = [
# Get major, minor version
PD_VERSION = tuple(map(int, pd.__version__.split('.')[0:2]))
+# Find the name pandas uses for the NaN column in get_dummies().
+# Older pandas versions (<2.3.0) use 'nan', whereas newer versions (>=2.3.0)
use 'NaN'.
+_DUMMY_NAN_COLUMN = 'NaN' if PD_VERSION >= (2, 3) else 'nan'
+
def populate_not_implemented(pd_type):
def wrapper(deferred_type):
@@ -5054,7 +5058,8 @@ class _DeferredStringMethods(frame_base.DeferredBase):
# the data includes NaNs, which is not valid to be casted as a Category,
# but nevertheless would be broadcasted as a column in get_dummies()
columns = sorted(set().union(*split_cats))
- columns = columns + ['nan'] if 'nan' not in columns else columns
+ if _DUMMY_NAN_COLUMN not in columns:
+ columns = columns + [_DUMMY_NAN_COLUMN]
proxy = pd.DataFrame(columns=columns).astype(int)
diff --git a/sdks/python/apache_beam/dataframe/frames_test.py
b/sdks/python/apache_beam/dataframe/frames_test.py
index bb46f0e29be..7a03af6220b 100644
--- a/sdks/python/apache_beam/dataframe/frames_test.py
+++ b/sdks/python/apache_beam/dataframe/frames_test.py
@@ -37,6 +37,8 @@ from apache_beam.runners.interactive.testing.mock_env import
isolated_env
# Get major, minor version
PD_VERSION = tuple(map(int, pd.__version__.split('.')[0:2]))
+# Get major, minor, patch version
+PD_FULL_VERSION = tuple(int(x) for x in re.findall(r'\d+',
pd.__version__)[0:3])
GROUPBY_DF = pd.DataFrame({
'group': ['a' if i % 5 == 0 or i % 3 == 0 else 'b' for i in range(100)],
@@ -1437,7 +1439,7 @@ class DeferredFrameTest(_AbstractFrameTest):
self._run_test(lambda s: s.unstack(level=0), s)
@unittest.skipIf(
- sys.version_info >= (3, 12) and PD_VERSION < (2, 3),
+ sys.version_info >= (3, 12) and PD_FULL_VERSION < (2, 3, 4),
'https://github.com/pandas-dev/pandas/issues/58604')
def test_unstack_pandas_example3(self):
index = self._unstack_get_categorical_index()
diff --git a/sdks/python/setup.py b/sdks/python/setup.py
index c3543995a85..b746fb1d180 100644
--- a/sdks/python/setup.py
+++ b/sdks/python/setup.py
@@ -162,7 +162,7 @@ else:
# https://github.com/pandas-dev/pandas/issues/45725
# must update the below "docs" and "test" for extras_require
dataframe_dependency = [
- 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.3',
+ 'pandas>=1.4.3,!=1.5.0,!=1.5.1,<2.4',
]
milvus_dependency = ['pymilvus>=2.5.10,<3.0.0']
@@ -485,7 +485,7 @@ if __name__ == '__main__':
'docstring-parser>=0.15,<1.0',
'docutils>=0.18.1',
'markdown',
- 'pandas<2.3.0',
+ 'pandas<2.4.0',
'openai',
'virtualenv-clone>=0.5,<1.0',
],
@@ -496,7 +496,7 @@ if __name__ == '__main__':
'jinja2>=3.0,<3.2',
'joblib>=1.0.1',
'mock>=1.0.1,<6.0.0',
- 'pandas<2.3.0',
+ 'pandas<2.4.0',
'parameterized>=0.7.1,<0.10.0',
'pydot>=1.2.0,<2',
'pyhamcrest>=1.9,!=1.10.0,<3.0.0',
