This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 15ef7bead203 [SPARK-57986][PS] Align Rolling/Expanding.sem with pandas
3
15ef7bead203 is described below
commit 15ef7bead203823add98bc649fa33541d4c7cbb5
Author: Fangchen Li <[email protected]>
AuthorDate: Wed Jul 8 09:01:45 2026 +0900
[SPARK-57986][PS] Align Rolling/Expanding.sem with pandas 3
### What changes were proposed in this pull request?
Align sem behavior with pandas 3.
### Why are the changes needed?
Align sem behavior with pandas 3.
### Does this PR introduce _any_ user-facing change?
Yes, sem behavior will align with pandas 3 when it's installed.
### How was this patch tested?
Tested locally with existing unittest.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Opus 4.8 (1M context)
Closes #57067 from fangchenli/pandas3-fix-window-sem.
Authored-by: Fangchen Li <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit 9b9e9729738217abb6f3621879a4469aae8438e2)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/pandas/tests/window/test_expanding_adv.py | 7 +++++++
python/pyspark/pandas/tests/window/test_rolling_adv.py | 12 ++++++++++++
python/pyspark/pandas/window.py | 10 +++++++++-
3 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/python/pyspark/pandas/tests/window/test_expanding_adv.py
b/python/pyspark/pandas/tests/window/test_expanding_adv.py
index dcb3b09f3d3e..086e53a003e5 100644
--- a/python/pyspark/pandas/tests/window/test_expanding_adv.py
+++ b/python/pyspark/pandas/tests/window/test_expanding_adv.py
@@ -17,6 +17,7 @@
import pandas as pd
+from pyspark import pandas as ps
from pyspark.loose_version import LooseVersion
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.pandas.tests.window.test_expanding import
ExpandingTestingFuncMixin
@@ -38,6 +39,12 @@ class ExpandingAdvMixin(ExpandingTestingFuncMixin):
def test_expanding_sem(self):
self._test_expanding_func("sem", int_almost=True)
self._test_expanding_func(lambda x: x.sem(ddof=0), lambda x:
x.sem(ddof=0), int_almost=True)
+ # A single-element window with ddof=0: the population std of one
element is 0, so
+ # pandas 3 returns 0.0 (not null); pandas < 3 returns nan. Both are
matched here.
+ # Guards against a var_samp-based sem, which is null at count == 1.
+ pser = pd.Series([5.0, 6.0, 7.0])
+ psser = ps.from_pandas(pser)
+ self.assert_eq(psser.expanding(1).sem(ddof=0),
pser.expanding(1).sem(ddof=0))
def test_expanding_skew(self):
self._test_expanding_func("skew", int_almost=True)
diff --git a/python/pyspark/pandas/tests/window/test_rolling_adv.py
b/python/pyspark/pandas/tests/window/test_rolling_adv.py
index 69c9515e6a1d..5f5907ed20cc 100644
--- a/python/pyspark/pandas/tests/window/test_rolling_adv.py
+++ b/python/pyspark/pandas/tests/window/test_rolling_adv.py
@@ -15,6 +15,9 @@
# limitations under the License.
#
+import pandas as pd
+
+from pyspark import pandas as ps
from pyspark.testing.pandasutils import PandasOnSparkTestCase
from pyspark.pandas.tests.window.test_rolling import RollingTestingFuncMixin
@@ -35,6 +38,15 @@ class RollingAdvMixin(RollingTestingFuncMixin):
def test_rolling_sem(self):
self._test_rolling_func("sem")
self._test_rolling_func(lambda x: x.sem(ddof=0), lambda x:
x.sem(ddof=0))
+ # A single-element window with ddof=0: the population std of one
element is 0, so
+ # pandas 3 returns 0.0 (not null); pandas < 3 returns nan. Both are
matched here.
+ # Guards against a var_samp-based sem, which is null at count == 1.
+ pser = pd.Series([5.0, 6.0, 7.0])
+ psser = ps.from_pandas(pser)
+ self.assert_eq(
+ psser.rolling(1, min_periods=1).sem(ddof=0),
+ pser.rolling(1, min_periods=1).sem(ddof=0),
+ )
def test_rolling_skew(self):
self._test_rolling_func("skew")
diff --git a/python/pyspark/pandas/window.py b/python/pyspark/pandas/window.py
index 486b76f2a04e..20ebbddd163b 100644
--- a/python/pyspark/pandas/window.py
+++ b/python/pyspark/pandas/window.py
@@ -19,7 +19,9 @@ from functools import partial
from typing import Any, Callable, Generic, List, Optional
import numpy as np
+import pandas as pd
+from pyspark.loose_version import LooseVersion
from pyspark.sql import Window
from pyspark.sql import functions as F
from pyspark.sql.internal import InternalFunction as SF
@@ -137,9 +139,15 @@ class RollingAndExpanding(Generic[FrameLike],
metaclass=ABCMeta):
def sem(scol: Column) -> Column:
count = F.count(scol).over(self._window)
+ if LooseVersion(pd.__version__) >= "3.0.0":
+ # pandas 3.0 fixed sem to std(ddof) / sqrt(count)
(pandas-dev/pandas#63181);
+ # earlier pandas divided by sqrt(count - ddof) instead.
+ sem_scol = SF.stddev(scol, ddof).over(self._window) /
F.sqrt(count)
+ else:
+ sem_scol = F.stddev(scol).over(self._window) / F.sqrt(count -
ddof)
return F.when(
(F.row_number().over(self._unbounded_window) >=
self._min_periods) & (count > ddof),
- F.stddev(scol).over(self._window) / F.sqrt(count - ddof),
+ sem_scol,
).otherwise(F.lit(None))
return self._apply_as_series_or_frame(sem)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
