Spenserrrr commented on code in PR #57435: URL: https://github.com/apache/spark/pull/57435#discussion_r3635794565
########## python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_non_default.py: ########## @@ -0,0 +1,276 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Tests for PyArrow Array.to_pandas(coerce_temporal_nanoseconds=True) using golden +file comparison. + +PySpark relies on this argument in production: when building pandas objects from +Arrow data (see ``python/pyspark/sql/pandas/conversion.py``), it calls +``to_pandas(coerce_temporal_nanoseconds=True)`` so that the Arrow path produces the +same nanosecond-resolution ``datetime64[ns]`` / ``timedelta64[ns]`` values as the +non-Arrow path. This test records how each Arrow temporal type behaves under that +argument so CI fails loudly if the behavior drifts across pandas/PyArrow/NumPy +upgrades. + +``coerce_temporal_nanoseconds=True`` forces temporal values to nanosecond +resolution. The interesting rows are therefore the temporal types (timestamp and +duration in units s/ms/us/ns, plus tz-aware timestamp, date, and time); a handful +of non-temporal control rows (int/float/string) are included to demonstrate that +the argument leaves non-temporal types unaffected. + +## Golden File Cell Format + +Each cell uses the value@type format: +- numpy ndarray: "python_list_repr@ndarray[dtype]" +- pandas Series: "python_list_repr@Series[dtype]" +- Error: "ERR@ExceptionClassName" + +Values are formatted via tolist() for stable, Python-native representation. + +## Regenerating Golden Files + +Set SPARK_GENERATE_GOLDEN_FILES=1 before running: + + SPARK_GENERATE_GOLDEN_FILES=1 python -m pytest \\ + python/pyspark/tests/upstream/pyarrow/test_pyarrow_arrow_to_pandas_coerce_temporal.py + +## PyArrow and pandas Version Compatibility + +The golden files capture behavior for specific PyArrow and pandas versions. +Regenerate when upgrading either dependency, as to_pandas() behavior may change. +The committed golden files were generated with pandas 2.3.3, pyarrow 24.0.0, and +numpy 2.4.1. +""" + +import datetime +import inspect +import os +import unittest +from typing import Callable, List, Optional + +from pyspark.loose_version import LooseVersion +from pyspark.testing.utils import ( + have_pyarrow, + have_pandas, + have_numpy, + pyarrow_requirement_message, + pandas_requirement_message, + numpy_requirement_message, +) +from pyspark.testing.goldenutils import GoldenFileTestMixin + +if have_pandas: + import pandas as pd +if have_pyarrow: + import pyarrow as pa + + [email protected]( + not have_pyarrow or not have_pandas or not have_numpy, + pyarrow_requirement_message or pandas_requirement_message or numpy_requirement_message, +) +class PyArrowArrayToPandasCoerceTemporalTests(GoldenFileTestMixin, unittest.TestCase): + """ + Tests pa.Array.to_pandas(coerce_temporal_nanoseconds=True) via golden file comparison. + + Covers the temporal Arrow types the argument affects (timestamp and duration in + units s/ms/us/ns, tz-aware timestamp, date, and time), plus an overflow case and + a few non-temporal control rows. Each type is tested without nulls, with a null, + and empty. + """ + + def compare_or_generate_golden_matrix( Review Comment: I've moved compare_or_generate_golden_matrix into a shared base class in this file for now. Fully centralizing it into GoldenFileTestMixin also means updating test_pyarrow_arrow_to_pandas_default.py and test_pyarrow_array_cast.py (which have the same helper), so I'd propose doing that as a separate follow-up PR to keep this one scoped to the coerce_temporal tests. Please let me know if you have other ideas. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
