This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new af3b747a09f [SPARK-41149][PYTHON] Fix `SparkSession.builder.config` to support bool af3b747a09f is described below commit af3b747a09f0df85856896c6241f5734b0b1dc54 Author: itholic <haejoon....@databricks.com> AuthorDate: Thu Nov 17 15:01:33 2022 +0900 [SPARK-41149][PYTHON] Fix `SparkSession.builder.config` to support bool ### What changes were proposed in this pull request? This PR proposes to support `bool` type for `SparkSession.builder.config` when building the new `SparkSession`. ### Why are the changes needed? Currently, Python `bool` type `True` or `False` is not working correctly, since JVM side only treat the String type of "true" or "false" as a valid option value. So, this PR basically fix the current behavior as below: **Before** ```python >>> s1 = spark.builder.config("spark.sql.pyspark.jvmStacktrace.enabled", True).getOrCreate() >>> s1.conf.get("spark.sql.pyspark.jvmStacktrace.enabled") 'True' # invalid value, no effect for options. ``` **After** ```python >>> s1 = spark.builder.config("spark.sql.pyspark.jvmStacktrace.enabled", True).getOrCreate() >>> s1.conf.get("spark.sql.pyspark.jvmStacktrace.enabled") 'true' # the lower case "true" only valid for options. ``` ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Unittest added. Closes #38673 from itholic/SPARK-41149. Authored-by: itholic <haejoon....@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- python/pyspark/sql/session.py | 6 +++--- python/pyspark/sql/tests/test_session.py | 21 +++++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index ee04c94cbd5..eec3246cac3 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -60,7 +60,7 @@ from pyspark.sql.types import ( _parse_datatype_string, _from_numpy_type, ) -from pyspark.sql.utils import install_exception_handler, is_timestamp_ntz_preferred +from pyspark.sql.utils import install_exception_handler, is_timestamp_ntz_preferred, to_str if TYPE_CHECKING: from pyspark.sql._typing import AtomicValue, RowLike, OptionalPrimitiveType @@ -256,9 +256,9 @@ class SparkSession(SparkConversionMixin): self._options[k] = v elif map is not None: for k, v in map.items(): # type: ignore[assignment] - self._options[k] = str(v) + self._options[k] = to_str(v) else: - self._options[cast(str, key)] = str(value) + self._options[cast(str, key)] = to_str(value) return self def master(self, master: str) -> "SparkSession.Builder": diff --git a/python/pyspark/sql/tests/test_session.py b/python/pyspark/sql/tests/test_session.py index 11544bf3131..80c05e1a3cb 100644 --- a/python/pyspark/sql/tests/test_session.py +++ b/python/pyspark/sql/tests/test_session.py @@ -327,6 +327,27 @@ class SparkSessionBuilderTests(unittest.TestCase): if sc is not None: sc.stop() + def test_create_spark_context_with_initial_session_options_bool(self): + session = None + # Test if `True` is set as "true". + try: + session = SparkSession.builder.config( + "spark.sql.pyspark.jvmStacktrace.enabled", True + ).getOrCreate() + self.assertEqual(session.conf.get("spark.sql.pyspark.jvmStacktrace.enabled"), "true") + finally: + if session is not None: + session.stop() + # Test if `False` is set as "false". + try: + session = SparkSession.builder.config( + "spark.sql.pyspark.jvmStacktrace.enabled", False + ).getOrCreate() + self.assertEqual(session.conf.get("spark.sql.pyspark.jvmStacktrace.enabled"), "false") + finally: + if session is not None: + session.stop() + class SparkExtensionsTest(unittest.TestCase): # These tests are separate because it uses 'spark.sql.extensions' which is --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org