This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new af3b747a09f [SPARK-41149][PYTHON] Fix `SparkSession.builder.config` to 
support bool
af3b747a09f is described below

commit af3b747a09f0df85856896c6241f5734b0b1dc54
Author: itholic <haejoon....@databricks.com>
AuthorDate: Thu Nov 17 15:01:33 2022 +0900

    [SPARK-41149][PYTHON] Fix `SparkSession.builder.config` to support bool
    
    ### What changes were proposed in this pull request?
    
    This PR proposes to support `bool` type for `SparkSession.builder.config` 
when building the new `SparkSession`.
    
    ### Why are the changes needed?
    
    Currently, Python `bool` type `True` or `False` is not working correctly, 
since JVM side only treat the String type of "true" or "false" as a valid 
option value.
    
    So, this PR basically fix the current behavior as below:
    
    **Before**
    ```python
    >>> s1 = spark.builder.config("spark.sql.pyspark.jvmStacktrace.enabled", 
True).getOrCreate()
    >>> s1.conf.get("spark.sql.pyspark.jvmStacktrace.enabled")
    'True'  # invalid value, no effect for options.
    ```
    
    **After**
    ```python
    >>> s1 = spark.builder.config("spark.sql.pyspark.jvmStacktrace.enabled", 
True).getOrCreate()
    >>> s1.conf.get("spark.sql.pyspark.jvmStacktrace.enabled")
    'true'  # the lower case "true" only valid for options.
    ```
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Unittest added.
    
    Closes #38673 from itholic/SPARK-41149.
    
    Authored-by: itholic <haejoon....@databricks.com>
    Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
 python/pyspark/sql/session.py            |  6 +++---
 python/pyspark/sql/tests/test_session.py | 21 +++++++++++++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py
index ee04c94cbd5..eec3246cac3 100644
--- a/python/pyspark/sql/session.py
+++ b/python/pyspark/sql/session.py
@@ -60,7 +60,7 @@ from pyspark.sql.types import (
     _parse_datatype_string,
     _from_numpy_type,
 )
-from pyspark.sql.utils import install_exception_handler, 
is_timestamp_ntz_preferred
+from pyspark.sql.utils import install_exception_handler, 
is_timestamp_ntz_preferred, to_str
 
 if TYPE_CHECKING:
     from pyspark.sql._typing import AtomicValue, RowLike, OptionalPrimitiveType
@@ -256,9 +256,9 @@ class SparkSession(SparkConversionMixin):
                         self._options[k] = v
                 elif map is not None:
                     for k, v in map.items():  # type: ignore[assignment]
-                        self._options[k] = str(v)
+                        self._options[k] = to_str(v)
                 else:
-                    self._options[cast(str, key)] = str(value)
+                    self._options[cast(str, key)] = to_str(value)
                 return self
 
         def master(self, master: str) -> "SparkSession.Builder":
diff --git a/python/pyspark/sql/tests/test_session.py 
b/python/pyspark/sql/tests/test_session.py
index 11544bf3131..80c05e1a3cb 100644
--- a/python/pyspark/sql/tests/test_session.py
+++ b/python/pyspark/sql/tests/test_session.py
@@ -327,6 +327,27 @@ class SparkSessionBuilderTests(unittest.TestCase):
             if sc is not None:
                 sc.stop()
 
+    def test_create_spark_context_with_initial_session_options_bool(self):
+        session = None
+        # Test if `True` is set as "true".
+        try:
+            session = SparkSession.builder.config(
+                "spark.sql.pyspark.jvmStacktrace.enabled", True
+            ).getOrCreate()
+            
self.assertEqual(session.conf.get("spark.sql.pyspark.jvmStacktrace.enabled"), 
"true")
+        finally:
+            if session is not None:
+                session.stop()
+        # Test if `False` is set as "false".
+        try:
+            session = SparkSession.builder.config(
+                "spark.sql.pyspark.jvmStacktrace.enabled", False
+            ).getOrCreate()
+            
self.assertEqual(session.conf.get("spark.sql.pyspark.jvmStacktrace.enabled"), 
"false")
+        finally:
+            if session is not None:
+                session.stop()
+
 
 class SparkExtensionsTest(unittest.TestCase):
     # These tests are separate because it uses 'spark.sql.extensions' which is


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

Reply via email to