This is an automated email from the ASF dual-hosted git repository.

gurwls223 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new b804b53e4a5 [SPARK-41659][CONNECT] Enable doctests in 
pyspark.sql.connect.readwriter
b804b53e4a5 is described below

commit b804b53e4a5b971bf2db676f4d552fff383ab586
Author: Sandeep Singh <sand...@techaddict.me>
AuthorDate: Tue Jan 3 09:41:57 2023 +0900

    [SPARK-41659][CONNECT] Enable doctests in pyspark.sql.connect.readwriter
    
    ### What changes were proposed in this pull request?
    This PR proposes to enable doctests in pyspark.sql.connect.readwriter that 
is virtually the same as pyspark.sql.readwriter.
    
    ### Why are the changes needed?
    To make sure on the PySpark compatibility and test coverage.
    
    ### Does this PR introduce any user-facing change?
    No, doctest's only.
    
    ### How was this patch tested?
    New Doctests Added
    
    Closes #39331 from techaddict/SPARK-41659-pyspark.sql.connect.readwriter.
    
    Authored-by: Sandeep Singh <sand...@techaddict.me>
    Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
 dev/sparktestsupport/modules.py          |  1 +
 python/pyspark/sql/connect/plan.py       |  2 +-
 python/pyspark/sql/connect/readwriter.py | 60 ++++++++++++++++++++++++++++++++
 python/pyspark/sql/readwriter.py         | 18 +++++-----
 4 files changed, 71 insertions(+), 10 deletions(-)

diff --git a/dev/sparktestsupport/modules.py b/dev/sparktestsupport/modules.py
index 99f1cc6894f..0eeb3dd9218 100644
--- a/dev/sparktestsupport/modules.py
+++ b/dev/sparktestsupport/modules.py
@@ -509,6 +509,7 @@ pyspark_connect = Module(
         "pyspark.sql.connect.session",
         "pyspark.sql.connect.window",
         "pyspark.sql.connect.column",
+        "pyspark.sql.connect.readwriter",
         # unittests
         "pyspark.sql.tests.connect.test_connect_plan",
         "pyspark.sql.tests.connect.test_connect_basic",
diff --git a/python/pyspark/sql/connect/plan.py 
b/python/pyspark/sql/connect/plan.py
index f10687cc82e..48a8fa598e7 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -1264,7 +1264,7 @@ class WriteOperation(LogicalPlan):
 
         for k in self.options:
             if self.options[k] is None:
-                del plan.write_operation.options[k]
+                plan.write_operation.options.pop(k, None)
             else:
                 plan.write_operation.options[k] = cast(str, self.options[k])
 
diff --git a/python/pyspark/sql/connect/readwriter.py 
b/python/pyspark/sql/connect/readwriter.py
index c3af39f864c..207ad8df74f 100644
--- a/python/pyspark/sql/connect/readwriter.py
+++ b/python/pyspark/sql/connect/readwriter.py
@@ -20,6 +20,7 @@ from typing import Dict
 from typing import Optional, Union, List, overload, Tuple, cast, Any
 from typing import TYPE_CHECKING
 
+from pyspark import SparkContext, SparkConf
 from pyspark.sql.connect.plan import Read, DataSource, LogicalPlan, 
WriteOperation
 from pyspark.sql.types import StructType
 from pyspark.sql.utils import to_str
@@ -458,3 +459,62 @@ class DataFrameWriter(OptionUtils):
 
     def jdbc(self, *args: Any, **kwargs: Any) -> None:
         raise NotImplementedError("jdbc() not supported for DataFrameWriter")
+
+
+def _test() -> None:
+    import os
+    import sys
+    import doctest
+    from pyspark.sql import SparkSession as PySparkSession
+    from pyspark.testing.connectutils import should_test_connect, 
connect_requirement_message
+
+    os.chdir(os.environ["SPARK_HOME"])
+
+    if should_test_connect:
+        import pyspark.sql.connect.readwriter
+
+        globs = pyspark.sql.connect.readwriter.__dict__.copy()
+        # Works around to create a regular Spark session
+        sc = SparkContext("local[4]", "sql.connect.readwriter tests", 
conf=SparkConf())
+        globs["_spark"] = PySparkSession(
+            sc, options={"spark.app.name": "sql.connect.readwriter tests"}
+        )
+
+        # TODO(SPARK-41817): Support reading with schema
+        del pyspark.sql.connect.readwriter.DataFrameReader.load.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameReader.option.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.csv.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.option.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.text.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.bucketBy.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.sortBy.__doc__
+
+        # TODO(SPARK-41818): Support saveAsTable
+        del pyspark.sql.connect.readwriter.DataFrameWriter.insertInto.__doc__
+        del pyspark.sql.connect.readwriter.DataFrameWriter.saveAsTable.__doc__
+
+        # Creates a remote Spark session.
+        os.environ["SPARK_REMOTE"] = "sc://localhost"
+        globs["spark"] = 
PySparkSession.builder.remote("sc://localhost").getOrCreate()
+
+        (failure_count, test_count) = doctest.testmod(
+            pyspark.sql.connect.readwriter,
+            globs=globs,
+            optionflags=doctest.ELLIPSIS
+            | doctest.NORMALIZE_WHITESPACE
+            | doctest.IGNORE_EXCEPTION_DETAIL,
+        )
+
+        globs["spark"].stop()
+        globs["_spark"].stop()
+        if failure_count:
+            sys.exit(-1)
+    else:
+        print(
+            f"Skipping pyspark.sql.connect.readwriter doctests: 
{connect_requirement_message}",
+            file=sys.stderr,
+        )
+
+
+if __name__ == "__main__":
+    _test()
diff --git a/python/pyspark/sql/readwriter.py b/python/pyspark/sql/readwriter.py
index cc50d7cb0a5..8ecb0cd6dcd 100644
--- a/python/pyspark/sql/readwriter.py
+++ b/python/pyspark/sql/readwriter.py
@@ -90,7 +90,7 @@ class DataFrameReader(OptionUtils):
         Examples
         --------
         >>> spark.read.format('json')
-        <pyspark.sql.readwriter.DataFrameReader object ...>
+        <...readwriter.DataFrameReader object ...>
 
         Write a DataFrame into a JSON file and read it back.
 
@@ -133,7 +133,7 @@ class DataFrameReader(OptionUtils):
         Examples
         --------
         >>> spark.read.schema("col0 INT, col1 DOUBLE")
-        <pyspark.sql.readwriter.DataFrameReader object ...>
+        <...readwriter.DataFrameReader object ...>
 
         Specify the schema with reading a CSV file.
 
@@ -177,7 +177,7 @@ class DataFrameReader(OptionUtils):
         >>> from pyspark.sql import SparkSession
         >>> spark = SparkSession.builder.master("local").getOrCreate()
         >>> spark.read.option("key", "value")
-        <pyspark.sql.readwriter.DataFrameReader object ...>
+        <...readwriter.DataFrameReader object ...>
 
         Specify the option 'nullValue' with reading a CSV file.
 
@@ -216,7 +216,7 @@ class DataFrameReader(OptionUtils):
         Examples
         --------
         >>> spark.read.option("key", "value")
-        <pyspark.sql.readwriter.DataFrameReader object ...>
+        <...readwriter.DataFrameReader object ...>
 
         Specify the option 'nullValue' and 'header' with reading a CSV file.
 
@@ -962,10 +962,10 @@ class DataFrameWriter(OptionUtils):
         >>> with tempfile.TemporaryDirectory() as d:
         ...     spark.createDataFrame(
         ...         [{"age": 80, "name": "Xinrong Meng"}]
-        ...     ).write.mode("error").format("parquet").save(d)
+        ...     ).write.mode("error").format("parquet").save(d) # doctest: 
+SKIP
         Traceback (most recent call last):
             ...
-        pyspark.sql.utils.AnalysisException: ...
+        ...AnalysisException: ...
 
         Write a Parquet file back with various options, and read it back.
 
@@ -1016,7 +1016,7 @@ class DataFrameWriter(OptionUtils):
         Examples
         --------
         >>> spark.range(1).write.format('parquet')
-        <pyspark.sql.readwriter.DataFrameWriter object ...>
+        <...readwriter.DataFrameWriter object ...>
 
         Write a DataFrame into a Parquet file and read it back.
 
@@ -1057,7 +1057,7 @@ class DataFrameWriter(OptionUtils):
         Examples
         --------
         >>> spark.range(1).write.option("key", "value")
-        <pyspark.sql.readwriter.DataFrameWriter object ...>
+        <...readwriter.DataFrameWriter object ...>
 
         Specify the option 'nullValue' with writing a CSV file.
 
@@ -1096,7 +1096,7 @@ class DataFrameWriter(OptionUtils):
         Examples
         --------
         >>> spark.range(1).write.option("key", "value")
-        <pyspark.sql.readwriter.DataFrameWriter object ...>
+        <...readwriter.DataFrameWriter object ...>
 
         Specify the option 'nullValue' and 'header' with writing a CSV file.
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

Reply via email to