This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1096801ead29 [SPARK-47828][CONNECT][PYTHON]
DataFrameWriterV2.overwrite` fails with invalid plan
1096801ead29 is described below
commit 1096801ead29a8b62e915a0348b8adaad060b215
Author: Ruifeng Zheng <ruife...@apache.org>
AuthorDate: Mon Apr 15 12:21:01 2024 +0800
[SPARK-47828][CONNECT][PYTHON] DataFrameWriterV2.overwrite` fails with
invalid plan
### What changes were proposed in this pull request?
`DataFrameWriterV2.overwrite` fails with invalid plan
### Why are the changes needed?
bug fix
### Does this PR introduce _any_ user-facing change?
yes
### How was this patch tested?
added test
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #46023 from zhengruifeng/connect_fix_overwrite.
Authored-by: Ruifeng Zheng <ruife...@apache.org>
Signed-off-by: Ruifeng Zheng <ruife...@apache.org>
---
python/pyspark/sql/connect/plan.py | 8 ++++----
python/pyspark/sql/tests/test_readwriter.py | 5 +++++
2 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/sql/connect/plan.py
b/python/pyspark/sql/connect/plan.py
index 72b8372c8039..239ee23c2061 100644
--- a/python/pyspark/sql/connect/plan.py
+++ b/python/pyspark/sql/connect/plan.py
@@ -1751,16 +1751,16 @@ class WriteOperationV2(LogicalPlan):
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_CREATE
elif wm == "overwrite":
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_OVERWRITE
+ if self.overwrite_condition is not None:
+ plan.write_operation_v2.overwrite_condition.CopyFrom(
+ self.overwrite_condition.to_plan(session)
+ )
elif wm == "overwrite_partitions":
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_OVERWRITE_PARTITIONS
elif wm == "append":
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_APPEND
elif wm == "replace":
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_REPLACE
- if self.overwrite_condition is not None:
- plan.write_operation_v2.overwrite_condition.CopyFrom(
- self.overwrite_condition.to_plan(session)
- )
elif wm == "create_or_replace":
plan.write_operation_v2.mode =
proto.WriteOperationV2.Mode.MODE_CREATE_OR_REPLACE
else:
diff --git a/python/pyspark/sql/tests/test_readwriter.py
b/python/pyspark/sql/tests/test_readwriter.py
index 85057f37a181..5784d2c72973 100644
--- a/python/pyspark/sql/tests/test_readwriter.py
+++ b/python/pyspark/sql/tests/test_readwriter.py
@@ -252,6 +252,11 @@ class ReadwriterV2TestsMixin:
):
df.writeTo("test_table").create()
+ def test_table_overwrite(self):
+ df = self.df
+ with self.assertRaisesRegex(AnalysisException,
"TABLE_OR_VIEW_NOT_FOUND"):
+ df.writeTo("test_table").overwrite(lit(True))
+
class ReadwriterTests(ReadwriterTestsMixin, ReusedSQLTestCase):
pass
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
