This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 03dfd4747ae [SPARK-44738][PYTHON][CONNECT] Add missing client metadata
to calls
03dfd4747ae is described below
commit 03dfd4747ae3b59e7e3ac348c30e82b63e1ed269
Author: Martin Grund <martin.gr...@databricks.com>
AuthorDate: Wed Aug 9 18:08:23 2023 +0900
[SPARK-44738][PYTHON][CONNECT] Add missing client metadata to calls
### What changes were proposed in this pull request?
The refactoring for the re-attachable execution missed properly propagating
the client metadata for the individual RPC calls.
### Why are the changes needed?
Compatibility.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
Existing UT
Closes #42409 from grundprinzip/SPARK-44738.
Authored-by: Martin Grund <martin.gr...@databricks.com>
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
(cherry picked from commit c73660c3e7279f61fe6e2f6bbf88f410f7ce25a1)
Signed-off-by: Hyukjin Kwon <gurwls...@apache.org>
---
python/pyspark/sql/connect/client/reattach.py | 11 +++++++----
1 file changed, 7 insertions(+), 4 deletions(-)
diff --git a/python/pyspark/sql/connect/client/reattach.py
b/python/pyspark/sql/connect/client/reattach.py
index c5c45904c9b..c6b1beaa121 100644
--- a/python/pyspark/sql/connect/client/reattach.py
+++ b/python/pyspark/sql/connect/client/reattach.py
@@ -135,7 +135,9 @@ class ExecutePlanResponseReattachableIterator(Generator):
if not attempt.is_first_try():
# on retry, the iterator is borked, so we need a
new one
self._iterator = iter(
-
self._stub.ReattachExecute(self._create_reattach_execute_request())
+ self._stub.ReattachExecute(
+ self._create_reattach_execute_request(),
metadata=self._metadata
+ )
)
if self._current is None:
@@ -154,7 +156,8 @@ class ExecutePlanResponseReattachableIterator(Generator):
while not has_next:
self._iterator = iter(
self._stub.ReattachExecute(
- self._create_reattach_execute_request()
+
self._create_reattach_execute_request(),
+ metadata=self._metadata,
)
)
# shouldn't change
@@ -192,7 +195,7 @@ class ExecutePlanResponseReattachableIterator(Generator):
can_retry=SparkConnectClient.retry_exception,
**self._retry_policy
):
with attempt:
- self._stub.ReleaseExecute(request)
+ self._stub.ReleaseExecute(request,
metadata=self._metadata)
except Exception as e:
warnings.warn(f"ReleaseExecute failed with exception: {e}.")
@@ -220,7 +223,7 @@ class ExecutePlanResponseReattachableIterator(Generator):
can_retry=SparkConnectClient.retry_exception,
**self._retry_policy
):
with attempt:
- self._stub.ReleaseExecute(request)
+ self._stub.ReleaseExecute(request,
metadata=self._metadata)
except Exception as e:
warnings.warn(f"ReleaseExecute failed with exception: {e}.")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
