[
https://issues.apache.org/jira/browse/SPARK-48090?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Hyukjin Kwon resolved SPARK-48090.
----------------------------------
Fix Version/s: 3.5.2
4.0.0
Resolution: Fixed
Issue resolved by pull request 46426
[https://github.com/apache/spark/pull/46426]
> Streaming exception catch failure in 3.5 client <> 4.0 server
> -------------------------------------------------------------
>
> Key: SPARK-48090
> URL: https://issues.apache.org/jira/browse/SPARK-48090
> Project: Spark
> Issue Type: Sub-task
> Components: PySpark, Structured Streaming
> Affects Versions: 4.0.0
> Reporter: Hyukjin Kwon
> Assignee: Hyukjin Kwon
> Priority: Major
> Labels: pull-request-available
> Fix For: 3.5.2, 4.0.0
>
>
> {code}
> ======================================================================
> FAIL [1.975s]: test_stream_exception
> (pyspark.sql.tests.connect.streaming.test_parity_streaming.StreamingParityTests.test_stream_exception)
> ----------------------------------------------------------------------
> Traceback (most recent call last):
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/tests/streaming/test_streaming.py",
> line 287, in test_stream_exception
> sq.processAllAvailable()
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/streaming/query.py",
> line 129, in processAllAvailable
> self._execute_streaming_query_cmd(cmd)
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/streaming/query.py",
> line 177, in _execute_streaming_query_cmd
> (_, properties) = self._session.client.execute_command(exec_cmd)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/client/core.py",
> line 982, in execute_command
> data, _, _, _, properties = self._execute_and_fetch(req)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/client/core.py",
> line 1283, in _execute_and_fetch
> for response in self._execute_and_fetch_as_iterator(req):
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/client/core.py",
> line 1264, in _execute_and_fetch_as_iterator
> self._handle_error(error)
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/client/core.py",
> line 1503, in _handle_error
> self._handle_rpc_error(error)
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/connect/client/core.py",
> line 1539, in _handle_rpc_error
> raise convert_exception(info, status.message) from None
> pyspark.errors.exceptions.connect.StreamingQueryException: [STREAM_FAILED]
> Query [id = 38d0d145-1f57-4b92-b317-d9de727d9468, runId =
> 2b963119-d391-4c62-abea-970274859b80] terminated with exception: Job aborted
> due to stage failure: Task 0 in stage 79.0 failed 1 times, most recent
> failure: Lost task 0.0 in stage 79.0 (TID 116)
> (fv-az1144-341.tm43j05r3bqe3lauap1nzddazg.ex.internal.cloudapp.net executor
> driver): org.apache.spark.api.python.PythonException: Traceback (most recent
> call last):
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/worker.py",
> line 1834, in main
> process()
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/worker.py",
> line 1826, in process
> serializer.dump_stream(out_iter, outfile)
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/serializers.py",
> line 224, in dump_stream
> self.serializer.dump_stream(self._batched(iterator), stream)
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/serializers.py",
> line 145, in dump_stream
> for obj in iterator:
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/serializers.py",
> line 213, in _batched
> for item in iterator:
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/worker.py",
> line 1734, in mapper
> result = tuple(f(*[a[o] for o in arg_offsets]) for arg_offsets, f in udfs)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/worker.py",
> line 1734, in <genexpr>
> result = tuple(f(*[a[o] for o in arg_offsets]) for arg_offsets, f in udfs)
> ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/worker.py",
> line 112, in <lambda>
> return args_kwargs_offsets, lambda *a: func(*a)
> ^^^^^^^^
> File
> "/home/runner/work/spark/spark/python/lib/pyspark.zip/pyspark/util.py", line
> 118, in wrapper
> return f(*args, **kwargs)
> ^^^^^^^^^^^^^^^^^^
> File "/home/runner/work/spark/spark-3....
> During handling of the above exception, another exception occurred:
> Traceback (most recent call last):
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/tests/streaming/test_streaming.py",
> line 291, in test_stream_exception
> self._assert_exception_tree_contains_msg(e, "ZeroDivisionError")
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/tests/streaming/test_streaming.py",
> line 300, in _assert_exception_tree_contains_msg
> self._assert_exception_tree_contains_msg_connect(exception, msg)
> File
> "/home/runner/work/spark/spark-3.5/python/pyspark/sql/tests/streaming/test_streaming.py",
> line 305, in _assert_exception_tree_contains_msg_connect
> self.assertTrue(
> AssertionError: False is not true : Exception tree doesn't contain the
> expected message: ZeroDivisionError
> ----------------------------------------------------------------------
> {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
