Hi Jungtaek,
Please find full logs:
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 06:40:31 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 06:45:32 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 06:50:33 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 06:55:34 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 07:00:35 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 07:05:36 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at
java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
21/03/02 07:10:37 WARN WatchConnectionManager: Exec Failure
java.io.EOFException
at okio.RealBufferedSource.require(RealBufferedSource.java:61)
at okio.RealBufferedSource.readByte(RealBufferedSource.java:74)
at
okhttp3.internal.ws.WebSocketReader.readHeader(WebSocketReader.java:117)
at
okhttp3.internal.ws.WebSocketReader.processNextFrame(WebSocketReader.java:101)
at
okhttp3.internal.ws.RealWebSocket.loopReader(RealWebSocket.java:274)
at
okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:214)
at okhttp3.RealCall$AsyncCall.execute(RealCall.java:203)
at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32)
at
java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
xecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:834)
[2021-03-02 07:25:35,740] - - [ERROR] - StreamingQueryException Exception
while calling run_data Traceback (most recent call last):
File "/opt/app/file.py", line 69, in process_streams
query.awaitTermination()
File "/opt/spark/python/lib/pyspark.zip/pyspark/sql/streaming.py", line
103, in awaitTermination
return self._jsq.awaitTermination()
File "/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py",
line 1304, in __call__
return_value = get_return_value(
File "/opt/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line 134,
in deco
raise_from(converted)
File "<string>", line 3, in raise_from
pyspark.sql.utils.StreamingQueryException: Writing job aborted.
=== Streaming Query ===
Identifier: [id = 4ee71252-c480-4d00-866b-0fbd88e9520e, runId =
8f1f1756-da8d-4983-9f76-dc1af626ad84]
Current Committed Offsets: {}
Current Available Offsets: {KafkaV2[Subscribe[test-topic]]:
{"test-topic":{"0":4628}}}
Current State: ACTIVE
Thread State: RUNNABLE
Logical Plan:
WriteToMicroBatchDataSource
org.apache.spark.sql.execution.streaming.sources.ForeachWriterTable$$anon$1$$anon$2@30f2abbb
+- Project [cast(value#8 as string) AS value#21]
+- StreamingDataSourceV2Relation [key#7, value#8, topic#9, partition#10,
offset#11L, timestamp#12, timestampType#13],
org.apache.spark.sql.kafka010.KafkaSourceProvider$KafkaScan@433a9c3b,
KafkaV2[Subscribe[test-topic]]
Kind Regards,
Sachit Murarka
On Tue, Mar 2, 2021 at 2:53 PM Jungtaek Lim <kabhwan.opensou...@gmail.com>
wrote:
> I feel this quite lacks information. Full stack traces from
> driver/executors are essential at least to determine what was happening.
>
> On Tue, Mar 2, 2021 at 5:26 PM Sachit Murarka <connectsac...@gmail.com>
> wrote:
>
>> Hi All,
>>
>> My spark job is crashing (Structured stream) . Can anyone help please. I
>> am using spark 3.0.1 with kubernetes.
>>
>> [ERROR] - StreamingQueryException Exception in query.awaitTermination()
>> File "/opt/spark/python/lib/pyspark.zip/pyspark/sql/streaming.py", line
>> 103, in awaitTermination
>> return self._jsq.awaitTermination()
>> File "/opt/spark/python/lib/py4j-0.10.9-src.zip/py4j/java_gateway.py",
>> line 1304, in __call__
>> return_value = get_return_value(
>> File "/opt/spark/python/lib/pyspark.zip/pyspark/sql/utils.py", line
>> 134, in deco
>> raise_from(converted)
>>
>> Kind Regards,
>> Sachit Murarka
>>
>
