[
https://issues.apache.org/jira/browse/HUDI-5670?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Alexey Kudinkin updated HUDI-5670:
----------------------------------
Summary: Server-based markers creation times out (was: Server-based Marker
creation times out)
> Server-based markers creation times out
> ---------------------------------------
>
> Key: HUDI-5670
> URL: https://issues.apache.org/jira/browse/HUDI-5670
> Project: Apache Hudi
> Issue Type: Bug
> Components: writer-core
> Reporter: Alexey Kudinkin
> Assignee: Ethan Guo
> Priority: Blocker
> Fix For: 0.13.1
>
>
> Running writing benchmarks w/ SparkRecordMerger enabled hitting this
> SocketTimeoutException trying to create markers:
> {code:java}
> ception.HoodieRemoteException: Failed to create marker file
> partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
> Read timed out
> at
> org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:121)
> at
> scala.collection.convert.Wrappers$JIteratorWrapper.next(Wrappers.scala:46)
> at scala.collection.Iterator$$anon$11.nextCur(Iterator.scala:486)
> at scala.collection.Iterator$$anon$11.hasNext(Iterator.scala:492)
> at
> org.apache.spark.storage.memory.MemoryStore.putIterator(MemoryStore.scala:223)
> at
> org.apache.spark.storage.memory.MemoryStore.putIteratorAsBytes(MemoryStore.scala:352)
> at
> org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1535)
> at
> org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1445)
> at
> org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1509)
> at
> org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1332)
> at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:376)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:327)
> at
> org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52)
> at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:365)
> at org.apache.spark.rdd.RDD.iterator(RDD.scala:329)
> at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:90)
> at org.apache.spark.scheduler.Task.run(Task.scala:136)
> at
> org.apache.spark.executor.Executor$TaskRunner.$anonfun$run$3(Executor.scala:548)
> at org.apache.spark.util.Utils$.tryWithSafeFinally(Utils.scala:1504)
> at
> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:551)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:750)
> Caused by: org.apache.hudi.exception.HoodieException:
> org.apache.hudi.exception.HoodieException:
> org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file
> partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
> Read timed out
> at
> org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:84)
> at
> org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:39)
> at
> org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119)
> ... 22 more
> Caused by: org.apache.hudi.exception.HoodieException:
> org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file
> partition=2020%2F10%2F29/69adadb4-d7ae-4b30-8af1-92ffa38be7df-0_1362-352-97811_20230201020238055.parquet.marker.CREATE
> Read timed out
> at
> org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:84)
>
>
> [3324/4704]
> at
> org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:39)
> at
> org.apache.hudi.client.utils.LazyIterableIterator.next(LazyIterableIterator.java:119)
> ... 22 more
> Caused by: org.apache.hudi.exception.HoodieException:
> org.apache.hudi.exception.HoodieRemoteException: Failed to create marker file
> partition=2020%2F11%2F29/8e3045e1-6de0-492e-bc34-85e2b8502767-0_1207-352-97656_20230201020238055.parquet.marker.CREATE
> Read timed out
> at
> org.apache.hudi.common.util.queue.SimpleExecutor.execute(SimpleExecutor.java:73)
> at
> org.apache.hudi.execution.SparkLazyInsertIterable.computeNext(SparkLazyInsertIterable.java:80)
> ... 24 more
> Caused by: org.apache.hudi.exception.HoodieRemoteException: Failed to create
> marker file
> partition=2020%2F11%2F29/8e3045e1-6de0-492e-bc34-85e2b8502767-0_1207-352-97656_20230201020238055.parquet.marker.CREATE
> Read timed out
> at
> org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeCreateMarkerRequest(TimelineServerBasedWriteMarkers.java:186)
> at
> org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.create(TimelineServerBasedWriteMarkers.java:141)
> at
> org.apache.hudi.table.marker.WriteMarkers.create(WriteMarkers.java:95)
> at
> org.apache.hudi.io.HoodieWriteHandle.createMarkerFile(HoodieWriteHandle.java:139)
> at
> org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:102)
> at
> org.apache.hudi.io.HoodieCreateHandle.<init>(HoodieCreateHandle.java:75)
> at
> org.apache.hudi.io.CreateHandleFactory.create(CreateHandleFactory.java:45)
> at
> org.apache.hudi.execution.CopyOnWriteInsertHandler.consume(CopyOnWriteInsertHandler.java:85)
> at
> org.apache.hudi.execution.CopyOnWriteInsertHandler.consume(CopyOnWriteInsertHandler.java:42)
> at
> org.apache.hudi.common.util.queue.SimpleExecutor.execute(SimpleExecutor.java:67)
> ... 25 more
> Caused by: java.net.SocketTimeoutException: Read timed out
> at java.net.SocketInputStream.socketRead0(Native Method)
> at java.net.SocketInputStream.socketRead(SocketInputStream.java:116)
> at java.net.SocketInputStream.read(SocketInputStream.java:171)
> at java.net.SocketInputStream.read(SocketInputStream.java:141)
> at
> org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.streamRead(SessionInputBufferImpl.java:139)
> at
> org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.fillBuffer(SessionInputBufferImpl.java:155)
> at
> org.apache.hudi.org.apache.http.impl.io.SessionInputBufferImpl.readLine(SessionInputBufferImpl.java:284)
> at
> org.apache.hudi.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:140)
> at
> org.apache.hudi.org.apache.http.impl.conn.DefaultHttpResponseParser.parseHead(DefaultHttpResponseParser.java:57)
> at
> org.apache.hudi.org.apache.http.impl.io.AbstractMessageParser.parse(AbstractMessageParser.java:261)
> at
> org.apache.hudi.org.apache.http.impl.DefaultBHttpClientConnection.receiveResponseHeader(DefaultBHttpClientConnection.java:165)
> at
> org.apache.hudi.org.apache.http.impl.conn.CPoolProxy.receiveResponseHeader(CPoolProxy.java:167)
> at
> org.apache.hudi.org.apache.http.protocol.HttpRequestExecutor.doReceiveResponse(HttpRequestExecutor.java:272)
> at
> org.apache.hudi.org.apache.http.protocol.HttpRequestExecutor.execute(HttpRequestExecutor.java:124)
> at
> org.apache.hudi.org.apache.http.impl.execchain.MainClientExec.execute(MainClientExec.java:271)
> at
> org.apache.hudi.org.apache.http.impl.execchain.ProtocolExec.execute(ProtocolExec.java:184)
> at
> org.apache.hudi.org.apache.http.impl.execchain.RetryExec.execute(RetryExec.java:88)
> at
> org.apache.hudi.org.apache.http.impl.execchain.RedirectExec.execute(RedirectExec.java:110)
> at
> org.apache.hudi.org.apache.http.impl.client.InternalHttpClient.doExecute(InternalHttpClient.java:184)
> at
> org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:82)
> at
> org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:107)
> at
> org.apache.hudi.org.apache.http.impl.client.CloseableHttpClient.execute(CloseableHttpClient.java:55)
> at
> org.apache.hudi.org.apache.http.client.fluent.Request.execute(Request.java:151)
> at
> org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeRequestToTimelineServer(TimelineServerBasedWriteMarkers.java:232)
> at
> org.apache.hudi.table.marker.TimelineServerBasedWriteMarkers.executeCreateMarkerRequest(TimelineServerBasedWriteMarkers.java:182)
> ... 34 more {code}
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
