[
https://issues.apache.org/jira/browse/SPARK-20225?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Darshan Mehta updated SPARK-20225:
----------------------------------
Attachment: Failed_Tasks_2.JPG
Failed_Tasks.JPG
> Spark Job hangs while writing parquet files to HDFS
> ---------------------------------------------------
>
> Key: SPARK-20225
> URL: https://issues.apache.org/jira/browse/SPARK-20225
> Project: Spark
> Issue Type: Bug
> Components: Input/Output
> Affects Versions: 2.1.0
> Reporter: Darshan Mehta
> Attachments: Failed_Tasks_2.JPG, Failed_Tasks.JPG
>
>
> While saving a dataframe to parquet using {{baseDataset.write.parquet("some
> location")}} 3 stages failed, however, it did not notify the (parent) tasks
> which got stuck on 80%.
> Here's the sample (pseudo) code:
> {code:title=Test.scala|borderStyle=solid}
> val values = Map(...)
> val range = Seq(..)
> val collection = //get collection
> collection.foreach(range=>{
> values.foreach(e=> {
> val baseDataset = //get dataset
> baseDataset.write.parquet("some location")
> })
> })
> {code}
> This is used as a paragraph in zeppelin and it gets stuck at 80% after
> completing 5-6 iterations (I have tried debugging it by writing {{println}}
> statements).
> Here's what I can see in Thread dump :
> {code}
> "pool-2-thread-10" #174 prio=5 os_prio=0 tid=0x00007f9f1807e800 nid=0x3945
> waiting on condition [0x00007f9eddec1000]
> java.lang.Thread.State: WAITING (parking)
> at sun.misc.Unsafe.park(Native Method)
> - parking to wait for <0x0000000552b4bef0> (a
> scala.concurrent.impl.Promise$CompletionLatch)
> at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
> at
> java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
> at
> scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202)
> at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218)
> at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153)
> at
> org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
> at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:127)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
> at
> org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
> at
> org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:121)
> at
> org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:101)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
> - locked <0x0000000552b92c08> (a
> org.apache.spark.sql.execution.command.ExecutedCommandExec)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
> at
> org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74)
> at
> org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
> at
> org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
> at
> org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135)
> at
> org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at
> org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132)
> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113)
> at
> org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87)
> - locked <0x0000000552b92c80> (a
> org.apache.spark.sql.execution.QueryExecution)
> at
> org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87)
> at
> org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:492)
> at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:215)
> at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:198)
> at
> org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:494)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:82)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:63)
> at scala.collection.immutable.Map$Map4.foreach(Map.scala:188)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:63)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:62)
> at scala.collection.immutable.Stream.foreach(Stream.scala:594)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:62)
> at
> $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:89)
> at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:91)
> at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:93)
> at $line167049345059.$read$$iw$$iw$$iw$$iw.<init>(<console>:95)
> at $line167049345059.$read$$iw$$iw$$iw.<init>(<console>:97)
> at $line167049345059.$read$$iw$$iw.<init>(<console>:99)
> at $line167049345059.$read$$iw.<init>(<console>:101)
> at $line167049345059.$read.<init>(<console>:103)
> at $line167049345059.$read$.<init>(<console>:107)
> at $line167049345059.$read$.<clinit>(<console>)
> at $line167049345059.$eval$.$print$lzycompute(<console>:7)
> - locked <0x000000054940c200> (a $line167049345059.$eval$)
> at $line167049345059.$eval$.$print(<console>:6)
> at $line167049345059.$eval.$print(<console>)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786)
> at
> scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637)
> at
> scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31)
> at
> scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19)
> at
> scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569)
> at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565)
> at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38)
> at
> org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:961)
> at
> org.apache.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:1187)
> at
> org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1133)
> - locked <0x0000000531629650> (a
> org.apache.zeppelin.spark.SparkInterpreter)
> at
> org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1126)
> at
> org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:94)
> at
> org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:489)
> at org.apache.zeppelin.scheduler.Job.run(Job.java:175)
> at
> org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139)
> at
> java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.run(FutureTask.java:266)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180)
> at
> java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293)
> at
> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
> at java.lang.Thread.run(Thread.java:745)
> Locked ownable synchronizers:
> - <0x0000000549eb7590> (a
> java.util.concurrent.ThreadPoolExecutor$Worker)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
