[ https://issues.apache.org/jira/browse/SPARK-20225?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15956877#comment-15956877 ]
Darshan Mehta commented on SPARK-20225: --------------------------------------- Attached are the Spark UI screenshots with Failed Tasks and stacktrace. > Spark Job hangs while writing parquet files to HDFS > --------------------------------------------------- > > Key: SPARK-20225 > URL: https://issues.apache.org/jira/browse/SPARK-20225 > Project: Spark > Issue Type: Bug > Components: Input/Output > Affects Versions: 2.1.0 > Reporter: Darshan Mehta > Attachments: Failed_Tasks_2.JPG, Failed_Tasks.JPG > > > While saving a dataframe to parquet using {{baseDataset.write.parquet("some > location")}} 3 stages failed, however, it did not notify the (parent) tasks > which got stuck on 80%. > Here's the sample (pseudo) code: > {code:title=Test.scala|borderStyle=solid} > val values = Map(...) > val range = Seq(..) > val collection = //get collection > collection.foreach(range=>{ > values.foreach(e=> { > val baseDataset = //get dataset > baseDataset.write.parquet("some location") > }) > }) > {code} > This is used as a paragraph in zeppelin and it gets stuck at 80% after > completing 5-6 iterations (I have tried debugging it by writing {{println}} > statements). > Here's what I can see in Thread dump : > {code} > "pool-2-thread-10" #174 prio=5 os_prio=0 tid=0x00007f9f1807e800 nid=0x3945 > waiting on condition [0x00007f9eddec1000] > java.lang.Thread.State: WAITING (parking) > at sun.misc.Unsafe.park(Native Method) > - parking to wait for <0x0000000552b4bef0> (a > scala.concurrent.impl.Promise$CompletionLatch) > at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997) > at > java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304) > at > scala.concurrent.impl.Promise$DefaultPromise.tryAwait(Promise.scala:202) > at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:218) > at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:153) > at > org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:619) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931) > at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:127) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121) > at > org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57) > at > org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:121) > at > org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:101) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58) > - locked <0x0000000552b92c08> (a > org.apache.spark.sql.execution.command.ExecutedCommandExec) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56) > at > org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74) > at > org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) > at > org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114) > at > org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135) > at > org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151) > at > org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132) > at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113) > at > org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87) > - locked <0x0000000552b92c80> (a > org.apache.spark.sql.execution.QueryExecution) > at > org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87) > at > org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:492) > at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:215) > at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:198) > at > org.apache.spark.sql.DataFrameWriter.parquet(DataFrameWriter.scala:494) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:82) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1$$anonfun$apply$1.apply(<console>:63) > at scala.collection.immutable.Map$Map4.foreach(Map.scala:188) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:63) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$anonfun$1.apply(<console>:62) > at scala.collection.immutable.Stream.foreach(Stream.scala:594) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:62) > at > $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:89) > at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw$$iw.<init>(<console>:91) > at $line167049345059.$read$$iw$$iw$$iw$$iw$$iw.<init>(<console>:93) > at $line167049345059.$read$$iw$$iw$$iw$$iw.<init>(<console>:95) > at $line167049345059.$read$$iw$$iw$$iw.<init>(<console>:97) > at $line167049345059.$read$$iw$$iw.<init>(<console>:99) > at $line167049345059.$read$$iw.<init>(<console>:101) > at $line167049345059.$read.<init>(<console>:103) > at $line167049345059.$read$.<init>(<console>:107) > at $line167049345059.$read$.<clinit>(<console>) > at $line167049345059.$eval$.$print$lzycompute(<console>:7) > - locked <0x000000054940c200> (a $line167049345059.$eval$) > at $line167049345059.$eval$.$print(<console>:6) > at $line167049345059.$eval.$print(<console>) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at scala.tools.nsc.interpreter.IMain$ReadEvalPrint.call(IMain.scala:786) > at > scala.tools.nsc.interpreter.IMain$Request.loadAndRun(IMain.scala:1047) > at > scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:638) > at > scala.tools.nsc.interpreter.IMain$WrappedRequest$$anonfun$loadAndRunReq$1.apply(IMain.scala:637) > at > scala.reflect.internal.util.ScalaClassLoader$class.asContext(ScalaClassLoader.scala:31) > at > scala.reflect.internal.util.AbstractFileClassLoader.asContext(AbstractFileClassLoader.scala:19) > at > scala.tools.nsc.interpreter.IMain$WrappedRequest.loadAndRunReq(IMain.scala:637) > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:569) > at scala.tools.nsc.interpreter.IMain.interpret(IMain.scala:565) > at sun.reflect.GeneratedMethodAccessor16.invoke(Unknown Source) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:498) > at org.apache.zeppelin.spark.Utils.invokeMethod(Utils.java:38) > at > org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:961) > at > org.apache.zeppelin.spark.SparkInterpreter.interpretInput(SparkInterpreter.java:1187) > at > org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1133) > - locked <0x0000000531629650> (a > org.apache.zeppelin.spark.SparkInterpreter) > at > org.apache.zeppelin.spark.SparkInterpreter.interpret(SparkInterpreter.java:1126) > at > org.apache.zeppelin.interpreter.LazyOpenInterpreter.interpret(LazyOpenInterpreter.java:94) > at > org.apache.zeppelin.interpreter.remote.RemoteInterpreterServer$InterpretJob.jobRun(RemoteInterpreterServer.java:489) > at org.apache.zeppelin.scheduler.Job.run(Job.java:175) > at > org.apache.zeppelin.scheduler.FIFOScheduler$1.run(FIFOScheduler.java:139) > at > java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511) > at java.util.concurrent.FutureTask.run(FutureTask.java:266) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$201(ScheduledThreadPoolExecutor.java:180) > at > java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:293) > at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:745) > Locked ownable synchronizers: > - <0x0000000549eb7590> (a > java.util.concurrent.ThreadPoolExecutor$Worker) > {code} -- This message was sent by Atlassian JIRA (v6.3.15#6346) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org