I have this Spark App and it fails when i run with 6 executors but succeeds with 8.
Any suggestions ? Command: ./bin/spark-submit -v --master yarn-cluster --driver-class-path /apache/hadoop/share/hadoop/common/hadoop-common-2.4.1-EBAY-2.jar:/apache/hadoop/lib/hadoop-lzo-0.6.0.jar:/apache/hadoop-2.4.1-2.1.3.0-2-EBAY/share/hadoop/yarn/lib/guava-11.0.2.jar:/apache/hadoop-2.4.1-2.1.3.0-2-EBAY/share/hadoop/hdfs/hadoop-hdfs-2.4.1-EBAY-2.jar --jars /apache/hadoop-2.4.1-2.1.3.0-2-EBAY/share/hadoop/hdfs/hadoop-hdfs-2.4.1-EBAY-2.jar,/home/dvasthimal/spark1.3/spark_reporting_dep_only-1.0-SNAPSHOT.jar --num-executors 6 --driver-memory 12g --driver-java-options "-XX:MaxPermSize=8G" --executor-memory 12g --executor-cores 6 --queue hdmi-express --class com.ebay.ep.poc.spark.reporting.SparkApp /home/dvasthimal/spark1.3/spark_reporting-1.0-SNAPSHOT.jar startDate=2015-04-6 endDate=2015-04-7 input=/user/dvasthimal/epdatasets_small/exptsession subcommand=viewItem output=/user/dvasthimal/epdatasets/viewItem buffersize=128 maxbuffersize=1068 maxResultSize=2G Input Data Sets Size: -sh-4.1$ hadoop fs -ls XX/YY/part-r-00000 2663019338 bytes -sh-4.1$ hadoop fs -ls /AA/BB/part-r-00000 2688348022 bytes -sh-4.1$ hadoop fs -ls /FOO/BAAR/ch196out83-r-00000.avro 1274065689 bytes -sh-4.1$ Any thouhgts ? Exception: 15/04/27 22:12:46 INFO Configuration.deprecation: mapred.output.compression.codec is deprecated. Instead, use mapreduce.output.fileoutputformat.compress.codec 15/04/27 22:12:47 ERROR executor.Executor: Exception in task 8.0 in stage 4.0 (TID 36) scala.reflect.internal.Symbols$CyclicReference: illegal cyclic reference involving object SchemaUtil at scala.reflect.internal.Symbols$Symbol$$anonfun$info$3.apply(Symbols.scala:1220) at scala.reflect.internal.Symbols$Symbol$$anonfun$info$3.apply(Symbols.scala:1218) at scala.Function0$class.apply$mcV$sp(Function0.scala:40) at scala.runtime.AbstractFunction0.apply$mcV$sp(AbstractFunction0.scala:12) at scala.reflect.internal.Symbols$Symbol.lock(Symbols.scala:482) at scala.reflect.internal.Symbols$Symbol.info(Symbols.scala:1218) at scala.reflect.internal.Symbols$Symbol.initialize(Symbols.scala:1374) at scala.reflect.internal.Symbols$Symbol.hasFlag(Symbols.scala:607) at scala.reflect.internal.Symbols$TermSymbol.isTermMacro(Symbols.scala:2453) at scala.reflect.internal.Symbols$Symbol.symbolKind(Symbols.scala:2263) at scala.reflect.internal.Symbols$Symbol.sanitizedKindString(Symbols.scala:2297) at scala.reflect.internal.Symbols$Symbol.kindString(Symbols.scala:2305) at scala.reflect.internal.Symbols$Symbol.toString(Symbols.scala:2350) at java.lang.String.valueOf(String.java:2847) at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:197) at scala.collection.TraversableOnce$$anonfun$addString$1.apply(TraversableOnce.scala:327) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableOnce$class.addString(TraversableOnce.scala:320) at scala.collection.AbstractTraversable.addString(Traversable.scala:105) at scala.collection.TraversableOnce$class.mkString(TraversableOnce.scala:286) at scala.collection.AbstractTraversable.mkString(Traversable.scala:105) at scala.collection.TraversableLike$class.toString(TraversableLike.scala:639) at scala.collection.SeqLike$class.toString(SeqLike.scala:646) at scala.collection.AbstractSeq.toString(Seq.scala:40) at java.lang.String.valueOf(String.java:2847) at scala.collection.mutable.StringBuilder.append(StringBuilder.scala:197) at scala.reflect.internal.Symbols$Symbol.suchThat(Symbols.scala:1678) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:44) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:61) at scala.reflect.internal.Mirrors$RootsBase.staticModuleOrClass(Mirrors.scala:72) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:161) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:21) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$typecreator3$1.apply(SchemaUtil.scala:108) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:231) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:231) at scala.reflect.api.TypeTags$class.typeOf(TypeTags.scala:335) at scala.reflect.api.Universe.typeOf(Universe.scala:59) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.com$ebay$ep$poc$spark$reporting$process$util$SchemaUtil$$getField(SchemaUtil.scala:108) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:94) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:90) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.foreach(Scopes.scala:315) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.flatMap(Scopes.scala:44) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:89) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:88) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.getDataFields(SchemaUtil.scala:87) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:39) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:224) at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) at scala.collection.AbstractTraversable.map(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchema(SchemaUtil.scala:39) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchemaInstance(SchemaUtil.scala:71) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:79) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:76) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:999) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:979) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61) at org.apache.spark.scheduler.Task.run(Task.scala:64) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 15/04/27 22:12:47 ERROR executor.Executor: Exception in task 2.0 in stage 4.0 (TID 30) java.lang.AssertionError: assertion failed: List(object SchemaUtil, object SchemaUtil) at scala.reflect.internal.Symbols$Symbol.suchThat(Symbols.scala:1678) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:44) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:61) at scala.reflect.internal.Mirrors$RootsBase.staticModuleOrClass(Mirrors.scala:72) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:161) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:21) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$typecreator3$1.apply(SchemaUtil.scala:108) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:231) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:231) at scala.reflect.api.TypeTags$class.typeOf(TypeTags.scala:335) at scala.reflect.api.Universe.typeOf(Universe.scala:59) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.com$ebay$ep$poc$spark$reporting$process$util$SchemaUtil$$getField(SchemaUtil.scala:108) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:94) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:90) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.foreach(Scopes.scala:315) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.flatMap(Scopes.scala:44) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:89) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:88) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.getDataFields(SchemaUtil.scala:87) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:39) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:224) at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) at scala.collection.AbstractTraversable.map(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchema(SchemaUtil.scala:39) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchemaInstance(SchemaUtil.scala:71) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:79) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:76) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:999) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:979) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61) at org.apache.spark.scheduler.Task.run(Task.scala:64) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 15/04/27 22:12:47 INFO executor.CoarseGrainedExecutorBackend: Got assigned task 41 15/04/27 22:12:47 INFO executor.Executor: Running task 6.1 in stage 4.0 (TID 41) 15/04/27 22:12:47 INFO storage.ShuffleBlockFetcherIterator: Getting 10 non-empty blocks out of 10 blocks 15/04/27 22:12:47 INFO storage.ShuffleBlockFetcherIterator: Started 5 remote fetches in 2 ms 15/04/27 22:12:48 ERROR executor.Executor: Exception in task 6.1 in stage 4.0 (TID 41) java.lang.AssertionError: assertion failed: List(object SchemaUtil, object SchemaUtil) at scala.reflect.internal.Symbols$Symbol.suchThat(Symbols.scala:1678) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:44) at scala.reflect.internal.Mirrors$RootsBase.getModuleOrClass(Mirrors.scala:61) at scala.reflect.internal.Mirrors$RootsBase.staticModuleOrClass(Mirrors.scala:72) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:161) at scala.reflect.internal.Mirrors$RootsBase.staticModule(Mirrors.scala:21) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$typecreator3$1.apply(SchemaUtil.scala:108) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe$lzycompute(TypeTags.scala:231) at scala.reflect.api.TypeTags$WeakTypeTagImpl.tpe(TypeTags.scala:231) at scala.reflect.api.TypeTags$class.typeOf(TypeTags.scala:335) at scala.reflect.api.Universe.typeOf(Universe.scala:59) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.com$ebay$ep$poc$spark$reporting$process$util$SchemaUtil$$getField(SchemaUtil.scala:108) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:94) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6$$anonfun$apply$4.apply(SchemaUtil.scala:90) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.foreach(Scopes.scala:315) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.reflect.internal.Scopes$Scope.flatMap(Scopes.scala:44) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:89) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$6.apply(SchemaUtil.scala:88) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.getDataFields(SchemaUtil.scala:87) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2$$anonfun$3.apply(SchemaUtil.scala:41) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.TraversableLike$$anonfun$flatMap$1.apply(TraversableLike.scala:251) at scala.collection.immutable.List.foreach(List.scala:318) at scala.collection.TraversableLike$class.flatMap(TraversableLike.scala:251) at scala.collection.AbstractTraversable.flatMap(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:41) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$$anonfun$2.apply(SchemaUtil.scala:39) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244) at scala.collection.immutable.HashMap$HashMap1.foreach(HashMap.scala:224) at scala.collection.TraversableLike$class.map(TraversableLike.scala:244) at scala.collection.AbstractTraversable.map(Traversable.scala:105) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchema(SchemaUtil.scala:39) at com.ebay.ep.poc.spark.reporting.process.util.SchemaUtil$.detailDataColumnSchemaInstance(SchemaUtil.scala:71) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:79) at com.ebay.ep.poc.spark.reporting.process.detail.AbstractInputHelper$$anonfun$1.apply(AbstractInputHelper.scala:76) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at scala.collection.Iterator$$anon$11.next(Iterator.scala:328) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:999) at org.apache.spark.rdd.PairRDDFunctions$$anonfun$12.apply(PairRDDFunctions.scala:979) at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:61) at org.apache.spark.scheduler.Task.run(Task.scala:64) at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:203) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) at java.lang.Thread.run(Thread.java:745) 15/04/27 22:12:50 INFO storage.BlockManager: Removing broadcast 7 15/04/27 22:12:50 INFO storage.BlockManager: Removing block broadcast_7 15/04/27 22:12:50 INFO storage.MemoryStore: Block broadcast_7 of size 3192 dropped from memory (free 5878101161) 15/04/27 22:12:50 INFO storage.BlockManager: Removing block broadcast_7_piece0 15/04/27 22:12:50 INFO storage.MemoryStore: Block broadcast_7_piece0 of size 1770 dropped from memory (free 5878102931) 15/04/27 22:12:50 INFO storage.BlockManagerMaster: Updated info of block broadcast_7_piece0 15/04/27 22:12:52 INFO executor.CoarseGrainedExecutorBackend: Driver commanded a shutdown 15/04/27 22:12:52 INFO storage.MemoryStore: MemoryStore cleared 15/04/27 22:12:52 INFO storage.BlockManager: BlockManager stopped 15/04/27 22:12:52 INFO remote.RemoteActorRefProvider$RemotingTerminator: Shutting down remote daemon. 15/04/27 22:12:52 INFO remote.RemoteActorRefProvider$RemotingTerminator: Remote daemon shut down; proceeding with flushing remote transports. 15/04/27 22:12:52 INFO remote.RemoteActorRefProvider$RemotingTerminator: Remoting shut down. -- Deepak
