Thanks for the pointer it led me to http://spark.apache.org/docs/1.2.0/tuning.html increasing parallelism resolved the issue.
On Mon, Feb 16, 2015 at 11:57 PM, Akhil Das <ak...@sigmoidanalytics.com> wrote: > Increase your executor memory, Also you can play around with increasing > the number of partitions/parallelism etc. > > Thanks > Best Regards > > On Tue, Feb 17, 2015 at 3:39 AM, Harshvardhan Chauhan <ha...@gumgum.com> > wrote: > >> Hi All, >> >> >> I need some help with Out Of Memory errors in my application. I am using >> Spark 1.1.0 and my application is using Java API. I am running my app on >> EC2 25 m3.xlarge (4 Cores 15GB Memory) instances. The app only fails >> sometimes. Lots of mapToPair tasks a failing. My app is configured to run >> 120 executors and executor memory is 2G. >> >> These are various errors i see the in my logs. >> >> 15/02/16 10:53:48 INFO storage.MemoryStore: Block broadcast_1 of size 4680 >> dropped from memory (free 257277829) >> 15/02/16 10:53:49 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0x6e0138a3, >> /10.61.192.194:35196 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> java.lang.OutOfMemoryError: Java heap space >> at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57) >> at java.nio.ByteBuffer.allocate(ByteBuffer.java:331) >> at >> org.jboss.netty.buffer.CompositeChannelBuffer.toByteBuffer(CompositeChannelBuffer.java:649) >> at >> org.jboss.netty.buffer.AbstractChannelBuffer.toByteBuffer(AbstractChannelBuffer.java:530) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:77) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:46) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:194) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromTaskLoop(AbstractNioWorker.java:152) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioChannel$WriteTask.run(AbstractNioChannel.java:335) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:366) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:290) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:90) >> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) >> at java.lang.Thread.run(Thread.java:745) >> 15/02/16 10:53:49 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0x2d0c1db1, >> /10.169.226.254:55790 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> java.lang.OutOfMemoryError: Java heap space >> at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57) >> at java.nio.ByteBuffer.allocate(ByteBuffer.java:331) >> at >> org.jboss.netty.buffer.CompositeChannelBuffer.toByteBuffer(CompositeChannelBuffer.java:649) >> at >> org.jboss.netty.buffer.AbstractChannelBuffer.toByteBuffer(AbstractChannelBuffer.java:530) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:77) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:46) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:194) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromTaskLoop(AbstractNioWorker.java:152) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioChannel$WriteTask.run(AbstractNioChannel.java:335) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:366) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:290) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:90) >> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) >> at java.lang.Thread.run(Thread.java:745) >> 15/02/16 10:53:50 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0xd4211985, >> /10.181.125.52:60959 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> java.lang.OutOfMemoryError: Java heap space >> at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57) >> at java.nio.ByteBuffer.allocate(ByteBuffer.java:331) >> at >> org.jboss.netty.buffer.CompositeChannelBuffer.toByteBuffer(CompositeChannelBuffer.java:649) >> at >> org.jboss.netty.buffer.AbstractChannelBuffer.toByteBuffer(AbstractChannelBuffer.java:530) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:77) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:46) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:194) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromTaskLoop(AbstractNioWorker.java:152) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioChannel$WriteTask.run(AbstractNioChannel.java:335) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:366) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:290) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:90) >> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) >> at java.lang.Thread.run(Thread.java:745) >> 15/02/16 10:53:50 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0x889ad652, >> /10.61.192.194:35199 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> java.lang.OutOfMemoryError: Java heap space >> at java.nio.HeapByteBuffer.<init>(HeapByteBuffer.java:57) >> at java.nio.ByteBuffer.allocate(ByteBuffer.java:331) >> at >> org.jboss.netty.buffer.CompositeChannelBuffer.toByteBuffer(CompositeChannelBuffer.java:649) >> at >> org.jboss.netty.buffer.AbstractChannelBuffer.toByteBuffer(AbstractChannelBuffer.java:530) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:77) >> at >> org.jboss.netty.channel.socket.nio.SocketSendBufferPool.acquire(SocketSendBufferPool.java:46) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.write0(AbstractNioWorker.java:194) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.writeFromTaskLoop(AbstractNioWorker.java:152) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioChannel$WriteTask.run(AbstractNioChannel.java:335) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.processTaskQueue(AbstractNioSelector.java:366) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioSelector.run(AbstractNioSelector.java:290) >> at >> org.jboss.netty.channel.socket.nio.AbstractNioWorker.run(AbstractNioWorker.java:90) >> at org.jboss.netty.channel.socket.nio.NioWorker.run(NioWorker.java:178) >> at >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) >> at >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) >> at java.lang.Thread.run(Thread.java:745) >> 15/02/16 10:53:51 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0xd4e2a539, >> /10.69.48.33:60287 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> java.lang.OutOfMemoryError: Java heap space >> 15/02/16 10:53:51 WARN channel.DefaultChannelPipeline: An exception was >> thrown by a user handler while handling an exception event ([id: 0x25831d30, >> /10.61.192.194:35197 => /10.164.164.228:49445] EXCEPTION: >> java.lang.OutOfMemoryError: Java heap space) >> >> >> >> 15/02/15 10:49:12 WARN scheduler.TaskSetManager: Lost task 96.0 in stage 0.0 >> (TID 2078, ip-10-149-84-49.ec2.internal): org.apache.spark.SparkException: >> Error communicating with MapOutputTracker >> >> org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:111) >> >> org.apache.spark.MapOutputTracker.getServerStatuses(MapOutputTracker.scala:160) >> >> org.apache.spark.shuffle.hash.BlockStoreShuffleFetcher$.fetch(BlockStoreShuffleFetcher.scala:43) >> >> org.apache.spark.shuffle.hash.HashShuffleReader.read(HashShuffleReader.scala:41) >> org.apache.spark.rdd.ShuffledRDD.compute(ShuffledRDD.scala:92) >> org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:262) >> org.apache.spark.CacheManager.getOrCompute(CacheManager.scala:61) >> org.apache.spark.rdd.RDD.iterator(RDD.scala:227) >> org.apache.spark.rdd.MappedRDD.compute(MappedRDD.scala:31) >> org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:262) >> org.apache.spark.rdd.RDD.iterator(RDD.scala:229) >> org.apache.spark.rdd.FilteredRDD.compute(FilteredRDD.scala:34) >> org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:262) >> org.apache.spark.rdd.RDD.iterator(RDD.scala:229) >> org.apache.spark.rdd.MappedRDD.compute(MappedRDD.scala:31) >> org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:262) >> org.apache.spark.rdd.RDD.iterator(RDD.scala:229) >> org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:62) >> org.apache.spark.scheduler.Task.run(Task.scala:54) >> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:177) >> >> java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1145) >> >> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615) >> java.lang.Thread.run(Thread.java:745) >> >> >> >> 15/02/16 10:57:26 INFO remote.RemoteActorRefProvider$RemotingTerminator: >> Remote daemon shut down; proceeding with flushing remote transports. >> Exception in thread "Driver" java.lang.reflect.InvocationTargetException >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> at >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) >> at >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> at java.lang.reflect.Method.invoke(Method.java:606) >> at >> org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:162) >> Caused by: org.apache.spark.SparkException: Job cancelled because >> SparkContext was shut down >> at >> org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:694) >> at >> org.apache.spark.scheduler.DAGScheduler$$anonfun$cleanUpAfterSchedulerStop$1.apply(DAGScheduler.scala:693) >> at scala.collection.mutable.HashSet.foreach(HashSet.scala:79) >> at >> org.apache.spark.scheduler.DAGScheduler.cleanUpAfterSchedulerStop(DAGScheduler.scala:693) >> at >> org.apache.spark.scheduler.DAGSchedulerEventProcessActor.postStop(DAGScheduler.scala:1399) >> at >> akka.actor.dungeon.FaultHandling$class.akka$actor$dungeon$FaultHandling$$finishTerminate(FaultHandling.scala:201) >> at >> akka.actor.dungeon.FaultHandling$class.terminate(FaultHandling.scala:163) >> at akka.actor.ActorCell.terminate(ActorCell.scala:338) >> at akka.actor.ActorCell.invokeAll$1(ActorCell.scala:431) >> at akka.actor.ActorCell.systemInvoke(ActorCell.scala:447) >> at akka.dispatch.Mailbox.processAllSystemMessages(Mailbox.scala:262) >> at akka.dispatch.Mailbox.run(Mailbox.scala:218) >> at >> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386) >> at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) >> at >> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) >> at >> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979) >> at >> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107) >> >> >> -- >> *Harshvardhan Chauhan* | Software Engineer >> *GumGum* <http://www.gumgum.com/> | *Ads that stick* >> 310-260-9666 | ha...@gumgum.com >> > > -- *Harshvardhan Chauhan* | Software Engineer *GumGum* <http://www.gumgum.com/> | *Ads that stick* 310-260-9666 | ha...@gumgum.com
