Spark 1.4.0-rc3: Actor not found
Just compiled Spark 1.4.0-rc3 for Yarn 2.2 and tried running a job that worked fine for Spark 1.3. The job starts on the cluster (yarn-cluster mode), initial stage starts, but the job fails before any task succeeds with the following error. Any hints? [ERROR] [06/02/2015 09:05:36.962] [Executor task launch worker-0] [akka.tcp://sparkDriver@10.254.6.15:33986/user/CoarseGrainedScheduler] swallowing exception during message send (akka.remote.RemoteTransportExceptionNoStackTrace) Exception in thread "main" akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://sparkDriver@10.254.6.15:33986/), Path(/user/OutputCommitCoordinator)] at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65) at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:67) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:82) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:58) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74) at akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:110) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:267) at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:508) at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:541) at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:531) at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87) at akka.remote.EndpointManager$$anonfun$1.applyOrElse(Remoting.scala:575) at akka.actor.Actor$class.aroundReceive(Actor.scala:465) at akka.remote.EndpointManager.aroundReceive(Remoting.scala:395) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) at akka.actor.ActorCell.invoke(ActorCell.scala:487) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) at akka.dispatch.Mailbox.run(Mailbox.scala:220) at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393) at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
Re: Spark 1.4.0-rc3: Actor not found
How about other jobs? Is it an executor log, or a driver log? Could you post other logs near this error, please? Thank you. Best Regards, Shixiong Zhu 2015-06-02 17:11 GMT+08:00 Anders Arpteg : > Just compiled Spark 1.4.0-rc3 for Yarn 2.2 and tried running a job that > worked fine for Spark 1.3. The job starts on the cluster (yarn-cluster > mode), initial stage starts, but the job fails before any task succeeds > with the following error. Any hints? > > [ERROR] [06/02/2015 09:05:36.962] [Executor task launch worker-0] > [akka.tcp://sparkDriver@10.254.6.15:33986/user/CoarseGrainedScheduler] > swallowing exception during message send > (akka.remote.RemoteTransportExceptionNoStackTrace) > Exception in thread "main" akka.actor.ActorNotFound: Actor not found for: > ActorSelection[Anchor(akka.tcp://sparkDriver@10.254.6.15:33986/), > Path(/user/OutputCommitCoordinator)] > at > akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65) > at > akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63) > at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) > at > akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:67) > at > akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:82) > at > akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) > at > akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) > at > scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) > at > akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:58) > at > akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74) > at > akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:110) > at > akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73) > at > scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) > at > scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248) > at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:267) > at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:508) > at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:541) > at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:531) > at > akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87) > at > akka.remote.EndpointManager$$anonfun$1.applyOrElse(Remoting.scala:575) > at akka.actor.Actor$class.aroundReceive(Actor.scala:465) > at akka.remote.EndpointManager.aroundReceive(Remoting.scala:395) > at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) > at akka.actor.ActorCell.invoke(ActorCell.scala:487) > at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) > at akka.dispatch.Mailbox.run(Mailbox.scala:220) > at > akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393) > at > scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) > at > scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) > >
Re: Spark 1.4.0-rc3: Actor not found
The log is from the log aggregation tool (hortonworks, "yarn logs ..."), so both executors and driver. I'll send a private mail to you with the full logs. Also, tried another job as you suggested, and it actually worked fine. The first job was reading from a parquet source, and the second from an avro source. Could there be some issues with the parquet reader? Thanks, Anders On Tue, Jun 2, 2015 at 11:53 AM, Shixiong Zhu wrote: > How about other jobs? Is it an executor log, or a driver log? Could you > post other logs near this error, please? Thank you. > > Best Regards, > Shixiong Zhu > > 2015-06-02 17:11 GMT+08:00 Anders Arpteg : > >> Just compiled Spark 1.4.0-rc3 for Yarn 2.2 and tried running a job that >> worked fine for Spark 1.3. The job starts on the cluster (yarn-cluster >> mode), initial stage starts, but the job fails before any task succeeds >> with the following error. Any hints? >> >> [ERROR] [06/02/2015 09:05:36.962] [Executor task launch worker-0] >> [akka.tcp://sparkDriver@10.254.6.15:33986/user/CoarseGrainedScheduler] >> swallowing exception during message send >> (akka.remote.RemoteTransportExceptionNoStackTrace) >> Exception in thread "main" akka.actor.ActorNotFound: Actor not found for: >> ActorSelection[Anchor(akka.tcp://sparkDriver@10.254.6.15:33986/), >> Path(/user/OutputCommitCoordinator)] >> at >> akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65) >> at >> akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63) >> at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) >> at >> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:67) >> at >> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:82) >> at >> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) >> at >> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) >> at >> scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) >> at >> akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:58) >> at >> akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74) >> at >> akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:110) >> at >> akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73) >> at >> scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) >> at >> scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248) >> at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:267) >> at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:508) >> at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:541) >> at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:531) >> at >> akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87) >> at >> akka.remote.EndpointManager$$anonfun$1.applyOrElse(Remoting.scala:575) >> at akka.actor.Actor$class.aroundReceive(Actor.scala:465) >> at akka.remote.EndpointManager.aroundReceive(Remoting.scala:395) >> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) >> at akka.actor.ActorCell.invoke(ActorCell.scala:487) >> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) >> at akka.dispatch.Mailbox.run(Mailbox.scala:220) >> at >> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393) >> at >> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) >> at >> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) >> >> >
Re: Spark 1.4.0-rc3: Actor not found
Does it happen every time you read a parquet source? On Tue, Jun 2, 2015 at 3:42 AM, Anders Arpteg wrote: > The log is from the log aggregation tool (hortonworks, "yarn logs ..."), > so both executors and driver. I'll send a private mail to you with the full > logs. Also, tried another job as you suggested, and it actually worked > fine. The first job was reading from a parquet source, and the second from > an avro source. Could there be some issues with the parquet reader? > > Thanks, > Anders > > On Tue, Jun 2, 2015 at 11:53 AM, Shixiong Zhu wrote: > >> How about other jobs? Is it an executor log, or a driver log? Could you >> post other logs near this error, please? Thank you. >> >> Best Regards, >> Shixiong Zhu >> >> 2015-06-02 17:11 GMT+08:00 Anders Arpteg : >> >>> Just compiled Spark 1.4.0-rc3 for Yarn 2.2 and tried running a job that >>> worked fine for Spark 1.3. The job starts on the cluster (yarn-cluster >>> mode), initial stage starts, but the job fails before any task succeeds >>> with the following error. Any hints? >>> >>> [ERROR] [06/02/2015 09:05:36.962] [Executor task launch worker-0] >>> [akka.tcp://sparkDriver@10.254.6.15:33986/user/CoarseGrainedScheduler] >>> swallowing exception during message send >>> (akka.remote.RemoteTransportExceptionNoStackTrace) >>> Exception in thread "main" akka.actor.ActorNotFound: Actor not found >>> for: ActorSelection[Anchor(akka.tcp://sparkDriver@10.254.6.15:33986/), >>> Path(/user/OutputCommitCoordinator)] >>> at >>> akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65) >>> at >>> akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63) >>> at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) >>> at >>> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:67) >>> at >>> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:82) >>> at >>> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) >>> at >>> akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) >>> at >>> scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) >>> at >>> akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:58) >>> at >>> akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74) >>> at >>> akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:110) >>> at >>> akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73) >>> at >>> scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) >>> at >>> scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248) >>> at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:267) >>> at >>> akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:508) >>> at >>> akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:541) >>> at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:531) >>> at >>> akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87) >>> at >>> akka.remote.EndpointManager$$anonfun$1.applyOrElse(Remoting.scala:575) >>> at akka.actor.Actor$class.aroundReceive(Actor.scala:465) >>> at akka.remote.EndpointManager.aroundReceive(Remoting.scala:395) >>> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) >>> at akka.actor.ActorCell.invoke(ActorCell.scala:487) >>> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) >>> at akka.dispatch.Mailbox.run(Mailbox.scala:220) >>> at >>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393) >>> at >>> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) >>> at >>> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) >>> >>> >> >
Re: Spark 1.4.0-rc3: Actor not found
Tried on some other data sources as well, and it actually works for some parquet sources. Potentially some specific problems with that first parquet source that I tried with, and not a Spark 1.4 problem. I'll get back with more info if I find any new information. Thanks, Anders On Tue, Jun 2, 2015 at 8:45 PM, Yin Huai wrote: > Does it happen every time you read a parquet source? > > On Tue, Jun 2, 2015 at 3:42 AM, Anders Arpteg wrote: > >> The log is from the log aggregation tool (hortonworks, "yarn logs ..."), >> so both executors and driver. I'll send a private mail to you with the full >> logs. Also, tried another job as you suggested, and it actually worked >> fine. The first job was reading from a parquet source, and the second from >> an avro source. Could there be some issues with the parquet reader? >> >> Thanks, >> Anders >> >> On Tue, Jun 2, 2015 at 11:53 AM, Shixiong Zhu wrote: >> >>> How about other jobs? Is it an executor log, or a driver log? Could you >>> post other logs near this error, please? Thank you. >>> >>> Best Regards, >>> Shixiong Zhu >>> >>> 2015-06-02 17:11 GMT+08:00 Anders Arpteg : >>> Just compiled Spark 1.4.0-rc3 for Yarn 2.2 and tried running a job that worked fine for Spark 1.3. The job starts on the cluster (yarn-cluster mode), initial stage starts, but the job fails before any task succeeds with the following error. Any hints? [ERROR] [06/02/2015 09:05:36.962] [Executor task launch worker-0] [akka.tcp://sparkDriver@10.254.6.15:33986/user/CoarseGrainedScheduler] swallowing exception during message send (akka.remote.RemoteTransportExceptionNoStackTrace) Exception in thread "main" akka.actor.ActorNotFound: Actor not found for: ActorSelection[Anchor(akka.tcp://sparkDriver@10.254.6.15:33986/), Path(/user/OutputCommitCoordinator)] at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:65) at akka.actor.ActorSelection$$anonfun$resolveOne$1.apply(ActorSelection.scala:63) at scala.concurrent.impl.CallbackRunnable.run(Promise.scala:32) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.processBatch$1(BatchingExecutor.scala:67) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply$mcV$sp(BatchingExecutor.scala:82) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) at akka.dispatch.BatchingExecutor$Batch$$anonfun$run$1.apply(BatchingExecutor.scala:59) at scala.concurrent.BlockContext$.withBlockContext(BlockContext.scala:72) at akka.dispatch.BatchingExecutor$Batch.run(BatchingExecutor.scala:58) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.unbatchedExecute(Future.scala:74) at akka.dispatch.BatchingExecutor$class.execute(BatchingExecutor.scala:110) at akka.dispatch.ExecutionContexts$sameThreadExecutionContext$.execute(Future.scala:73) at scala.concurrent.impl.CallbackRunnable.executeWithValue(Promise.scala:40) at scala.concurrent.impl.Promise$DefaultPromise.tryComplete(Promise.scala:248) at akka.pattern.PromiseActorRef.$bang(AskSupport.scala:267) at akka.actor.EmptyLocalActorRef.specialHandle(ActorRef.scala:508) at akka.actor.DeadLetterActorRef.specialHandle(ActorRef.scala:541) at akka.actor.DeadLetterActorRef.$bang(ActorRef.scala:531) at akka.remote.RemoteActorRefProvider$RemoteDeadLetterActorRef.$bang(RemoteActorRefProvider.scala:87) at akka.remote.EndpointManager$$anonfun$1.applyOrElse(Remoting.scala:575) at akka.actor.Actor$class.aroundReceive(Actor.scala:465) at akka.remote.EndpointManager.aroundReceive(Remoting.scala:395) at akka.actor.ActorCell.receiveMessage(ActorCell.scala:516) at akka.actor.ActorCell.invoke(ActorCell.scala:487) at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:238) at akka.dispatch.Mailbox.run(Mailbox.scala:220) at akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:393) at scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260) at scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339) >>> >> >
