submit 12 spark applications in the same time. yarn web page shows: two task fail.
the cmd: ./spark-submit --class org.apache.spark.examples.JavaWordCount --master yarn-cluster ---executor-memory 2g ../lib/spark-examples_2.10-1.1.0.jar hdfs://hacluster/bigData driver log of one fail task: SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory] Exception in thread "Driver" java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) at scala.concurrent.Await$.result(package.scala:107) at akka.remote.Remoting.start(Remoting.scala:173) at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579) at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577) at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588) at akka.actor.ActorSystem$.apply(ActorSystem.scala:111) at akka.actor.ActorSystem$.apply(ActorSystem.scala:104) at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54) at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458) at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448) at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56) at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161) at org.apache.spark.SparkContext.<init>(SparkContext.scala:213) at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56) 2014-11-23 18:41:19,010 | INFO | [main] | Registered signal handlers for [TERM, HUP, INT] | org.apache.spark.util.SignalLogger$.register(SignalLogger.scala:47) 2014-11-23 18:41:54,403 | WARN | [main] | Unable to load native-hadoop library for your platform... using builtin-java classes where applicable | org.apache.hadoop.util.NativeCodeLoader.<clinit>(NativeCodeLoader.java:62) 2014-11-23 18:42:10,319 | INFO | [main] | ApplicationAttemptId: appattempt_1416732306135_0043_000001 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:12,213 | INFO | [main] | Changing view acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:12,280 | INFO | [main] | Changing modify acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:12,300 | INFO | [main] | SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(omm, spark); users with modify permissions: Set(omm, spark) | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:18,597 | INFO | [main] | Starting the user JAR in a separate Thread | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:18,787 | INFO | [main] | Waiting for spark context initialization | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:18,788 | INFO | [main] | Waiting for spark context initialization ... 0 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:19,801 | WARN | [Driver] | In Spark 1.0 and later spark.local.dir will be overridden by the value set by the cluster manager (via SPARK_LOCAL_DIRS in mesos/standalone and LOCAL_DIRS in YARN). | org.apache.spark.Logging$class.logWarning(Logging.scala:71) 2014-11-23 18:42:22,495 | INFO | [Driver] | Changing view acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:22,521 | INFO | [Driver] | Changing modify acls to: omm,spark | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:22,521 | INFO | [Driver] | SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(omm, spark); users with modify permissions: Set(omm, spark) | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:28,823 | INFO | [main] | Waiting for spark context initialization ... 1 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:38,896 | INFO | [main] | Waiting for spark context initialization ... 2 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:47,737 | INFO | [sparkDriver-akka.actor.default-dispatcher-3] | Slf4jLogger started | akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.applyOrElse(Slf4jLogger.scala:80) 2014-11-23 18:42:48,938 | INFO | [main] | Waiting for spark context initialization ... 3 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:42:53,192 | INFO | [sparkDriver-akka.actor.default-dispatcher-4] | Starting remoting | akka.event.slf4j.Slf4jLogger$$anonfun$receive$1$$anonfun$applyOrElse$3.apply$mcV$sp(Slf4jLogger.scala:74) 2014-11-23 18:42:58,969 | INFO | [main] | Waiting for spark context initialization ... 4 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:43:04,721 | ERROR | [sparkDriver-akka.actor.default-dispatcher-3] | Remoting error: [Startup timed out] [ akka.remote.RemoteTransportException: Startup timed out at akka.remote.Remoting.akka$remote$Remoting$$notifyError(Remoting.scala:129) at akka.remote.Remoting.start(Remoting.scala:191) at akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184) at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579) at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577) at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588) at akka.actor.ActorSystem$.apply(ActorSystem.scala:111) at akka.actor.ActorSystem$.apply(ActorSystem.scala:104) at org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121) at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54) at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53) at org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458) at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141) at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448) at org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56) at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161) at org.apache.spark.SparkContext.<init>(SparkContext.scala:213) at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56) at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460) Caused by: java.util.concurrent.TimeoutException: Futures timed out after [10000 milliseconds] at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219) at scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223) at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107) at scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53) at scala.concurrent.Await$.result(package.scala:107) at akka.remote.Remoting.start(Remoting.scala:173) ... 22 more driver log of other fail task: 2014-11-23 18:49:15,962 | INFO | [main] | Waiting for spark context initialization ... 9 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:17,188 | INFO | [Driver] | Successfully started service 'SparkUI' on port 23702. | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:17,246 | INFO | [Driver] | Adding filter: org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:17,482 | INFO | [Driver] | Started SparkUI at http://vm-181:23702 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:20,526 | INFO | [Driver] | Created YarnClusterScheduler | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:26,121 | ERROR | [main] | SparkContext did not initialize after waiting for 100000 ms. Please check earlier log output for errors. Failing the application. | org.apache.spark.Logging$class.logError(Logging.scala:75) 2014-11-23 18:49:27,082 | INFO | [main] | Final app status: FAILED, exitCode: 13, (reason: Timed out waiting for SparkContext.) | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:27,482 | INFO | [main] | In securityManager checkExit, exit code: 13 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:49:28,699 | INFO | [Thread-3] | Unregistering ApplicationMaster with FAILED (diag message: Timed out waiting for SparkContext.) | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:50:22,133 | INFO | [Thread-3] | Deleting staging directory .sparkStaging/application_1416732306135_0043 | org.apache.spark.Logging$class.logInfo(Logging.scala:59) 2014-11-23 18:50:36,274 | WARN | [Driver] | interrupted waiting to send rpc request to server | org.apache.hadoop.ipc.Client.call(Client.java:1388) java.lang.InterruptedException at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:400) at java.util.concurrent.FutureTask.get(FutureTask.java:187) at org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1029) at org.apache.hadoop.ipc.Client.call(Client.java:1383) at org.apache.hadoop.ipc.Client.call(Client.java:1363) at org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206) at com.sun.proxy.$Proxy16.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:701) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187) at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102) at com.sun.proxy.$Proxy17.getFileInfo(Unknown Source) at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1780) at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1165) at org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1161) at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1161) at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1398) at org.apache.spark.util.FileLogger.createLogDir(FileLogger.scala:108) at org.apache.spark.util.FileLogger.start(FileLogger.scala:100) at org.apache.spark.scheduler.EventLoggingListener.start(EventLoggingListener.scala:74) at org.apache.spark.SparkContext.<init>(SparkContext.scala:323) at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56) at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460) --------------------------------------------------------------------- To unsubscribe, e-mail: dev-unsubscr...@spark.apache.org For additional commands, e-mail: dev-h...@spark.apache.org