submit 12 spark applications in the same time. yarn web page shows: two task
fail.
the cmd:
./spark-submit --class org.apache.spark.examples.JavaWordCount --master
yarn-cluster ---executor-memory 2g ../lib/spark-examples_2.10-1.1.0.jar
hdfs://hacluster/bigData
driver log of one fail task:
SLF4J: Actual binding is of type [org.slf4j.impl.Log4jLoggerFactory]
Exception in thread "Driver" java.util.concurrent.TimeoutException: Futures
timed out after [10000 milliseconds]
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219)
at
scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
at
scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
at scala.concurrent.Await$.result(package.scala:107)
at akka.remote.Remoting.start(Remoting.scala:173)
at
akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184)
at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579)
at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577)
at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588)
at akka.actor.ActorSystem$.apply(ActorSystem.scala:111)
at akka.actor.ActorSystem$.apply(ActorSystem.scala:104)
at
org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121)
at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54)
at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53)
at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458)
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448)
at
org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:213)
at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
2014-11-23 18:41:19,010 | INFO | [main] | Registered signal handlers for
[TERM, HUP, INT] |
org.apache.spark.util.SignalLogger$.register(SignalLogger.scala:47)
2014-11-23 18:41:54,403 | WARN | [main] | Unable to load native-hadoop library
for your platform... using builtin-java classes where applicable |
org.apache.hadoop.util.NativeCodeLoader.<clinit>(NativeCodeLoader.java:62)
2014-11-23 18:42:10,319 | INFO | [main] | ApplicationAttemptId:
appattempt_1416732306135_0043_000001 |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,213 | INFO | [main] | Changing view acls to: omm,spark |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,280 | INFO | [main] | Changing modify acls to: omm,spark |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:12,300 | INFO | [main] | SecurityManager: authentication
disabled; ui acls disabled; users with view permissions: Set(omm, spark); users
with modify permissions: Set(omm, spark) |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,597 | INFO | [main] | Starting the user JAR in a separate
Thread | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,787 | INFO | [main] | Waiting for spark context
initialization | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:18,788 | INFO | [main] | Waiting for spark context
initialization ... 0 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:19,801 | WARN | [Driver] | In Spark 1.0 and later
spark.local.dir will be overridden by the value set by the cluster manager (via
SPARK_LOCAL_DIRS in mesos/standalone and LOCAL_DIRS in YARN). |
org.apache.spark.Logging$class.logWarning(Logging.scala:71)
2014-11-23 18:42:22,495 | INFO | [Driver] | Changing view acls to: omm,spark |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:22,521 | INFO | [Driver] | Changing modify acls to: omm,spark
| org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:22,521 | INFO | [Driver] | SecurityManager: authentication
disabled; ui acls disabled; users with view permissions: Set(omm, spark); users
with modify permissions: Set(omm, spark) |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:28,823 | INFO | [main] | Waiting for spark context
initialization ... 1 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:38,896 | INFO | [main] | Waiting for spark context
initialization ... 2 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:47,737 | INFO | [sparkDriver-akka.actor.default-dispatcher-3]
| Slf4jLogger started |
akka.event.slf4j.Slf4jLogger$$anonfun$receive$1.applyOrElse(Slf4jLogger.scala:80)
2014-11-23 18:42:48,938 | INFO | [main] | Waiting for spark context
initialization ... 3 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:42:53,192 | INFO | [sparkDriver-akka.actor.default-dispatcher-4]
| Starting remoting |
akka.event.slf4j.Slf4jLogger$$anonfun$receive$1$$anonfun$applyOrElse$3.apply$mcV$sp(Slf4jLogger.scala:74)
2014-11-23 18:42:58,969 | INFO | [main] | Waiting for spark context
initialization ... 4 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:43:04,721 | ERROR | [sparkDriver-akka.actor.default-dispatcher-3]
| Remoting error: [Startup timed out] [
akka.remote.RemoteTransportException: Startup timed out
at
akka.remote.Remoting.akka$remote$Remoting$$notifyError(Remoting.scala:129)
at akka.remote.Remoting.start(Remoting.scala:191)
at
akka.remote.RemoteActorRefProvider.init(RemoteActorRefProvider.scala:184)
at akka.actor.ActorSystemImpl._start$lzycompute(ActorSystem.scala:579)
at akka.actor.ActorSystemImpl._start(ActorSystem.scala:577)
at akka.actor.ActorSystemImpl.start(ActorSystem.scala:588)
at akka.actor.ActorSystem$.apply(ActorSystem.scala:111)
at akka.actor.ActorSystem$.apply(ActorSystem.scala:104)
at
org.apache.spark.util.AkkaUtils$.org$apache$spark$util$AkkaUtils$$doCreateActorSystem(AkkaUtils.scala:121)
at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:54)
at org.apache.spark.util.AkkaUtils$$anonfun$1.apply(AkkaUtils.scala:53)
at
org.apache.spark.util.Utils$$anonfun$startServiceOnPort$1.apply$mcVI$sp(Utils.scala:1458)
at scala.collection.immutable.Range.foreach$mVc$sp(Range.scala:141)
at org.apache.spark.util.Utils$.startServiceOnPort(Utils.scala:1448)
at
org.apache.spark.util.AkkaUtils$.createActorSystem(AkkaUtils.scala:56)
at org.apache.spark.SparkEnv$.create(SparkEnv.scala:161)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:213)
at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460)
Caused by: java.util.concurrent.TimeoutException: Futures timed out after
[10000 milliseconds]
at scala.concurrent.impl.Promise$DefaultPromise.ready(Promise.scala:219)
at
scala.concurrent.impl.Promise$DefaultPromise.result(Promise.scala:223)
at scala.concurrent.Await$$anonfun$result$1.apply(package.scala:107)
at
scala.concurrent.BlockContext$DefaultBlockContext$.blockOn(BlockContext.scala:53)
at scala.concurrent.Await$.result(package.scala:107)
at akka.remote.Remoting.start(Remoting.scala:173)
... 22 more
driver log of other fail task:
2014-11-23 18:49:15,962 | INFO | [main] | Waiting for spark context
initialization ... 9 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,188 | INFO | [Driver] | Successfully started service
'SparkUI' on port 23702. |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,246 | INFO | [Driver] | Adding filter:
org.apache.hadoop.yarn.server.webproxy.amfilter.AmIpFilter |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:17,482 | INFO | [Driver] | Started SparkUI at
http://vm-181:23702 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:20,526 | INFO | [Driver] | Created YarnClusterScheduler |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:26,121 | ERROR | [main] | SparkContext did not initialize
after waiting for 100000 ms. Please check earlier log output for errors.
Failing the application. |
org.apache.spark.Logging$class.logError(Logging.scala:75)
2014-11-23 18:49:27,082 | INFO | [main] | Final app status: FAILED, exitCode:
13, (reason: Timed out waiting for SparkContext.) |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:27,482 | INFO | [main] | In securityManager checkExit, exit
code: 13 | org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:49:28,699 | INFO | [Thread-3] | Unregistering ApplicationMaster
with FAILED (diag message: Timed out waiting for SparkContext.) |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:50:22,133 | INFO | [Thread-3] | Deleting staging directory
.sparkStaging/application_1416732306135_0043 |
org.apache.spark.Logging$class.logInfo(Logging.scala:59)
2014-11-23 18:50:36,274 | WARN | [Driver] | interrupted waiting to send rpc
request to server | org.apache.hadoop.ipc.Client.call(Client.java:1388)
java.lang.InterruptedException
at java.util.concurrent.FutureTask.awaitDone(FutureTask.java:400)
at java.util.concurrent.FutureTask.get(FutureTask.java:187)
at
org.apache.hadoop.ipc.Client$Connection.sendRpcRequest(Client.java:1029)
at org.apache.hadoop.ipc.Client.call(Client.java:1383)
at org.apache.hadoop.ipc.Client.call(Client.java:1363)
at
org.apache.hadoop.ipc.ProtobufRpcEngine$Invoker.invoke(ProtobufRpcEngine.java:206)
at com.sun.proxy.$Proxy16.getFileInfo(Unknown Source)
at
org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.getFileInfo(ClientNamenodeProtocolTranslatorPB.java:701)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:187)
at
org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:102)
at com.sun.proxy.$Proxy17.getFileInfo(Unknown Source)
at org.apache.hadoop.hdfs.DFSClient.getFileInfo(DFSClient.java:1780)
at
org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1165)
at
org.apache.hadoop.hdfs.DistributedFileSystem$18.doCall(DistributedFileSystem.java:1161)
at
org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
at
org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1161)
at org.apache.hadoop.fs.FileSystem.exists(FileSystem.java:1398)
at org.apache.spark.util.FileLogger.createLogDir(FileLogger.scala:108)
at org.apache.spark.util.FileLogger.start(FileLogger.scala:100)
at
org.apache.spark.scheduler.EventLoggingListener.start(EventLoggingListener.scala:74)
at org.apache.spark.SparkContext.<init>(SparkContext.scala:323)
at
org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:56)
at org.apache.spark.examples.JavaWordCount.main(JavaWordCount.java:44)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:606)
at
org.apache.spark.deploy.yarn.ApplicationMaster$$anon$2.run(ApplicationMaster.scala:460)
---------------------------------------------------------------------
To unsubscribe, e-mail: dev-unsubscr...@spark.apache.org
For additional commands, e-mail: dev-h...@spark.apache.org
