Hi Jenny,
How are you packaging your jar.
Can you please confirm if you have included the Mlib jar inside the fat jar
you have created for your code.
libraryDependencies += "org.apache.spark" % "spark-mllib_2.9.3" %
"0.8.1-incubating"
Thanks,
Jagat Singh
On Thu, Apr 10, 2014 at 8:05 AM, Jenny Zhao <linlin200...@gmail.com> wrote:
>
> Hi all,
>
> I have been able to run LR in local mode, but I am facing problem to run
> it in cluster mode, below is the source script, and stack trace when
> running it cluster mode, I used sbt package to build the project, not sure
> what it is complaining?
>
> another question I have is for LogisticRegression itself:
>
> 1) I noticed, the LogisticRegressionWithSGD doesn't ask information about
> the input features, for instance, if the feature is scale, norminal or
> ordinal, or if MLLib only supports scale features?
>
> 2) Trainning error is pretty high even when the iteration is set to very
> high, do we have number about the accuracy rate of LR model?
>
> Thank you for your help!
>
> /**
> * Logistic regression
> */
> object SparkLogisticRegression {
>
>
> def main(args: Array[String]) {
> if ( args.length != 3) {
> System.err.println("Usage: SparkLogisticRegression <master> <input
> file path> <number of iterations] ")
> System.exit(1)
> }
>
> val numIterations = args(2).toInt;
>
> val sc = new SparkContext(args(0), "SparkLogisticRegression",
> System.getenv("SPARK_HOME"),
> SparkContext.jarOfClass(this.getClass))
>
> // parse in the input data
> val data = sc.textFile(args(1))
> val lpoints = data.map{ line =>
> val parts = line.split(',')
> LabeledPoint(parts(0).toDouble, parts.tail.map( x =>
> x.toDouble).toArray)
> }
>
> // setup LR
> val model = LogisticRegressionWithSGD.train(lpoints, numIterations)
>
> val labelPred = lpoints.map { p =>
> val pred = model.predict(p.features)
> (p.label, pred)
> }
>
> val predErr = labelPred.filter (r => r._1 != r._2).count
> println("Training Error: " + predErr.toDouble/lpoints.count + " " +
> predErr + "/" + lpoints.count)
> }
>
> }
>
> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 0 (task 0.0:0)
> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> at java.lang.Class.forName(Class.java:211)
> at
> org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:37)
> at
> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1609)
> at
> java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1514)
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1768)
> at
> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
> at
> java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1988)
> at
> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1913)
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1795)
> at
> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:364)
> at
> org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
> at
> org.apache.spark.scheduler.ResultTask$.deserializeInfo(ResultTask.scala:63)
> at
> org.apache.spark.scheduler.ResultTask.readExternal(ResultTask.scala:139)
> at
> java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1834)
> at
> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1793)
> at
> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
> at java.io.ObjectInputStream.readObject(ObjectInputStream.java:364)
> at
> org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
> at
> org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:62)
> at
> org.apache.spark.executor.Executor$TaskRunner$$anonfun$run$1.apply$mcV$sp(Executor.scala:195)
> at
> org.apache.spark.deploy.SparkHadoopUtil.runAsUser(SparkHadoopUtil.scala:49)
> at
> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:178)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:906)
> at
> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:929)
> at java.lang.Thread.run(Thread.java:796)
> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 1 (task 0.0:1)
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 1]
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:1 as
> TID 2 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:1 as
> 1696 bytes in 0 ms
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
> TID 3 on executor 0: hdtest023.svl.ibm.com (NODE_LOCAL)
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
> 1696 bytes in 0 ms
> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 3 (task 0.0:0)
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 2]
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
> TID 4 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
> 1696 bytes in 1 ms
> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 4 (task 0.0:0)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 3]
> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 2 (task 0.0:1)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 4]
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Starting task 0.0:1 as
> TID 5 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Serialized task 0.0:1 as
> 1696 bytes in 1 ms
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
> TID 6 on executor 0: hdtest023.svl.ibm.com (NODE_LOCAL)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
> 1696 bytes in 1 ms
> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 5 (task 0.0:1)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 5]
> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 6 (task 0.0:0)
> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
> [duplicate 6]
> 14/04/09 14:50:49 ERROR scheduler.TaskSetManager: Task 0.0:0 failed 4
> times; aborting job
> 14/04/09 14:50:49 INFO scheduler.TaskSchedulerImpl: Remove TaskSet 0.0
> from pool
> 14/04/09 14:50:49 INFO scheduler.DAGScheduler: Failed to run collect at
> SparkLinearRegression.scala:34
> ^[[0m[^[[31merror^[[0m] ^[[0m(run-main) org.apache.spark.SparkException:
> Job aborted: Task 0.0:0 failed 4 times (most recent failure: Exception
> failure: java.lang.ClassNotFoundException:
> SparkLinearRegression$$anonfun$2)^[[0m
> org.apache.spark.SparkException: Job aborted: Task 0.0:0 failed 4 times
> (most recent failure: Exception failure: java.lang.ClassNotFoundException:
> SparkLinearRegression$$anonfun$2)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1028)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1026)
> at
> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
> at
> scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
> at org.apache.spark.scheduler.DAGScheduler.org
> $apache$spark$scheduler$DAGScheduler$$abortStage(DAGScheduler.scala:1026)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)
> at scala.Option.foreach(Option.scala:236)
> at
> org.apache.spark.scheduler.DAGScheduler.processEvent(DAGScheduler.scala:619)
> at
> org.apache.spark.scheduler.DAGScheduler$$anonfun$start$1$$anon$2$$anonfun$receive$1.applyOrElse(DAGScheduler.scala:207)
> at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
> at akka.actor.ActorCell.invoke(ActorCell.scala:456)
> at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
> at akka.dispatch.Mailbox.run(Mailbox.scala:219)
> at
> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
> at
> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
> at
> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
> at
> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
> at
> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
> ^[[0m[^[[31mtrace^[[0m] ^[[0mStack trace suppressed: run ^[[34mlast
> compile:run-main^[[0m for the full output.^[[0m
> 14/04/09 14:50:49 INFO network.ConnectionManager: Selector thread was
> interrupted!
> java.lang.RuntimeException: Nonzero exit code: 1
> at scala.sys.package$.error(package.scala:27)
>
>
