Hi Jagat,

yes, I did specify mllib in build.sbt

name := "Spark LogisticRegression"

version :="1.0"

scalaVersion := "2.10.3"

libraryDependencies += "org.apache.spark" % "spark-core_2.10" %
"0.9.0-incubating"

libraryDependencies += "org.apache.spark" % "spark-mllib_2.10" %
"0.9.0-incubating"

libraryDependencies += "org.apache.hadoop" % "hadoop-client" % "1.2.1"

resolvers += "Akka Repository" at "http://repo.akka.io/releases/";



On Wed, Apr 9, 2014 at 3:23 PM, Jagat Singh <jagatsi...@gmail.com> wrote:

> Hi Jenny,
>
> How are you packaging your jar.
>
> Can you please confirm if you have included the Mlib jar inside the fat
> jar you have created for your code.
>
> libraryDependencies += "org.apache.spark" % "spark-mllib_2.9.3" %
> "0.8.1-incubating"
>
> Thanks,
>
> Jagat Singh
>
>
> On Thu, Apr 10, 2014 at 8:05 AM, Jenny Zhao <linlin200...@gmail.com>wrote:
>
>>
>> Hi all,
>>
>> I have been able to run LR in local mode,  but I am facing problem to run
>> it in cluster mode,  below is the source script, and stack trace when
>> running it cluster mode, I used sbt package to build the project, not sure
>> what it is complaining?
>>
>> another question I have is for LogisticRegression itself:
>>
>> 1) I noticed, the LogisticRegressionWithSGD doesn't ask information about
>> the input features, for instance, if the feature is scale, norminal or
>> ordinal, or if MLLib only supports scale features?
>>
>> 2) Trainning error is pretty high even when the iteration is set to very
>> high, do we have number about the accuracy rate of LR model?
>>
>> Thank you for your help!
>>
>> /**
>>  * Logistic regression
>>  */
>> object SparkLogisticRegression {
>>
>>
>>   def main(args: Array[String]) {
>>     if ( args.length != 3) {
>>       System.err.println("Usage: SparkLogisticRegression <master> <input
>> file path> <number of iterations]  ")
>>       System.exit(1)
>>     }
>>
>>     val numIterations = args(2).toInt;
>>
>>     val sc = new SparkContext(args(0), "SparkLogisticRegression",
>>                               System.getenv("SPARK_HOME"),
>>                               SparkContext.jarOfClass(this.getClass))
>>
>>     // parse in the input data
>>     val data = sc.textFile(args(1))
>>     val lpoints = data.map{ line =>
>>       val parts = line.split(',')
>>       LabeledPoint(parts(0).toDouble, parts.tail.map( x =>
>> x.toDouble).toArray)
>>     }
>>
>>     // setup LR
>>     val model = LogisticRegressionWithSGD.train(lpoints, numIterations)
>>
>>     val labelPred = lpoints.map { p =>
>>           val pred = model.predict(p.features)
>>           (p.label, pred)
>>     }
>>
>>     val predErr = labelPred.filter (r => r._1 != r._2).count
>>     println("Training Error: " + predErr.toDouble/lpoints.count + " " +
>> predErr + "/" + lpoints.count)
>>  }
>>
>> }
>>
>> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 0 (task 0.0:0)
>> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>>         at java.lang.Class.forName(Class.java:211)
>>         at
>> org.apache.spark.serializer.JavaDeserializationStream$$anon$1.resolveClass(JavaSerializer.scala:37)
>>         at
>> java.io.ObjectInputStream.readNonProxyDesc(ObjectInputStream.java:1609)
>>         at
>> java.io.ObjectInputStream.readClassDesc(ObjectInputStream.java:1514)
>>         at
>> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1768)
>>         at
>> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
>>         at
>> java.io.ObjectInputStream.defaultReadFields(ObjectInputStream.java:1988)
>>         at
>> java.io.ObjectInputStream.readSerialData(ObjectInputStream.java:1913)
>>         at
>> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1795)
>>         at
>> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
>>         at
>> java.io.ObjectInputStream.readObject(ObjectInputStream.java:364)
>>         at
>> org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
>>         at
>> org.apache.spark.scheduler.ResultTask$.deserializeInfo(ResultTask.scala:63)
>>         at
>> org.apache.spark.scheduler.ResultTask.readExternal(ResultTask.scala:139)
>>         at
>> java.io.ObjectInputStream.readExternalData(ObjectInputStream.java:1834)
>>         at
>> java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:1793)
>>         at
>> java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1347)
>>         at
>> java.io.ObjectInputStream.readObject(ObjectInputStream.java:364)
>>         at
>> org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:40)
>>         at
>> org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:62)
>>         at
>> org.apache.spark.executor.Executor$TaskRunner$$anonfun$run$1.apply$mcV$sp(Executor.scala:195)
>>         at
>> org.apache.spark.deploy.SparkHadoopUtil.runAsUser(SparkHadoopUtil.scala:49)
>>         at
>> org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:178)
>>         at
>> java.util.concurrent.ThreadPoolExecutor$Worker.runTask(ThreadPoolExecutor.java:906)
>>         at
>> java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:929)
>>         at java.lang.Thread.run(Thread.java:796)
>> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 1 (task 0.0:1)
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 1]
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:1 as
>> TID 2 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:1 as
>> 1696 bytes in 0 ms
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
>> TID 3 on executor 0: hdtest023.svl.ibm.com (NODE_LOCAL)
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
>> 1696 bytes in 0 ms
>> 14/04/09 14:50:48 WARN scheduler.TaskSetManager: Lost TID 3 (task 0.0:0)
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 2]
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
>> TID 4 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
>> 14/04/09 14:50:48 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
>> 1696 bytes in 1 ms
>> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 4 (task 0.0:0)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 3]
>> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 2 (task 0.0:1)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 4]
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Starting task 0.0:1 as
>> TID 5 on executor 1: hdtest022.svl.ibm.com (NODE_LOCAL)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Serialized task 0.0:1 as
>> 1696 bytes in 1 ms
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Starting task 0.0:0 as
>> TID 6 on executor 0: hdtest023.svl.ibm.com (NODE_LOCAL)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Serialized task 0.0:0 as
>> 1696 bytes in 1 ms
>> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 5 (task 0.0:1)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 5]
>> 14/04/09 14:50:49 WARN scheduler.TaskSetManager: Lost TID 6 (task 0.0:0)
>> 14/04/09 14:50:49 INFO scheduler.TaskSetManager: Loss was due to
>> java.lang.ClassNotFoundException: SparkLinearRegression$$anonfun$2
>> [duplicate 6]
>> 14/04/09 14:50:49 ERROR scheduler.TaskSetManager: Task 0.0:0 failed 4
>> times; aborting job
>> 14/04/09 14:50:49 INFO scheduler.TaskSchedulerImpl: Remove TaskSet 0.0
>> from pool
>> 14/04/09 14:50:49 INFO scheduler.DAGScheduler: Failed to run collect at
>> SparkLinearRegression.scala:34
>> ^[[0m[^[[31merror^[[0m] ^[[0m(run-main) org.apache.spark.SparkException:
>> Job aborted: Task 0.0:0 failed 4 times (most recent failure: Exception
>> failure: java.lang.ClassNotFoundException:
>> SparkLinearRegression$$anonfun$2)^[[0m
>> org.apache.spark.SparkException: Job aborted: Task 0.0:0 failed 4 times
>> (most recent failure: Exception failure: java.lang.ClassNotFoundException:
>> SparkLinearRegression$$anonfun$2)
>>         at
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1028)
>>         at
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$org$apache$spark$scheduler$DAGScheduler$$abortStage$1.apply(DAGScheduler.scala:1026)
>>         at
>> scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
>>         at
>> scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47)
>>         at org.apache.spark.scheduler.DAGScheduler.org
>> $apache$spark$scheduler$DAGScheduler$$abortStage(DAGScheduler.scala:1026)
>>         at
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)
>>         at
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$processEvent$10.apply(DAGScheduler.scala:619)
>>         at scala.Option.foreach(Option.scala:236)
>>         at
>> org.apache.spark.scheduler.DAGScheduler.processEvent(DAGScheduler.scala:619)
>>         at
>> org.apache.spark.scheduler.DAGScheduler$$anonfun$start$1$$anon$2$$anonfun$receive$1.applyOrElse(DAGScheduler.scala:207)
>>         at akka.actor.ActorCell.receiveMessage(ActorCell.scala:498)
>>         at akka.actor.ActorCell.invoke(ActorCell.scala:456)
>>         at akka.dispatch.Mailbox.processMailbox(Mailbox.scala:237)
>>         at akka.dispatch.Mailbox.run(Mailbox.scala:219)
>>         at
>> akka.dispatch.ForkJoinExecutorConfigurator$AkkaForkJoinTask.exec(AbstractDispatcher.scala:386)
>>         at
>> scala.concurrent.forkjoin.ForkJoinTask.doExec(ForkJoinTask.java:260)
>>         at
>> scala.concurrent.forkjoin.ForkJoinPool$WorkQueue.runTask(ForkJoinPool.java:1339)
>>         at
>> scala.concurrent.forkjoin.ForkJoinPool.runWorker(ForkJoinPool.java:1979)
>>         at
>> scala.concurrent.forkjoin.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:107)
>> ^[[0m[^[[31mtrace^[[0m] ^[[0mStack trace suppressed: run ^[[34mlast
>> compile:run-main^[[0m for the full output.^[[0m
>> 14/04/09 14:50:49 INFO network.ConnectionManager: Selector thread was
>> interrupted!
>> java.lang.RuntimeException: Nonzero exit code: 1
>>         at scala.sys.package$.error(package.scala:27)
>>
>>
>

Reply via email to