Hi,
I run into Task not Serializable excption with following code below. When I
remove the threads and run, it works, but with threads I run into Task not
serializable exception.
object SparkKart extends Serializable{
def parseVector(line: String): Vector[Double] = {
DenseVector(line.split(' ').map(_.toDouble))
}
def main(args: Array[String]) {
val sparkConf = new SparkConf().setAppName("SparkKart")
val sc = new SparkContext(sparkConf)
val lines = sc.textFile(args(0))
val data = lines.map(parseVector _)
val kPoints = data.takeSample(withReplacement = false, 4, 42).toArray
val thread1= new Thread (new Runnable {
def run() {
val dist1 = data.map(squaredDistance(_ , kPoints(0)))
dist1.saveAsTextFile("hdfs:/kart3")
}
})
val thread2= new Thread (new Runnable {
def run() {
val dist1 =data.map(squaredDistance(_, kPoints(1)))
dist1.saveAsTextFile("hdfs:/kart2")
}
})
val thread3= new Thread (new Runnable {
def run() {
val dist1 =data.map(squaredDistance(_, kPoints(2)))
dist1.saveAsTextFile("hdfs:/kart1")
}
})
thread1.start
thread2.start
thread3.start
}
}
Any help please?
--
View this message in context:
http://apache-spark-user-list.1001560.n3.nabble.com/Task-not-serializable-exception-tp21795.html
Sent from the Apache Spark User List mailing list archive at Nabble.com.