Hello Folks,
I am trying to chain a couple of map operations and it seems the second map
fails with a mismatch in arguments(event though the compiler prints them to
be the same.) I checked the function and variable types using :t and they
look ok to me.
Have you seen this earlier? I am posting the code, data and output below.
Any pointers will be greatly appreciated.
Thanks,
Boromir.
/////////////////////// SCRIPT
val data = sc.textFile("data/testpv.csv")
case class KVV(key: String, valvec: Array[Double])
def mapToKV(line: String) : KVV = {
val splits = line.split(",")
val key = splits(0).trim
val valvec = splits.drop(1).map(x => x.trim.toDouble)
val kvv = KVV(key, valvec)
return kvv
}
val kvs = data.map(line => mapToKV(line))
def mapKVtoKVL(kvv: KVV) : KVV = {
return kvv
}
val tvar = kvs.map(x => mapKVtoKVL(x))
/////////////////////// SAMPLE DATA in testpv.csv
x,1.1,1.2,1.3
y,2.1,2.2,2.3
/////////////////////// REPL OUTPUT
scala> val data = sc.textFile("data/testpv.csv")
14/09/15 10:53:23 INFO MemoryStore: ensureFreeSpace(146579) called with
curMem=0, maxMem=308713881
14/09/15 10:53:23 INFO MemoryStore: Block broadcast_0 stored as values to
memory (estimated size 143.1 KB, free 294.3 MB)
data: org.apache.spark.rdd.RDD[String] = MappedRDD[1] at textFile at
<console>:12
scala> case class KVV(key: String, valvec: Array[Double])
defined class KVV
scala>
scala> def mapToKV(line: String) : KVV = {
| val splits = line.split(",")
| val key = splits(0).trim
| val valvec = splits.drop(1).map(x => x.trim.toDouble)
|
| val kvv = KVV(key, valvec)
| return kvv
| }
mapToKV: (line: String)KVV
scala> val kvs = data.map(line => mapToKV(line))
kvs: org.apache.spark.rdd.RDD[KVV] = MappedRDD[2] at map at <console>:18
scala>
scala> def mapKVtoKVL(kvv: KVV) : KVV = {
| return kvv
| }
mapKVtoKVL: (kvv: KVV)KVV
scala> val tvar = kvs.map(x => mapKVtoKVL(x))
<console>:22: error: type mismatch;
found : KVV
required: KVV
val tvar = kvs.map(x => mapKVtoKVL(x))
^
