hi all,

my sbt file is like this:

name := "Spark"

version := "1.0"

scalaVersion := "2.10.4"

libraryDependencies += "org.apache.spark" %% "spark-core" % "1.1.0"

libraryDependencies += "net.sf.opencsv" % "opencsv" % "2.3"


*code:*

object SparkJob
{

  def pLines(lines:Iterator[String])={
    val parser=new CSVParser()
    lines.map(l=>{val vs=parser.parseLine(l)
      (vs(0),vs(1).toInt)})
  }

  def main(args: Array[String]) {
    val conf = new SparkConf().setAppName("Spark Job").setMaster("local")
    val sc = new SparkContext(conf)
    val data = sc.textFile("/home/amit/testData.csv").cache()
    val result = data.mapPartitions(pLines).groupByKey
    //val list = result.filter(x=> {(x._1).contains("24050881")})

  }

}


Here groupByKey is not working . But same thing is working from *spark-shell.*

Please help me


Thanks

Amit

Reply via email to