Hi, Spark-version : 1.4.1 Runing the code getting the following error, how can I fix the code and run collectly ? I don’t know why the schema don’t support this type system. If I use callUDF instead of udf. Everything is good.
Thanks, Minglei. val index:(String => (String => Int)) = (value:String) => { (a:String) => if ( value.equals(a)) 1 else 0 } val sqlfunc = udf(index) var temp = df val meetsConditionValue = List("fergubo01m" ,"wrighha01m" ,"woodji01m" ,"mcbridi01m" ,"cravebi01m") for (i <- 0 until j) { temp = temp.withColumn(columnName + "_" + meetsConditionValue(i), sqlfunc(col(columnName))) } Exception in thread "main" java.lang.UnsupportedOperationException: Schema for type String => Int is not supported at org.apache.spark.sql.catalyst.ScalaReflection$class.schemaFor(ScalaReflection.scala:152) at org.apache.spark.sql.catalyst.ScalaReflection$.schemaFor(ScalaReflection.scala:28) at org.apache.spark.sql.catalyst.ScalaReflection$class.schemaFor(ScalaReflection.scala:63) at org.apache.spark.sql.catalyst.ScalaReflection$.schemaFor(ScalaReflection.scala:28) at org.apache.spark.sql.functions$.udf(functions.scala:1363) at com.asa.ml.toolimpl.DummyImpl.create_dummy(DummyImpl.scala:60) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at com.asa.ml.client.Client$.main(Client.scala:26) at com.asa.ml.client.Client.main(Client.scala) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:606) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:665) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:170) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:193) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:112) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)