Hi, The stack trace suggests you're doing a join as well? and it's python..
I wonder if you're seeing this? https://issues.apache.org/jira/browse/SPARK-17100 Are you using spark 2.0.0? Tim On Tue, 16 Aug 2016 at 16:58 Sumit Khanna <sumit.kha...@askme.in> wrote: > This is just the stacktrace,but where is it you ccalling the UDF? > > Regards, > Sumit > > On 16-Aug-2016 2:20 pm, "pseudo oduesp" <pseudo20...@gmail.com> wrote: > >> hi, >> i cretae new columns with udf after i try to filter this columns : >> i get this error why ? >> >> : java.lang.UnsupportedOperationException: Cannot evaluate expression: >> fun_nm(input[0, string, true]) >> at >> org.apache.spark.sql.catalyst.expressions.Unevaluable$class.eval(Expression.scala:221) >> at >> org.apache.spark.sql.execution.python.PythonUDF.eval(PythonUDF.scala:27) >> at >> org.apache.spark.sql.catalyst.expressions.BinaryExpression.eval(Expression.scala:408) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$canFilterOutNull(Optimizer.scala:1234) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$anonfun$55.apply(Optimizer.scala:1248) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$anonfun$55.apply(Optimizer.scala:1248) >> at >> scala.collection.LinearSeqOptimized$class.exists(LinearSeqOptimized.scala:93) >> at scala.collection.immutable.List.exists(List.scala:84) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$buildNewJoinType(Optimizer.scala:1248) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$anonfun$apply$30.applyOrElse(Optimizer.scala:1264) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$anonfun$apply$30.applyOrElse(Optimizer.scala:1262) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:279) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3.apply(TreeNode.scala:279) >> at >> org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:69) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:278) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:321) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:319) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:321) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:319) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5.apply(TreeNode.scala:321) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:179) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformChildren(TreeNode.scala:319) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:284) >> at >> org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:268) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.apply(Optimizer.scala:1262) >> at >> org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$.apply(Optimizer.scala:1225) >> at >> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85) >> at >> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82) >> at >> scala.collection.IndexedSeqOptimized$class.foldl(IndexedSeqOptimized.scala:57) >> at >> scala.collection.IndexedSeqOptimized$class.foldLeft(IndexedSeqOptimized.scala:66) >> at >> scala.collection.mutable.WrappedArray.foldLeft(WrappedArray.scala:35) >> at >> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:82) >> at >> org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$execute$1.apply(RuleExecutor.scala:74) >> at scala.collection.immutable.List.foreach(List.scala:381) >> at >> org.apache.spark.sql.catalyst.rules.RuleExecutor.execute(RuleExecutor.scala:74) >> at >> org.apache.spark.sql.execution.QueryExecution.optimizedPlan$lzycompute(QueryExecution.scala:74) >> at >> org.apache.spark.sql.execution.QueryExecution.optimizedPlan(QueryExecution.scala:74) >> at >> org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute(QueryExecution.scala:78) >> at >> org.apache.spark.sql.execution.QueryExecution.sparkPlan(QueryExecution.scala:76) >> at >> org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:83) >> at >> org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:83) >> at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2558) >> at org.apache.spark.sql.Dataset.head(Dataset.scala:1924) >> at org.apache.spark.sql.Dataset.take(Dataset.scala:2139) >> at org.apache.spark.sql.Dataset.showString(Dataset.scala:239) >> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) >> at >> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57) >> at >> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) >> at java.lang.reflect.Method.invoke(Method.java:606) >> at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237) >> at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) >> at py4j.Gateway.invoke(Gateway.java:280) >> at >> py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:128) >> at py4j.commands.CallCommand.execute(CallCommand.java:79) >> at py4j.GatewayConnection.run(GatewayConnection.java:211) >> at java.lang.Thread.run(Thread.java:745) >> >>
