This is just the stacktrace,but where is it you ccalling the UDF? Regards, Sumit
On 16-Aug-2016 2:20 pm, "pseudo oduesp" <pseudo20...@gmail.com> wrote: > hi, > i cretae new columns with udf after i try to filter this columns : > i get this error why ? > > : java.lang.UnsupportedOperationException: Cannot evaluate expression: > fun_nm(input[0, string, true]) > at org.apache.spark.sql.catalyst.expressions.Unevaluable$class. > eval(Expression.scala:221) > at org.apache.spark.sql.execution.python.PythonUDF. > eval(PythonUDF.scala:27) > at org.apache.spark.sql.catalyst.expressions.BinaryExpression. > eval(Expression.scala:408) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$. > org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$ > canFilterOutNull(Optimizer.scala:1234) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$ > anonfun$55.apply(Optimizer.scala:1248) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$ > anonfun$55.apply(Optimizer.scala:1248) > at scala.collection.LinearSeqOptimized$class. > exists(LinearSeqOptimized.scala:93) > at scala.collection.immutable.List.exists(List.scala:84) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$. > org$apache$spark$sql$catalyst$optimizer$EliminateOuterJoin$$ > buildNewJoinType(Optimizer.scala:1248) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$ > anonfun$apply$30.applyOrElse(Optimizer.scala:1264) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$$ > anonfun$apply$30.applyOrElse(Optimizer.scala:1262) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3. > apply(TreeNode.scala:279) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$3. > apply(TreeNode.scala:279) > at org.apache.spark.sql.catalyst.trees.CurrentOrigin$. > withOrigin(TreeNode.scala:69) > at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown( > TreeNode.scala:278) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5. > apply(TreeNode.scala:321) > at org.apache.spark.sql.catalyst.trees.TreeNode. > mapProductIterator(TreeNode.scala:179) > at org.apache.spark.sql.catalyst.trees.TreeNode. > transformChildren(TreeNode.scala:319) > at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown( > TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5. > apply(TreeNode.scala:321) > at org.apache.spark.sql.catalyst.trees.TreeNode. > mapProductIterator(TreeNode.scala:179) > at org.apache.spark.sql.catalyst.trees.TreeNode. > transformChildren(TreeNode.scala:319) > at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown( > TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$ > transformDown$1.apply(TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$5. > apply(TreeNode.scala:321) > at org.apache.spark.sql.catalyst.trees.TreeNode. > mapProductIterator(TreeNode.scala:179) > at org.apache.spark.sql.catalyst.trees.TreeNode. > transformChildren(TreeNode.scala:319) > at org.apache.spark.sql.catalyst.trees.TreeNode.transformDown( > TreeNode.scala:284) > at org.apache.spark.sql.catalyst.trees.TreeNode.transform( > TreeNode.scala:268) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$. > apply(Optimizer.scala:1262) > at org.apache.spark.sql.catalyst.optimizer.EliminateOuterJoin$. > apply(Optimizer.scala:1225) > at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$ > execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:85) > at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$ > execute$1$$anonfun$apply$1.apply(RuleExecutor.scala:82) > at scala.collection.IndexedSeqOptimized$class. > foldl(IndexedSeqOptimized.scala:57) > at scala.collection.IndexedSeqOptimized$class. > foldLeft(IndexedSeqOptimized.scala:66) > at scala.collection.mutable.WrappedArray.foldLeft( > WrappedArray.scala:35) > at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$ > execute$1.apply(RuleExecutor.scala:82) > at org.apache.spark.sql.catalyst.rules.RuleExecutor$$anonfun$ > execute$1.apply(RuleExecutor.scala:74) > at scala.collection.immutable.List.foreach(List.scala:381) > at org.apache.spark.sql.catalyst.rules.RuleExecutor.execute( > RuleExecutor.scala:74) > at org.apache.spark.sql.execution.QueryExecution. > optimizedPlan$lzycompute(QueryExecution.scala:74) > at org.apache.spark.sql.execution.QueryExecution. > optimizedPlan(QueryExecution.scala:74) > at org.apache.spark.sql.execution.QueryExecution.sparkPlan$lzycompute( > QueryExecution.scala:78) > at org.apache.spark.sql.execution.QueryExecution. > sparkPlan(QueryExecution.scala:76) > at org.apache.spark.sql.execution.QueryExecution. > executedPlan$lzycompute(QueryExecution.scala:83) > at org.apache.spark.sql.execution.QueryExecution. > executedPlan(QueryExecution.scala:83) > at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2558) > at org.apache.spark.sql.Dataset.head(Dataset.scala:1924) > at org.apache.spark.sql.Dataset.take(Dataset.scala:2139) > at org.apache.spark.sql.Dataset.showString(Dataset.scala:239) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at sun.reflect.NativeMethodAccessorImpl.invoke( > NativeMethodAccessorImpl.java:57) > at sun.reflect.DelegatingMethodAccessorImpl.invoke( > DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:606) > at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237) > at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) > at py4j.Gateway.invoke(Gateway.java:280) > at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand. > java:128) > at py4j.commands.CallCommand.execute(CallCommand.java:79) > at py4j.GatewayConnection.run(GatewayConnection.java:211) > at java.lang.Thread.run(Thread.java:745) > >
