Ravi Kaushik created SPARK-26562:
------------------------------------
Summary: countDistinct and user-defined function cannot be used in
SELECT
Key: SPARK-26562
URL: https://issues.apache.org/jira/browse/SPARK-26562
Project: Spark
Issue Type: Bug
Components: PySpark
Affects Versions: 2.3.1
Environment: Macbook Pro 10.14.2
spark 2.3.1
Reporter: Ravi Kaushik
df=spark.createDataFrame([ [1,2,3], [2,3,4], [4,5,6] ], ['a', 'b', 'c'])
from pyspark.sql import functions as F, types as T
df.select(F.sum(df['a']), F.count(df['a']), F.countDistinct(df['a']),
F.approx_count_distinct(df['a'])).show()
func =F.udf(lambda x: 1.0, T.DoubelType())
df.select(F.sum(df['a']), F.count(df['a']), F.countDistinct(df['a']),
F.approx_count_distinct(df['a']), F.sum(func(df['a'])) ).show()
Error
2019-01-07 18:30:50 ERROR Executor:91 - Exception in task 6.0 in stage 4.0 (TID
223)
org.apache.spark.sql.catalyst.errors.package$TreeNodeException: Binding
attribute, tree: sum#45849
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:56)
at
org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:91)
at
org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:90)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$2.apply(TreeNode.scala:267)
at
org.apache.spark.sql.catalyst.trees.CurrentOrigin$.withOrigin(TreeNode.scala:70)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:266)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:306)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$transformDown$1.apply(TreeNode.scala:272)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4$$anonfun$apply$11.apply(TreeNode.scala:335)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.catalyst.trees.TreeNode$$anonfun$4.apply(TreeNode.scala:333)
at
org.apache.spark.sql.catalyst.trees.TreeNode.mapProductIterator(TreeNode.scala:187)
at org.apache.spark.sql.catalyst.trees.TreeNode.mapChildren(TreeNode.scala:304)
at
org.apache.spark.sql.catalyst.trees.TreeNode.transformDown(TreeNode.scala:272)
at org.apache.spark.sql.catalyst.trees.TreeNode.transform(TreeNode.scala:256)
at
org.apache.spark.sql.catalyst.expressions.BindReferences$.bindReference(BoundAttribute.scala:90)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$$anonfun$bind$1.apply(GenerateMutableProjection.scala:38)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$$anonfun$bind$1.apply(GenerateMutableProjection.scala:38)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.bind(GenerateMutableProjection.scala:38)
at
org.apache.spark.sql.catalyst.expressions.codegen.GenerateMutableProjection$.generate(GenerateMutableProjection.scala:44)
at
org.apache.spark.sql.execution.SparkPlan.newMutableProjection(SparkPlan.scala:383)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4$$anonfun$5.apply(HashAggregateExec.scala:119)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4$$anonfun$5.apply(HashAggregateExec.scala:118)
at
org.apache.spark.sql.execution.aggregate.AggregationIterator.generateProcessRow(AggregationIterator.scala:180)
at
org.apache.spark.sql.execution.aggregate.AggregationIterator.<init>(AggregationIterator.scala:199)
at
org.apache.spark.sql.execution.aggregate.TungstenAggregationIterator.<init>(TungstenAggregationIterator.scala:97)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4.apply(HashAggregateExec.scala:111)
at
org.apache.spark.sql.execution.aggregate.HashAggregateExec$$anonfun$doExecute$1$$anonfun$4.apply(HashAggregateExec.scala:101)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndex$1$$anonfun$apply$26.apply(RDD.scala:847)
at
org.apache.spark.rdd.RDD$$anonfun$mapPartitionsWithIndex$1$$anonfun$apply$26.apply(RDD.scala:847)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:96)
at org.apache.spark.scheduler.ShuffleMapTask.runTask(ShuffleMapTask.scala:53)
at org.apache.spark.scheduler.Task.run(Task.scala:109)
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
at
java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at
java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.RuntimeException: Couldn't find sum#45849 in
[sum#43751L,count#43753L,MS[0]#45328L,MS[1]#45329L,MS[2]#45330L,MS[3]#45331L,MS[4]#45332L,MS[5]#45333L,MS[6]#45334L,MS[7]#45335L,MS[8]#45336L,MS[9]#45337L,MS[10]#45338L,MS[11]#45339L,MS[12]#45340L,MS[13]#45341L,MS[14]#45342L,MS[15]#45343L,MS[16]#45344L,MS[17]#45345L,MS[18]#45346L,MS[19]#45347L,MS[20]#45348L,MS[21]#45349L,MS[22]#45350L,MS[23]#45351L,MS[24]#45352L,MS[25]#45353L,MS[26]#45354L,MS[27]#45355L,MS[28]#45356L,MS[29]#45357L,MS[30]#45358L,MS[31]#45359L,MS[32]#45360L,MS[33]#45361L,MS[34]#45362L,MS[35]#45363L,MS[36]#45364L,MS[37]#45365L,MS[38]#45366L,MS[39]#45367L,MS[40]#45368L,MS[41]#45369L,MS[42]#45370L,MS[43]#45371L,MS[44]#45372L,MS[45]#45373L,MS[46]#45374L,MS[47]#45375L,MS[48]#45376L,MS[49]#45377L,MS[50]#45378L,MS[51]#45379L,sum#43766,count#43758L,a#0L,sum#43752L,count#43754L,MS[0]#43463L,MS[1]#43464L,MS[2]#43465L,MS[3]#43466L,MS[4]#43467L,MS[5]#43468L,MS[6]#43469L,MS[7]#43470L,MS[8]#43471L,MS[9]#43472L,MS[10]#43473L,MS[11]#43474L,MS[12]#43475L,MS[13]#43476L,MS[14]#43477L,MS[15]#43478L,MS[16]#43479L,MS[17]#43480L,MS[18]#43481L,MS[19]#43482L,MS[20]#43483L,MS[21]#43484L,MS[22]#43485L,MS[23]#43486L,MS[24]#43487L,MS[25]#43488L,MS[26]#43489L,MS[27]#43490L,MS[28]#43491L,MS[29]#43492L,MS[30]#43493L,MS[31]#43494L,MS[32]#43495L,MS[33]#43496L,MS[34]#43497L,MS[35]#43498L,MS[36]#43499L,MS[37]#43500L,MS[38]#43501L,MS[39]#43502L,MS[40]#43503L,MS[41]#43504L,MS[42]#43505L,MS[43]#43506L,MS[44]#43507L,MS[45]#43508L,MS[46]#43509L,MS[47]#43510L,MS[48]#43511L,MS[49]#43512L,MS[50]#43513L,MS[51]#43514L,sum#43756,pythonUDF0#43765]
at scala.sys.package$.error(package.scala:27)
at
org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:97)
at
org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1$$anonfun$applyOrElse$1.apply(BoundAttribute.scala:91)
at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:52)
... 59 more
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
