[ https://issues.apache.org/jira/browse/SPARK-40288?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
hgs updated SPARK-40288: ------------------------ Description: {{--table}} {{create}} {{table}} {{miss_expr(id }}{{{}int{}}}{{{},{}}}{{{}name{}}} {{string,age }}{{{}double{}}}{{{}) stored {}}}{{as}} {{textfile}} {{--data}} {{insert}} {{overwrite }}{{table}} {{miss_expr }}{{{}values{}}}{{{}(1,{}}}{{{}'ox'{}}}{{{},1.0),(1,{}}}{{{}'oox'{}}}{{{},2.0),(2,{}}}{{{}'ox'{}}}{{{},3.0),(2,{}}}{{{}'xxo'{}}}{{{},4.0){}}} {{--failure sql}} {{insert}} {{overwrite }}{{table}} {{miss_expr}} {{select}} {{{}id,{}}}{{{}name{}}}{{{},nage {}}}{{as}} {{nnnnn }}{{{}from{}}}{{{}({}}} {{select}} {{{}id,{}}}{{{}name{}}}{{{},if(age>3,100,200) {}}}{{as}} {{nage }}{{from}} {{miss_expr }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},age{}}} {{) }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},nage{}}} --error stack {{Caused by: java.lang.IllegalStateException: Couldn't find age#4 in [id#2,name#3,if ((age#4 > 3.0)) 100 else 200#12|#2,name#3,if ((age#4 > 3.0)) 100 else 200#12]}} {{at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:80)}} {{at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:73)}} {{at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481)}} was: {{--table}} {{create}} {{table}} {{miss_expr(id }}{{{}int{}}}{{{},{}}}{{{}name{}}} {{string,age }}{{{}double{}}}{{{}) stored {}}}{{as}} {{textfile}} {{--data}} {{insert}} {{overwrite }}{{table}} {{miss_expr }}{{{}values{}}}{{{}(1,{}}}{{{}'ox'{}}}{{{},1.0),(1,{}}}{{{}'oox'{}}}{{{},2.0),(2,{}}}{{{}'ox'{}}}{{{},3.0),(2,{}}}{{{}'xxo'{}}}{{{},4.0){}}} {{--failure sql}} {{insert}} {{overwrite }}{{table}} {{miss_expr}} {{select}} {{{}id,{}}}{{{}name{}}}{{{},nage {}}}{{as}} {{nnnnn }}{{{}from{}}}{{{}({}}} {{select}} {{{}id,{}}}{{{}name{}}}{{{},if(age>3,100,200) {}}}{{as}} {{nage }}{{from}} {{miss_expr }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},age{}}} {{) }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},nage{}}} {{}} --error stack {{Caused by: java.lang.IllegalStateException: Couldn't find age#4 in [id#2,name#3,if ((age#4 > 3.0)) 100 else 200#12]}} {{at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:80)}} {{at org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:73)}} {{at org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481)}} > After `RemoveRedundantAggregates`, `PullOutGroupingExpressions` should > applied to avoid attribute missing when use complex expression. > -------------------------------------------------------------------------------------------------------------------------------------- > > Key: SPARK-40288 > URL: https://issues.apache.org/jira/browse/SPARK-40288 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 3.0.0, 3.2.0, 3.3.0 > Environment: spark 3.2.0 spark 3.2.2 spark 3.3.0 > Reporter: hgs > Priority: Minor > > {{--table}} > {{create}} {{table}} {{miss_expr(id }}{{{}int{}}}{{{},{}}}{{{}name{}}} > {{string,age }}{{{}double{}}}{{{}) stored {}}}{{as}} {{textfile}} > {{--data}} > {{insert}} {{overwrite }}{{table}} {{miss_expr > }}{{{}values{}}}{{{}(1,{}}}{{{}'ox'{}}}{{{},1.0),(1,{}}}{{{}'oox'{}}}{{{},2.0),(2,{}}}{{{}'ox'{}}}{{{},3.0),(2,{}}}{{{}'xxo'{}}}{{{},4.0){}}} > {{--failure sql}} > {{insert}} {{overwrite }}{{table}} {{miss_expr}} > {{select}} {{{}id,{}}}{{{}name{}}}{{{},nage {}}}{{as}} {{nnnnn > }}{{{}from{}}}{{{}({}}} > {{select}} {{{}id,{}}}{{{}name{}}}{{{},if(age>3,100,200) {}}}{{as}} {{nage > }}{{from}} {{miss_expr }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},age{}}} > {{) }}{{group}} {{by}} {{{}id,{}}}{{{}name{}}}{{{},nage{}}} > --error stack > {{Caused by: java.lang.IllegalStateException: Couldn't find age#4 in > [id#2,name#3,if ((age#4 > 3.0)) 100 else 200#12|#2,name#3,if ((age#4 > 3.0)) > 100 else 200#12]}} > {{at > org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:80)}} > {{at > org.apache.spark.sql.catalyst.expressions.BindReferences$$anonfun$bindReference$1.applyOrElse(BoundAttribute.scala:73)}} > {{at > org.apache.spark.sql.catalyst.trees.TreeNode.$anonfun$transformDownWithPruning$1(TreeNode.scala:481)}} -- This message was sent by Atlassian Jira (v8.20.10#820010) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org