[ https://issues.apache.org/jira/browse/SPARK-18300?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Emlyn Corrin updated SPARK-18300: --------------------------------- Component/s: SQL > ClassCastException during count distinct > ---------------------------------------- > > Key: SPARK-18300 > URL: https://issues.apache.org/jira/browse/SPARK-18300 > Project: Spark > Issue Type: Bug > Components: SQL > Affects Versions: 2.0.1 > Reporter: Emlyn Corrin > > While trying to reproduce SPARK-18172 in SQL, I found the following SQL query > fails in Spark 2.0.1 (via spark-beeline) with a CassCastException: > {code} > select count(distinct b), count(distinct b, c) from (select 1 as a, 2 as b, 3 > as c) group by a; > {code} > Selecting either of the two counts on their own runs fine, it is only when > both are selected in the same query that it fails. I also tried the same > query in pyspark: > {code} > spark.sql('select count(distinct b) as x, count(distinct b, c) as y from > (select 1 as a, 2 as b, 3 as c) group by a').show() > {code} > And get the same error, with a full stack trace: > {code} > : java.lang.ClassCastException: > org.apache.spark.sql.catalyst.expressions.Literal cannot be cast to > org.apache.spark.sql.catalyst.expressions.Attribute > at > org.apache.spark.sql.types.StructType$$anonfun$fromAttributes$1.apply(StructType.scala:388) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) > at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.AbstractTraversable.map(Traversable.scala:104) > at > org.apache.spark.sql.types.StructType$.fromAttributes(StructType.scala:388) > at > org.apache.spark.sql.catalyst.plans.QueryPlan.schema$lzycompute(QueryPlan.scala:242) > at > org.apache.spark.sql.catalyst.plans.QueryPlan.schema(QueryPlan.scala:242) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.supportCodegen(WholeStageCodegenExec.scala:454) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:482) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at > scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234) > at scala.collection.immutable.List.foreach(List.scala:381) > at scala.collection.TraversableLike$class.map(TraversableLike.scala:234) > at scala.collection.immutable.List.map(List.scala:285) > at > org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485) > at > org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:490) > at > org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:430) > at > org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93) > at > org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93) > at > scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124) > at scala.collection.immutable.List.foldLeft(List.scala:84) > at > org.apache.spark.sql.execution.QueryExecution.prepareForExecution(QueryExecution.scala:93) > at > org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:83) > at > org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:83) > at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2572) > at org.apache.spark.sql.Dataset.head(Dataset.scala:1934) > at org.apache.spark.sql.Dataset.take(Dataset.scala:2149) > at org.apache.spark.sql.Dataset.showString(Dataset.scala:239) > at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) > at > sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) > at > sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) > at java.lang.reflect.Method.invoke(Method.java:483) > at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237) > at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357) > at py4j.Gateway.invoke(Gateway.java:280) > at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) > at py4j.commands.CallCommand.execute(CallCommand.java:79) > at py4j.GatewayConnection.run(GatewayConnection.java:214) > at java.lang.Thread.run(Thread.java:745) > {code} -- This message was sent by Atlassian JIRA (v6.3.4#6332) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org