Emlyn Corrin created SPARK-18300:
------------------------------------
Summary: ClassCastException during count distinct
Key: SPARK-18300
URL: https://issues.apache.org/jira/browse/SPARK-18300
Project: Spark
Issue Type: Bug
Affects Versions: 2.0.1
Reporter: Emlyn Corrin
While trying to reproduce SPARK-18172 in SQL, I found the following SQL query
fails in Spark 2.0.1 (via spark-beeline) with a CassCastException:
{code}
select count(distinct b), count(distinct b, c) from (select 1 as a, 2 as b, 3
as c) group by a;
{code}
Selecting either of the two counts on their own runs fine, it is only when both
are selected in the same query that it fails. I also tried the same query in
pyspark:
{code}
spark.sql('select count(distinct b) as x, count(distinct b, c) as y from
(select 1 as a, 2 as b, 3 as c) group by a').show()
{code}
And get the same error, with a full stack trace:
{code}
: java.lang.ClassCastException:
org.apache.spark.sql.catalyst.expressions.Literal cannot be cast to
org.apache.spark.sql.catalyst.expressions.Attribute
at
org.apache.spark.sql.types.StructType$$anonfun$fromAttributes$1.apply(StructType.scala:388)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.AbstractTraversable.map(Traversable.scala:104)
at
org.apache.spark.sql.types.StructType$.fromAttributes(StructType.scala:388)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.schema$lzycompute(QueryPlan.scala:242)
at
org.apache.spark.sql.catalyst.plans.QueryPlan.schema(QueryPlan.scala:242)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$10.apply(WholeStageCodegenExec.scala:454)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.supportCodegen(WholeStageCodegenExec.scala:454)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:482)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:469)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter$1.apply(WholeStageCodegenExec.scala:471)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertInputAdapter(WholeStageCodegenExec.scala:471)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:483)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages$$anonfun$org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen$2.apply(WholeStageCodegenExec.scala:485)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at
scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:234)
at scala.collection.immutable.List.foreach(List.scala:381)
at scala.collection.TraversableLike$class.map(TraversableLike.scala:234)
at scala.collection.immutable.List.map(List.scala:285)
at
org.apache.spark.sql.execution.CollapseCodegenStages.org$apache$spark$sql$execution$CollapseCodegenStages$$insertWholeStageCodegen(WholeStageCodegenExec.scala:485)
at
org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:490)
at
org.apache.spark.sql.execution.CollapseCodegenStages.apply(WholeStageCodegenExec.scala:430)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93)
at
org.apache.spark.sql.execution.QueryExecution$$anonfun$prepareForExecution$1.apply(QueryExecution.scala:93)
at
scala.collection.LinearSeqOptimized$class.foldLeft(LinearSeqOptimized.scala:124)
at scala.collection.immutable.List.foldLeft(List.scala:84)
at
org.apache.spark.sql.execution.QueryExecution.prepareForExecution(QueryExecution.scala:93)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan$lzycompute(QueryExecution.scala:83)
at
org.apache.spark.sql.execution.QueryExecution.executedPlan(QueryExecution.scala:83)
at org.apache.spark.sql.Dataset.withTypedCallback(Dataset.scala:2572)
at org.apache.spark.sql.Dataset.head(Dataset.scala:1934)
at org.apache.spark.sql.Dataset.take(Dataset.scala:2149)
at org.apache.spark.sql.Dataset.showString(Dataset.scala:239)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at
sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at
sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:483)
at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:237)
at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
at py4j.Gateway.invoke(Gateway.java:280)
at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
at py4j.commands.CallCommand.execute(CallCommand.java:79)
at py4j.GatewayConnection.run(GatewayConnection.java:214)
at java.lang.Thread.run(Thread.java:745)
{code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
