[
https://issues.apache.org/jira/browse/SPARK-32906?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17197373#comment-17197373
]
Apache Spark commented on SPARK-32906:
--------------------------------------
User 'maropu' has created a pull request for this issue:
https://github.com/apache/spark/pull/29780
> Struct field names should not change after normalizing floats
> -------------------------------------------------------------
>
> Key: SPARK-32906
> URL: https://issues.apache.org/jira/browse/SPARK-32906
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 3.0.2, 3.1.0
> Reporter: Takeshi Yamamuro
> Priority: Minor
>
> This ticket aims at fixing a minor bug when normalizing floats for struct
> types;
> {code}
> scala> import org.apache.spark.sql.execution.aggregate.HashAggregateExec
> scala> val df = Seq(Tuple1(Tuple1(-0.0d)), Tuple1(Tuple1(0.0d))).toDF("k")
> scala> val agg = df.distinct()
> scala> agg.explain()
> == Physical Plan ==
> *(2) HashAggregate(keys=[k#40], functions=[])
> +- Exchange hashpartitioning(k#40, 200), true, [id=#62]
> +- *(1) HashAggregate(keys=[knownfloatingpointnormalized(if (isnull(k#40))
> null else named_struct(col1,
> knownfloatingpointnormalized(normalizenanandzero(k#40._1)))) AS k#40],
> functions=[])
> +- *(1) LocalTableScan [k#40]
> scala> val aggOutput = agg.queryExecution.sparkPlan.collect { case a:
> HashAggregateExec => a.output.head }
> scala> aggOutput.foreach { attr => println(attr.prettyJson) }
> ### Final Aggregate ###
> [ {
> "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference",
> "num-children" : 0,
> "name" : "k",
> "dataType" : {
> "type" : "struct",
> "fields" : [ {
> "name" : "_1",
> ^^^
> "type" : "double",
> "nullable" : false,
> "metadata" : { }
> } ]
> },
> "nullable" : true,
> "metadata" : { },
> "exprId" : {
> "product-class" : "org.apache.spark.sql.catalyst.expressions.ExprId",
> "id" : 40,
> "jvmId" : "a824e83f-933e-4b85-a1ff-577b5a0e2366"
> },
> "qualifier" : [ ]
> } ]
> ### Partial Aggregate ###
> [ {
> "class" : "org.apache.spark.sql.catalyst.expressions.AttributeReference",
> "num-children" : 0,
> "name" : "k",
> "dataType" : {
> "type" : "struct",
> "fields" : [ {
> "name" : "col1",
> ^^^^
> "type" : "double",
> "nullable" : true,
> "metadata" : { }
> } ]
> },
> "nullable" : true,
> "metadata" : { },
> "exprId" : {
> "product-class" : "org.apache.spark.sql.catalyst.expressions.ExprId",
> "id" : 40,
> "jvmId" : "a824e83f-933e-4b85-a1ff-577b5a0e2366"
> },
> "qualifier" : [ ]
> } ]
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
