Github user ueshin commented on a diff in the pull request:
https://github.com/apache/spark/pull/20211#discussion_r161146793
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/RelationalGroupedDataset.scala ---
@@ -457,13 +458,26 @@ class RelationalGroupedDataset protected[sql](
val groupingNamedExpressions = groupingExprs.map {
case ne: NamedExpression => ne
- case other => Alias(other, other.toString)()
+ case other => Alias(other, toPrettySQL(other))()
}
val groupingAttributes = groupingNamedExpressions.map(_.toAttribute)
val child = df.logicalPlan
val project = Project(groupingNamedExpressions ++ child.output, child)
- val output = expr.dataType.asInstanceOf[StructType].toAttributes
- val plan = FlatMapGroupsInPandas(groupingAttributes, expr, output,
project)
+ val udfOutput: Seq[Attribute] =
expr.dataType.asInstanceOf[StructType].toAttributes
+ val additionalGroupingAttributes = mutable.ArrayBuffer[Attribute]()
+
+ for (attribute <- groupingAttributes) {
+ if (!udfOutput.map(_.name).contains(attribute.name)) {
--- End diff --
Maybe this relates to the discussion above
(https://github.com/apache/spark/pull/20211#discussion_r160524679).
Let's wait and see for now.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
