This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 6639cbcdd3 [GLUTEN-13178][VL] Re-enable the Spark 4.2 broadcast, hash
aggregate, acosh and XML TIME array tests (#13431)
6639cbcdd3 is described below
commit 6639cbcdd33f540956f6c227f8c949bcdfd7654e
Author: Akshay Tayal <[email protected]>
AuthorDate: Mon Oct 26 14:54:41 2026 -0700
[GLUTEN-13178][VL] Re-enable the Spark 4.2 broadcast, hash aggregate, acosh
and XML TIME array tests (#13431)
---
.../org/apache/spark/sql/GlutenSQLTestsTrait.scala | 6 ++-
.../org/apache/spark/sql/GlutenTestsTrait.scala | 4 +-
.../gluten/utils/velox/VeloxTestSettings.scala | 16 ++------
.../expressions/GlutenXmlExpressionsSuite.scala | 43 +++++++++++++++++++++-
.../GlutenReplaceHashWithSortAggSuite.scala | 5 ++-
5 files changed, 58 insertions(+), 16 deletions(-)
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsTrait.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsTrait.scala
index ef0b46c69f..e2102ce912 100644
---
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsTrait.scala
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenSQLTestsTrait.scala
@@ -241,13 +241,17 @@ object GlutenQueryTestUtil extends Assertions {
case (a: Row, b: Row) =>
compare(a.toSeq, b.toSeq)
// 0.0 == -0.0, turn float/double to bits before comparison, to
distinguish 0.0 and -0.0.
+ // NaN can be represented with different bits (e.g. by native code), so
first check for it.
case (a: Double, b: Double) =>
- if ((isNaNOrInf(a) || isNaNOrInf(b)) || (a == -0.0) || (b == -0.0)) {
+ if (a.isNaN && b.isNaN) {
+ true
+ } else if ((isNaNOrInf(a) || isNaNOrInf(b)) || (a == -0.0) || (b ==
-0.0)) {
java.lang.Double.doubleToRawLongBits(a) ==
java.lang.Double.doubleToRawLongBits(b)
} else {
Precision.equalsWithRelativeTolerance(a, b, 0.00001d)
}
case (a: Float, b: Float) =>
+ a.isNaN && b.isNaN ||
java.lang.Float.floatToRawIntBits(a) ==
java.lang.Float.floatToRawIntBits(b)
case (a, b) => a == b
}
diff --git
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
index cae6fe4147..edfb0ce93e 100644
---
a/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
+++
b/gluten-ut/common/src/test/scala/org/apache/spark/sql/GlutenTestsTrait.scala
@@ -245,7 +245,9 @@ trait GlutenTestsTrait extends GlutenTestsCommonTrait {
ArrayType(vt, vcn),
exprNullable = false)
case (result: Double, expected: Double) =>
- if (
+ if (result.isNaN && expected.isNaN) {
+ true
+ } else if (
(isNaNOrInf(result) || isNaNOrInf(expected))
|| (result == -0.0) || (expected == -0.0)
) {
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 18dda9c179..156d730611 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -228,9 +228,6 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenMathExpressionsSuite]
// Spark round UT for round(3.1415,3) is not correct.
.exclude("round/bround/floor/ceil")
- .exclude(
- "SPARK-56089: asinh/acosh fdlibm algorithm coverage"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenMiscExpressionsSuite]
enableSuite[GlutenNondeterministicSuite]
.exclude("MonotonicallyIncreasingID")
@@ -296,8 +293,10 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenValidateExternalTypeSuite]
enableSuite[GlutenXmlExpressionsSuite]
.exclude("from_xml- invalid data")
- // Generated suites for org.apache.spark.sql.connector
- .exclude("TIME type with arrays") // GLUTEN-12569 Spark 4.2 UT enablement
(CI-confirmed)
+ // Replaced by a gluten test that checks internal rows: Dataset.collect()
can't deserialize
+ // ARRAY<TIME(p)> with p < 6.
+ .exclude("TIME type with arrays")
+ // Generated suites for org.apache.spark.sql.connector
enableSuite[GlutenDataSourceV2MetricsSuite]
enableSuite[GlutenDataSourceV2OptionSuite]
enableSuite[GlutenDataSourceV2UtilsSuite]
@@ -812,9 +811,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("SPARK-47238: Test broadcast threshold for generated code")
enableSuite[GlutenBroadcastExchangeSuite]
.exclude("SPARK-52962: broadcast exchange should not reset metrics") //
Add Gluten test
- .exclude(
- "SPARK-56455: broadcast should fail when table size exceeds
maxBroadcastTableSize"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenLocalBroadcastExchangeSuite]
enableSuite[GlutenCoalesceShufflePartitionsSuite]
// Rewrite for Gluten. Change details are in the inline comments in
individual tests.
@@ -826,9 +822,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("replace partial and final hash aggregate together with sort
aggregate")
.exclude("do not replace hash aggregate if child does not have sort order")
.exclude("do not replace hash aggregate if there is no group-by column")
- .exclude(
- "Gluten - do not replace hash aggregate if child does not have sort
order"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenReuseExchangeAndSubquerySuite]
enableSuite[GlutenSameResultSuite]
enableSuite[GlutenSortSuite]
@@ -1216,7 +1209,6 @@ class VeloxTestSettings extends BackendTestSettings {
// TODO: fix on Spark-4.1 introduced by
https://github.com/apache/spark/pull/47856
.exclude("SPARK-49386: test SortMergeJoin (with spill by size threshold)")
enableSuite[GlutenMathFunctionsSuite]
- .exclude("acosh") // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenMapStatusEndToEndSuite]
enableSuite[GlutenMetadataCacheSuite]
.exclude("SPARK-16336,SPARK-27961 Suggest fixing FileNotFoundException")
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenXmlExpressionsSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenXmlExpressionsSuite.scala
index 44b6304ff2..77dc2a88cf 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenXmlExpressionsSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenXmlExpressionsSuite.scala
@@ -16,6 +16,47 @@
*/
package org.apache.spark.sql.catalyst.expressions
+import org.apache.spark.sql.catalyst.InternalRow
+import org.apache.spark.sql.catalyst.util.{GenericArrayData,
SparkDateTimeUtils}
+import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.UTC_OPT
import org.apache.spark.sql.shim
+import org.apache.spark.sql.types.{ArrayType, StructField, StructType,
TimeType}
+import org.apache.spark.unsafe.types.UTF8String
-class GlutenXmlExpressionsSuite extends XmlExpressionsSuite with
shim.GlutenTestsTrait {}
+class GlutenXmlExpressionsSuite extends XmlExpressionsSuite with
shim.GlutenTestsTrait {
+
+ // Dataset.collect() cannot deserialize ARRAY<TIME(p)> with p < 6 into
external rows because
+ // Spark's UpCastRule has no TIME(p) -> TIME(6) rule, so compare the
collected internal rows.
+ private def checkEvaluationOnInternalRows(expression: Expression, expected:
Any): Unit = {
+ val expr = resolveExpression(expression)
+ val rows = buildResultDF(expr,
EmptyRow).queryExecution.executedPlan.executeCollect()
+ assert(rows.length == 1)
+ val actual = rows.head.get(0, expr.dataType)
+ assert(
+ checkResult(actual, expected, expr.dataType, expr.nullable),
+ s"Incorrect evaluation: $expr, actual: $actual, expected: $expected")
+ }
+
+ testGluten("TIME type with arrays") {
+ val schema = StructType(StructField("times", ArrayType(TimeType(3))) ::
Nil)
+ val time1 =
SparkDateTimeUtils.stringToTimeAnsi(UTF8String.fromString("09:00:00"))
+ val time2 =
SparkDateTimeUtils.stringToTimeAnsi(UTF8String.fromString("17:30:45.123"))
+ val times = new GenericArrayData(Array(time1, time2))
+
+ val xmlInput =
"""<record><times>09:00:00</times><times>17:30:45.123</times></record>"""
+
+ // Test parsing array
+ checkEvaluationOnInternalRows(
+ XmlToStructs(schema, Map("rowTag" -> "record"), Literal(xmlInput),
UTC_OPT),
+ InternalRow(times)
+ )
+
+ // Test roundtrip
+ val struct = Literal.create(InternalRow(times), schema)
+ val xmlResult = StructsToXml(Map("rowTag" -> "record"), struct, UTC_OPT)
+ checkEvaluationOnInternalRows(
+ XmlToStructs(schema, Map("rowTag" -> "record"), xmlResult, UTC_OPT),
+ InternalRow(times)
+ )
+ }
+}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala
index 33bf1a1ec9..26ee7bcffc 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/execution/GlutenReplaceHashWithSortAggSuite.scala
@@ -121,7 +121,10 @@ class GlutenReplaceHashWithSortAggSuite
|)
|GROUP BY key
""".stripMargin
- checkAggs(query, 2, 0, 2, 0)
+ // Since SPARK-55551 the broadcast hash join also expands the
streamed side's range
+ // partitioning to the build keys, so no shuffle is needed between
the partial and final
+ // aggregates and Gluten merges them into one complete mode
aggregate.
+ checkAggs(query, 1, 0, 1, 0)
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
