This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e6ea497d21 [GLUTEN-13179][VL] Fix Spark 4.2 function-registry and
error-wrapping test failures (#13442)
e6ea497d21 is described below
commit e6ea497d210ec0e5a812b72cb10e77ce3a2efe7a
Author: Akshay Tayal <[email protected]>
AuthorDate: Mon Oct 26 15:14:59 2026 -0700
[GLUTEN-13179][VL] Fix Spark 4.2 function-registry and error-wrapping test
failures (#13442)
---
.../gluten/utils/velox/VeloxTestSettings.scala | 11 +++--------
.../spark/sql/GlutenResolveDefaultColumnsSuite.scala | 20 +++++++++++++++++++-
.../sql/extension/GlutenSessionExtensionSuite.scala | 5 ++++-
3 files changed, 26 insertions(+), 10 deletions(-)
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 550f1f1b11..c85c1ceac9 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -832,9 +832,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("test with low buffer spill threshold")
enableSuite[GlutenTakeOrderedAndProjectSuite]
enableSuite[GlutenSessionExtensionSuite]
- .exclude(
- "Gluten - test gluten extensions"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenBucketedReadWithoutHiveSupportSuite]
// Exclude the following suite for plan changed from SMJ to SHJ.
.exclude("avoid shuffle when join 2 bucketed tables")
@@ -1093,9 +1090,6 @@ class VeloxTestSettings extends BackendTestSettings {
// expression referenced multiple times yields row-wise equal values
(rand/randn).
// Same class of difference as SPARK-9083. Not really an issue.
.exclude("SPARK-45216: Non-deterministic functions with seed")
- .exclude(
- "SPARK-57725: resolve columns when the input plan has a null-named
attribute"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenDataFrameTimeWindowingSuite]
enableSuite[GlutenDataFrameTungstenSuite]
enableSuite[GlutenDataFrameWindowFunctionsSuite]
@@ -1293,9 +1287,10 @@ class VeloxTestSettings extends BackendTestSettings {
enableSuite[GlutenLegacyParameterSubstitutionSuite]
enableSuite[GlutenParametersSuite]
enableSuite[GlutenResolveDefaultColumnsSuite]
+ // Velox char/varchar write-side check throws VeloxUserError instead of
SparkRuntimeException.
+ // Rewritten in GlutenResolveDefaultColumnsSuite.
.exclude(
- "SPARK-57187: non-foldable default exceeding CHAR/VARCHAR length fails
at INSERT time (explicit DEFAULT keyword)"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
+ "SPARK-57187: non-foldable default exceeding CHAR/VARCHAR length fails
at INSERT time (explicit DEFAULT keyword)")
enableSuite[GlutenSubqueryHintPropagationSuite]
enableSuite[GlutenUrlFunctionsSuite]
enableSuite[GlutenParquetRowIndexSuite]
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala
index 2d1570be26..7d11c7dceb 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenResolveDefaultColumnsSuite.scala
@@ -16,6 +16,24 @@
*/
package org.apache.spark.sql
+import org.apache.spark.SparkException
+
class GlutenResolveDefaultColumnsSuite
extends ResolveDefaultColumnsSuite
- with GlutenSQLTestsTrait {}
+ with GlutenSQLTestsTrait {
+
+ // The explicit DEFAULT value is checked by the native project, so the
INSERT fails with Velox's
+ // error wrapped in a SparkException instead of Spark's EXCEED_LIMIT_LENGTH
SparkRuntimeException.
+ testGluten(
+ "SPARK-57187: non-foldable default exceeding CHAR/VARCHAR length fails at
INSERT time " +
+ "(explicit DEFAULT keyword)") {
+ Seq("CHAR", "VARCHAR").foreach {
+ typeName =>
+ withTable("t") {
+ sql(s"CREATE TABLE t(i INT, s $typeName(1) DEFAULT current_user())
USING parquet")
+ val e = intercept[SparkException](sql("INSERT INTO t VALUES (1,
DEFAULT)"))
+ assert(e.getMessage.contains("Exceeds allowed length limitation: 1"))
+ }
+ }
+ }
+}
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala
index 4924ee4c4f..8f57a7ed9a 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/extension/GlutenSessionExtensionSuite.scala
@@ -21,6 +21,7 @@ import org.apache.gluten.utils.BackendTestUtils
import org.apache.spark.SparkConf
import org.apache.spark.sql._
+import org.apache.spark.sql.catalyst.analysis.FunctionRegistry
import org.apache.spark.sql.internal.StaticSQLConf.SPARK_SESSION_EXTENSIONS
class GlutenSessionExtensionSuite extends GlutenSQLTestsTrait {
@@ -45,9 +46,11 @@ class GlutenSessionExtensionSuite extends
GlutenSQLTestsTrait {
} else {
assert(spark.sessionState.sqlParser.isInstanceOf[MyParser])
}
+ // Spark 4.2 registers extension functions as builtins under a fully
qualified identifier.
assert(
spark.sessionState.functionRegistry
- .lookupFunction(MyExtensions.myFunction._1)
+ .lookupFunction(
+
FunctionRegistry.builtinFunctionIdentifier(MyExtensions.myFunction._1.funcName))
.isDefined)
assert(
spark.sessionState.columnarRules.contains(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
