This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 1fdabb14a0 [GLUTEN-13180][VL] Re-enable the Spark 4.2 bucketed scan AE
and Hive ADD JAR tests (#13432)
1fdabb14a0 is described below
commit 1fdabb14a00c74df016cd16aca3b29b60c13f457
Author: Akshay Tayal <[email protected]>
AuthorDate: Mon Oct 26 15:06:55 2026 -0700
[GLUTEN-13180][VL] Re-enable the Spark 4.2 bucketed scan AE and Hive ADD
JAR tests (#13432)
---
.../gluten/utils/velox/VeloxTestSettings.scala | 16 --------------
.../sql/hive/execution/GlutenHiveQuerySuite.scala | 25 ++++++++++++++++++++++
...GlutenDisableUnnecessaryBucketedScanSuite.scala | 19 +++++++++++++++-
3 files changed, 43 insertions(+), 17 deletions(-)
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
index 156d730611..550f1f1b11 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala
@@ -868,21 +868,6 @@ class VeloxTestSettings extends BackendTestSettings {
disableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite](
"GLUTEN-4893: Vanilla UT checks scan operator by exactly matching the
class type")
enableSuite[GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE]
- .exclude(
- "Aggregates with no groupby over tables having 1 BUCKET, return multiple
rows"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
- .exclude(
- "SPARK-32859: disable unnecessary bucketed table scan - basic test"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
- .exclude(
- "SPARK-32859: disable unnecessary bucketed table scan - multiple
bucketed columns test"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
- .exclude(
- "SPARK-32859: disable unnecessary bucketed table scan - multiple joins
test"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
- .exclude(
- "SPARK-32859: disable unnecessary bucketed table scan - other operators
test"
- ) // GLUTEN-12569 Spark 4.2 UT enablement (CI-confirmed)
enableSuite[GlutenExternalCommandRunnerSuite]
enableSuite[GlutenFilteredScanSuite]
enableSuite[GlutenFiltersSuite]
@@ -1278,7 +1263,6 @@ class VeloxTestSettings extends BackendTestSettings {
.exclude("EXPLAIN CODEGEN command")
enableSuite[GlutenHivePlanTest]
enableSuite[GlutenHiveQuerySuite]
- .exclude("ADD JAR command 2") // GLUTEN-12569 Spark 4.2 UT enablement
(CI-confirmed)
enableSuite[GlutenHiveResolutionSuite]
enableSuite[GlutenHiveSQLQuerySuite]
enableSuite[GlutenHiveSQLViewSuite]
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala
index 4b72a08986..184af6b7d5 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/hive/execution/GlutenHiveQuerySuite.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.hive.execution
import org.apache.spark.SparkFiles
import org.apache.spark.sql.GlutenTestSetWithSystemPropertyTrait
import org.apache.spark.sql.hive.HiveUtils.{builtinHiveVersion => hiveVersion}
+import org.apache.spark.sql.hive.test.{HiveTestJars, TestUDTFJar}
import org.apache.spark.tags.SlowHiveTest
import java.io.File
@@ -31,6 +32,7 @@ class GlutenHiveQuerySuite
override def testNameBlackList: Seq[String] = Seq(
"ADD FILE command",
+ "ADD JAR command 2",
"SPARK-33084: Add jar support Ivy URI in SQL"
)
@@ -48,6 +50,29 @@ class GlutenHiveQuerySuite
assert(sql(s"list file $testFile").count() == 1)
}
+ testGluten("ADD JAR command 2") {
+ val testJar = HiveTestJars.getHiveHcatalogCoreJar().toURI
+ val testData = hiveResourcePath("data/files/sample.json").toUri
+ sql(s"ADD JAR $testJar")
+ withTable("t1") {
+ sql("""CREATE TABLE t1(a string, b string)
+ |ROW FORMAT SERDE
'org.apache.hive.hcatalog.data.JsonSerDe'""".stripMargin)
+ sql(s"""LOAD DATA LOCAL INPATH "$testData" INTO TABLE t1""")
+ sql("select * from src join t1 on src.key = t1.a")
+ }
+ assert(
+ sql("list jars")
+
.filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName))
+ .count() > 0)
+ assert(
+ sql("list jar")
+
.filter(_.getString(0).contains(HiveTestJars.getHiveHcatalogCoreJar().getName))
+ .count() > 0)
+ val testJar2 = TestUDTFJar.jar.getCanonicalPath
+ sql(s"ADD JAR $testJar2")
+ assert(sql(s"list jar $testJar").count() == 1)
+ }
+
testGluten("SPARK-33084: Add jar support Ivy URI in SQL") {
val testData = hiveResourcePath("data/files/sample.json").toUri
withTable("t") {
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala
index fd77663985..bbe504f04a 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/sources/GlutenDisableUnnecessaryBucketedScanSuite.scala
@@ -16,7 +16,10 @@
*/
package org.apache.spark.sql.sources
+import org.apache.gluten.execution.FileSourceScanExecTransformer
+
import org.apache.spark.sql.GlutenSQLTestsBaseTrait
+import org.apache.spark.sql.execution.FileSourceScanExec
class GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite
extends DisableUnnecessaryBucketedScanWithoutHiveSupportSuite
@@ -24,4 +27,18 @@ class
GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuite
class GlutenDisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE
extends DisableUnnecessaryBucketedScanWithoutHiveSupportSuiteAE
- with GlutenSQLTestsBaseTrait {}
+ with GlutenSQLTestsBaseTrait {
+
+ // The executed plan contains Gluten's FileSourceScanExecTransformer instead
of the vanilla
+ // FileSourceScanExec, so count bucketed scans of both.
+ override protected def checkNumBucketedScan(query: String,
expectedNumBucketedScan: Int): Unit = {
+ val df = sql(query)
+ df.collect()
+ val plan = df.queryExecution.executedPlan
+ val bucketedScan = collect(plan) {
+ case s: FileSourceScanExec if s.bucketedScan => s
+ case s: FileSourceScanExecTransformer if s.bucketedScan => s
+ }
+ assert(bucketedScan.length == expectedNumBucketedScan)
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
