This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 0c4a180c11 [GLUTEN-12861] Fix GlutenCachedTableSuite silently losing
spark.sql.shuffle.partitions=5 (#13287)
0c4a180c11 is described below
commit 0c4a180c11b9453ddc5461488c9acc0c54863f94
Author: sabbasani <[email protected]>
AuthorDate: Fri Oct 30 11:17:56 2026 +0530
[GLUTEN-12861] Fix GlutenCachedTableSuite silently losing
spark.sql.shuffle.partitions=5 (#13287)
## Why are the changes needed?
`SharedSparkSessionBase.sparkConf` is a `def` that creates a new
`SparkConf` instance on every invocation. The existing override called
`super.sparkConf` twice, causing `spark.sql.shuffle.partitions=5` to be set on
a different instance and discarded.
As a result, the test suite silently used the default shuffle partition
count of 1 instead of the intended value of 5. This could lead to testing
cached table behavior under an unintended partitioning configuration.
Additionally, `sys.props.put()` introduced an unnecessary JVM-wide system
property side effect, and its comment incorrectly stated that it disabled the
columnar table cache.
## What changes are proposed in this pull request?
- Chain both `.set()` calls on a single `super.sparkConf` instance.
- Ensure `spark.sql.shuffle.partitions=5` and
`COLUMNAR_TABLE_CACHE_ENABLED=true` are applied to the same configuration.
- Remove the redundant `sys.props.put()` JVM-wide system property
modification and its misleading comment.
- Apply the fix consistently across Spark 3.4, 3.5, 4.0, and 4.1 modules.
## How was this patch tested?
- Verified the `sparkConf` override uses a single `SparkConf` instance.
- Verified the changes across all four Spark version modules.
- Ran `GlutenCachedTableSuite` to validate the changes.
## Was this patch authored or co-authored using generative AI tooling?
yes.
Related issue: #12861
---
.../scala/org/apache/spark/sql/GlutenCachedTableSuite.scala | 10 ++++------
.../scala/org/apache/spark/sql/GlutenCachedTableSuite.scala | 10 ++++------
.../scala/org/apache/spark/sql/GlutenCachedTableSuite.scala | 10 ++++------
.../scala/org/apache/spark/sql/GlutenCachedTableSuite.scala | 10 ++++------
.../scala/org/apache/spark/sql/GlutenCachedTableSuite.scala | 10 ++++------
5 files changed, 20 insertions(+), 30 deletions(-)
diff --git
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
index c7ff740589..80c18fddc6 100644
---
a/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
+++
b/gluten-ut/spark34/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -29,12 +29,10 @@ class GlutenCachedTableSuite
extends CachedTableSuite
with GlutenSQLTestsTrait
with AdaptiveSparkPlanHelper {
- // for temporarily disable the columnar table cache globally.
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- override def sparkConf: SparkConf = {
- super.sparkConf.set("spark.sql.shuffle.partitions", "5")
- super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- }
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.shuffle.partitions", "5")
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
testGluten("InMemoryRelation statistics") {
sql("CACHE TABLE testData")
diff --git
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
index b0009c2ec4..38fd7bd97e 100644
---
a/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
+++
b/gluten-ut/spark35/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -32,12 +32,10 @@ class GlutenCachedTableSuite
with GlutenSQLTestsTrait
with AdaptiveSparkPlanHelper {
import testImplicits._
- // for temporarily disable the columnar table cache globally.
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- override def sparkConf: SparkConf = {
- super.sparkConf.set("spark.sql.shuffle.partitions", "5")
- super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- }
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.shuffle.partitions", "5")
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
testGluten("InMemoryRelation statistics") {
sql("CACHE TABLE testData")
diff --git
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
index 2961b7c7a9..d54ac57f87 100644
---
a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
+++
b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -31,12 +31,10 @@ class GlutenCachedTableSuite
with GlutenSQLTestsTrait
with AdaptiveSparkPlanHelper {
import testImplicits._
- // for temporarily disable the columnar table cache globally.
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- override def sparkConf: SparkConf = {
- super.sparkConf.set("spark.sql.shuffle.partitions", "5")
- super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- }
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.shuffle.partitions", "5")
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
testGluten("SPARK-52684: Atomicity of cache table on error") {
withTempView("SPARK_52684") {
diff --git
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
index 2961b7c7a9..d54ac57f87 100644
---
a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
+++
b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -31,12 +31,10 @@ class GlutenCachedTableSuite
with GlutenSQLTestsTrait
with AdaptiveSparkPlanHelper {
import testImplicits._
- // for temporarily disable the columnar table cache globally.
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- override def sparkConf: SparkConf = {
- super.sparkConf.set("spark.sql.shuffle.partitions", "5")
- super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- }
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.shuffle.partitions", "5")
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
testGluten("SPARK-52684: Atomicity of cache table on error") {
withTempView("SPARK_52684") {
diff --git
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
index 2961b7c7a9..d54ac57f87 100644
---
a/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
+++
b/gluten-ut/spark42/src/test/scala/org/apache/spark/sql/GlutenCachedTableSuite.scala
@@ -31,12 +31,10 @@ class GlutenCachedTableSuite
with GlutenSQLTestsTrait
with AdaptiveSparkPlanHelper {
import testImplicits._
- // for temporarily disable the columnar table cache globally.
- sys.props.put(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- override def sparkConf: SparkConf = {
- super.sparkConf.set("spark.sql.shuffle.partitions", "5")
- super.sparkConf.set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
- }
+ override def sparkConf: SparkConf =
+ super.sparkConf
+ .set("spark.sql.shuffle.partitions", "5")
+ .set(GlutenConfig.COLUMNAR_TABLE_CACHE_ENABLED.key, "true")
testGluten("SPARK-52684: Atomicity of cache table on error") {
withTempView("SPARK_52684") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
