This is an automated email from the ASF dual-hosted git repository.
marin-ma pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8eb0413549 [GLUTEN-11539][VL] Improve error message for unsupported
spark.io.compression.codec in native shuffle (#12333)
8eb0413549 is described below
commit 8eb04135492e55d9d07837753c6d679229c31546
Author: BRIJ RAJ KISHORE <[email protected]>
AuthorDate: Wed Jun 24 17:02:19 2026 +0530
[GLUTEN-11539][VL] Improve error message for unsupported
spark.io.compression.codec in native shuffle (#12333)
---
.../gluten/execution/MiscOperatorSuite.scala | 27 ++++++++++++++++++++++
.../apache/spark/shuffle/GlutenShuffleUtils.scala | 12 ++++++----
2 files changed, 35 insertions(+), 4 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
index 2cc8b18fbf..fcf1fbbbf7 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/MiscOperatorSuite.scala
@@ -20,6 +20,7 @@ import org.apache.gluten.config.{GlutenConfig,
GlutenCoreConfig, VeloxConfig}
import org.apache.gluten.expression.VeloxDummyExpression
import org.apache.spark.SparkConf
+import org.apache.spark.shuffle.GlutenShuffleUtils
import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.execution._
import org.apache.spark.sql.execution.adaptive.{AdaptiveSparkPlanHelper,
AQEShuffleReadExec, ShuffleQueryStageExec}
@@ -2250,4 +2251,30 @@ class MiscOperatorSuite extends
VeloxWholeStageTransformerSuite with AdaptiveSpa
}
}
}
+
+ test("GLUTEN-11539: unsupported spark.io.compression.codec throws with
actionable message") {
+ val conf =
spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "snappy")
+ val ex = intercept[IllegalArgumentException] {
+ GlutenShuffleUtils.getCompressionCodec(conf)
+ }
+ assert(ex.getMessage.contains("snappy is not supported"))
+ assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
+ }
+
+ test("GLUTEN-11539: spark.io.compression.codec=none throws with actionable
message") {
+ val conf =
spark.sparkContext.getConf.clone().set("spark.io.compression.codec", "none")
+ val ex = intercept[IllegalArgumentException] {
+ GlutenShuffleUtils.getCompressionCodec(conf)
+ }
+ assert(ex.getMessage.contains("none is not supported"))
+ assert(ex.getMessage.contains(GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key))
+ }
+
+ test("GLUTEN-11539: supported spark.io.compression.codec is accepted") {
+ Seq("lz4", "zstd").foreach {
+ codec =>
+ val conf =
spark.sparkContext.getConf.clone().set("spark.io.compression.codec", codec)
+ assert(GlutenShuffleUtils.getCompressionCodec(conf) === codec)
+ }
+ }
}
diff --git
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
index 213b2831f4..4bd89582c1 100644
---
a/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
+++
b/gluten-substrait/src/main/scala/org/apache/spark/shuffle/GlutenShuffleUtils.scala
@@ -74,10 +74,14 @@ object GlutenShuffleUtils {
conf
.get(sparkCodecKey, IO_COMPRESSION_CODEC.defaultValueString)
.toLowerCase(Locale.ROOT)
- checkCodecValues(
- sparkCodecKey,
- codec,
- BackendsApiManager.getSettings.shuffleSupportedCodec())
+ val supportedCodecs =
BackendsApiManager.getSettings.shuffleSupportedCodec()
+ if (!supportedCodecs.contains(codec)) {
+ throw new IllegalArgumentException(
+ s"Gluten shuffle only supports ${supportedCodecs.mkString(", ")}.
" +
+ s"$codec is not supported. " +
+ s"You may configure ${GlutenConfig.COLUMNAR_SHUFFLE_CODEC.key} "
+
+ s"to ${supportedCodecs.mkString(" or ")}.")
+ }
codec
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
