This is an automated email from the ASF dual-hosted git repository.
philo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new e646fe70b9 [GLUTEN-10613][VL] Add `experimental` method to
ConfigBuilder and document experimental configs (#10659)
e646fe70b9 is described below
commit e646fe70b9998a4cf34935da21cb1fc6204e1c4a
Author: Terry Wang <[email protected]>
AuthorDate: Mon Sep 15 15:10:19 2025 +0800
[GLUTEN-10613][VL] Add `experimental` method to ConfigBuilder and document
experimental configs (#10659)
---
.../org/apache/gluten/config/VeloxConfig.scala | 2 +-
.../gluten/config/AllVeloxConfiguration.scala | 21 ++++++++++++++++++++
docs/Configuration.md | 12 +++++++++++
docs/velox-configuration.md | 6 ++++++
.../org/apache/gluten/config/ConfigBuilder.scala | 23 +++++++++++++++++++---
.../org/apache/gluten/config/ConfigEntry.scala | 14 +++++++++++++
.../apache/gluten/config/GlutenCoreConfig.scala | 4 ++--
.../org/apache/gluten/config/GlutenConfig.scala | 10 +++++-----
.../gluten/config/AllGlutenConfiguration.scala | 20 +++++++++++++++++++
9 files changed, 101 insertions(+), 11 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index bc9665641d..b821d2d35a 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -537,7 +537,7 @@ object VeloxConfig {
val VELOX_BROADCAST_BUILD_RELATION_USE_OFFHEAP =
buildConf("spark.gluten.velox.offHeapBroadcastBuildRelation.enabled")
- .internal()
+ .experimental()
.doc("Experimental: If enabled, broadcast build relation will use
offheap memory. " +
"Otherwise, broadcast build relation will use onheap memory.")
.booleanConf
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/config/AllVeloxConfiguration.scala
b/backends-velox/src/test/scala/org/apache/gluten/config/AllVeloxConfiguration.scala
index 7792518435..98c6ee0c8c 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/config/AllVeloxConfiguration.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/config/AllVeloxConfiguration.scala
@@ -52,6 +52,27 @@ class AllVeloxConfiguration extends AnyFunSuite {
ConfigEntry.getAllEntries
.filter(_.key.contains("velox"))
.filter(_.isPublic)
+ .filter(!_.isExperimental)
+ .sortBy(_.key)
+ .foreach {
+ entry =>
+ val dft = entry.defaultValueString.replace("<", "<").replace(">",
">")
+ builder += Seq(s"${entry.key}", s"$dft", s"${entry.doc}")
+ .mkString("|")
+ }
+
+ builder ++=
+ s"""
+ |## Gluten Velox backend *experimental* configurations
+ |
+ | Key | Default | Description
+ | --- | --- | ---
+ |"""
+
+ ConfigEntry.getAllEntries
+ .filter(_.key.contains("velox"))
+ .filter(_.isPublic)
+ .filter(_.isExperimental)
.sortBy(_.key)
.foreach {
entry =>
diff --git a/docs/Configuration.md b/docs/Configuration.md
index e3350b8d87..b7e725278a 100644
--- a/docs/Configuration.md
+++ b/docs/Configuration.md
@@ -140,3 +140,15 @@ nav_order: 15
| spark.gluten.supported.scala.udfs
|| Supported scala udf names.
|
| spark.gluten.ui.enabled | true
| Whether to enable the gluten web UI, If true, attach the gluten UI
page to the Spark web UI.
|
+## Gluten *experimental* configurations
+
+| Key | Default
|
Description
[...]
+|-------------------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
[...]
+| spark.gluten.auto.adjustStageResource.enabled | false
| Experimental: If enabled, gluten will try to set the stage resource according
to stage execution plan. Only worked when aqe is enabled at the same time!!
[...]
+| spark.gluten.auto.adjustStageResources.fallenNode.ratio.threshold | 0.5
| Experimental: Increase executor heap memory when stage contains fallen node
count exceeds the total node count ratio.
[...]
+| spark.gluten.auto.adjustStageResources.heap.ratio | 2.0
| Experimental: Increase executor heap memory when match adjust stage resource
rule.
[...]
+| spark.gluten.auto.adjustStageResources.offheap.ratio | 0.5
| Experimental: Decrease executor offheap memory when match adjust stage
resource rule.
[...]
+| spark.gluten.memory.dynamic.offHeap.sizing.enabled | false
| Experimental: When set to true, the offheap config
(spark.memory.offHeap.size) will be ignored and instead we will consider onheap
and offheap memory in combination, both counting towards the executor memory
config (spark.executor.memory). We will make use of JVM APIs to determine how
much onheap memory is use, alongside tracking offheap allocations made by
Gluten. We will then proceed to enforcing a total me [...]
+| spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction | 0.6
| Experimental: Determines the memory fraction used to determine the total
memory available for offheap and onheap allocations when the dynamic offheap
sizing feature is enabled. The default is set to match
spark.executor.memoryFraction.
[...]
+| spark.gluten.sql.columnar.cudf | false
| Enable or disable cudf support. This is an experimental feature.
[...]
+
diff --git a/docs/velox-configuration.md b/docs/velox-configuration.md
index 4980f97767..cf10d5625c 100644
--- a/docs/velox-configuration.md
+++ b/docs/velox-configuration.md
@@ -70,3 +70,9 @@ nav_order: 16
| spark.gluten.velox.castFromVarcharAddTrimNode
| false | If true, will add a trim node which has the same
sementic as vanilla Spark to CAST-from-varchar.Otherwise, do nothing.
[...]
| spark.gluten.velox.fs.s3a.connect.timeout
| 200s | Timeout for AWS s3 connection.
[...]
+## Gluten Velox backend *experimental* configurations
+
+| Key | Default |
Description
|
+|----------------------------------------------------------|---------|-----------------------------------------------------------------------------------------------------------------------------------------|
+| spark.gluten.velox.offHeapBroadcastBuildRelation.enabled | false |
Experimental: If enabled, broadcast build relation will use offheap memory.
Otherwise, broadcast build relation will use onheap memory. |
+
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala
b/gluten-core/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala
index 199bcdec23..cde86ebc70 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/config/ConfigBuilder.scala
@@ -33,6 +33,7 @@ private[gluten] case class ConfigBuilder(key: String) {
private[config] var _version = ""
private[config] var _backend = BackendType.COMMON
private[config] var _public = true
+ private[config] var _experimental = false
private[config] var _alternatives = List.empty[String]
private[config] var _onCreate: Option[ConfigEntry[_] => Unit] = None
@@ -54,7 +55,6 @@ private[gluten] case class ConfigBuilder(key: String) {
/**
* This method marks a config as internal for any of the following reasons:
* - Intended exclusively for developers or advanced users
- * - Experimental or unstable, not yet exposed to end users
* - Allows for flexibility in development and testing without
compromising the public API's
* stability
*/
@@ -63,6 +63,11 @@ private[gluten] case class ConfigBuilder(key: String) {
this
}
+ def experimental(): ConfigBuilder = {
+ _experimental = true
+ this
+ }
+
def onCreate(callback: ConfigEntry[_] => Unit): ConfigBuilder = {
_onCreate = Option(callback)
this
@@ -103,7 +108,15 @@ private[gluten] case class ConfigBuilder(key: String) {
def fallbackConf[T](fallback: ConfigEntry[T]): ConfigEntry[T] = {
val entry =
- new ConfigEntryFallback[T](key, _doc, _version, _backend, _public,
_alternatives, fallback)
+ new ConfigEntryFallback[T](
+ key,
+ _doc,
+ _version,
+ _backend,
+ _public,
+ _experimental,
+ _alternatives,
+ fallback)
_onCreate.foreach(_(entry))
entry
}
@@ -189,6 +202,7 @@ private[gluten] class TypedConfigBuilder[T](
parent._version,
parent._backend,
parent._public,
+ parent._experimental,
parent._alternatives,
converter,
stringConverter)
@@ -208,10 +222,12 @@ private[gluten] class TypedConfigBuilder[T](
parent._version,
parent._backend,
parent._public,
+ parent._experimental,
parent._alternatives,
converter,
stringConverter,
- transformedDefault)
+ transformedDefault
+ )
parent._onCreate.foreach(_(entry))
entry
}
@@ -224,6 +240,7 @@ private[gluten] class TypedConfigBuilder[T](
parent._version,
parent._backend,
parent._public,
+ parent._experimental,
parent._alternatives,
converter,
stringConverter,
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/config/ConfigEntry.scala
b/gluten-core/src/main/scala/org/apache/gluten/config/ConfigEntry.scala
index 78111f5b45..ed861ec833 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/config/ConfigEntry.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/config/ConfigEntry.scala
@@ -51,6 +51,8 @@ trait ConfigEntry[T] {
*/
def isPublic: Boolean
+ def isExperimental: Boolean
+
/** the alternative keys for the configuration. */
def alternatives: List[String]
@@ -90,6 +92,7 @@ private[gluten] class OptionalConfigEntry[T](
_version: String,
_backend: BackendType,
_isPublic: Boolean,
+ _isExperimental: Boolean,
_alternatives: List[String],
_valueConverter: String => T,
_stringConverter: T => String)
@@ -104,6 +107,8 @@ private[gluten] class OptionalConfigEntry[T](
override def isPublic: Boolean = _isPublic
+ override def isExperimental: Boolean = _isExperimental
+
override def alternatives: List[String] = _alternatives
override def valueConverter: String => Option[T] = s =>
Option(_valueConverter(s))
@@ -124,6 +129,7 @@ private[gluten] class ConfigEntryWithDefault[T](
_version: String,
_backend: BackendType,
_isPublic: Boolean,
+ _isExperimental: Boolean,
_alternatives: List[String],
_valueConverter: String => T,
_stringConverter: T => String,
@@ -139,6 +145,8 @@ private[gluten] class ConfigEntryWithDefault[T](
override def isPublic: Boolean = _isPublic
+ override def isExperimental: Boolean = _isExperimental
+
override def alternatives: List[String] = _alternatives
override def valueConverter: String => T = _valueConverter
@@ -160,6 +168,7 @@ private[gluten] class ConfigEntryWithDefaultString[T](
_version: String,
_backend: BackendType,
_isPublic: Boolean,
+ _isExperimental: Boolean,
_alternatives: List[String],
_valueConverter: String => T,
_stringConverter: T => String,
@@ -175,6 +184,8 @@ private[gluten] class ConfigEntryWithDefaultString[T](
override def isPublic: Boolean = _isPublic
+ override def isExperimental: Boolean = _isExperimental
+
override def alternatives: List[String] = _alternatives
override def valueConverter: String => T = _valueConverter
@@ -197,6 +208,7 @@ private[gluten] class ConfigEntryFallback[T](
_version: String,
_backend: BackendType,
_isPublic: Boolean,
+ _isExperimental: Boolean,
_alternatives: List[String],
fallback: ConfigEntry[T])
extends ConfigEntry[T] {
@@ -210,6 +222,8 @@ private[gluten] class ConfigEntryFallback[T](
override def isPublic: Boolean = _isPublic
+ override def isExperimental: Boolean = _isExperimental
+
override def alternatives: List[String] = _alternatives
override def valueConverter: String => T = fallback.valueConverter
diff --git
a/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
b/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
index 85d954f54f..4d27b7b451 100644
--- a/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
+++ b/gluten-core/src/main/scala/org/apache/gluten/config/GlutenCoreConfig.scala
@@ -195,7 +195,7 @@ object GlutenCoreConfig {
// Since https://github.com/apache/incubator-gluten/issues/5439.
val DYNAMIC_OFFHEAP_SIZING_ENABLED =
buildStaticConf("spark.gluten.memory.dynamic.offHeap.sizing.enabled")
- .internal()
+ .experimental()
.doc(
"Experimental: When set to true, the offheap config
(spark.memory.offHeap.size) will " +
"be ignored and instead we will consider onheap and offheap memory
in combination, " +
@@ -213,7 +213,7 @@ object GlutenCoreConfig {
// Since https://github.com/apache/incubator-gluten/issues/5439.
val DYNAMIC_OFFHEAP_SIZING_MEMORY_FRACTION =
buildStaticConf("spark.gluten.memory.dynamic.offHeap.sizing.memory.fraction")
- .internal()
+ .experimental()
.doc(
"Experimental: Determines the memory fraction used to determine the
total " +
"memory available for offheap and onheap allocations when the
dynamic offheap " +
diff --git
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
index 6c28443079..c2339dda43 100644
---
a/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
+++
b/gluten-substrait/src/main/scala/org/apache/gluten/config/GlutenConfig.scala
@@ -1497,7 +1497,7 @@ object GlutenConfig {
val AUTO_ADJUST_STAGE_RESOURCE_PROFILE_ENABLED =
buildConf("spark.gluten.auto.adjustStageResource.enabled")
- .internal()
+ .experimental()
.doc("Experimental: If enabled, gluten will try to set the stage
resource according " +
"to stage execution plan. Only worked when aqe is enabled at the same
time!!")
.booleanConf
@@ -1505,21 +1505,21 @@ object GlutenConfig {
val AUTO_ADJUST_STAGE_RESOURCES_HEAP_RATIO =
buildConf("spark.gluten.auto.adjustStageResources.heap.ratio")
- .internal()
+ .experimental()
.doc("Experimental: Increase executor heap memory when match adjust
stage resource rule.")
.doubleConf
.createWithDefault(2.0d)
val AUTO_ADJUST_STAGE_RESOURCES_OFFHEAP_RATIO =
buildConf("spark.gluten.auto.adjustStageResources.offheap.ratio")
- .internal()
+ .experimental()
.doc("Experimental: Decrease executor offheap memory when match adjust
stage resource rule.")
.doubleConf
.createWithDefault(0.5d)
val AUTO_ADJUST_STAGE_RESOURCES_FALLEN_NODE_RATIO_THRESHOLD =
buildConf("spark.gluten.auto.adjustStageResources.fallenNode.ratio.threshold")
- .internal()
+ .experimental()
.doc("Experimental: Increase executor heap memory when stage contains
fallen node " +
"count exceeds the total node count ratio.")
.doubleConf
@@ -1547,7 +1547,7 @@ object GlutenConfig {
val COLUMNAR_CUDF_ENABLED =
buildConf("spark.gluten.sql.columnar.cudf")
- .internal()
+ .experimental()
.doc("Enable or disable cudf support. This is an experimental feature.")
.booleanConf
.createWithDefault(false)
diff --git
a/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
b/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
index ab0574a602..776b4b559d 100644
---
a/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
+++
b/gluten-substrait/src/test/scala/org/apache/gluten/config/AllGlutenConfiguration.scala
@@ -112,6 +112,26 @@ class AllGlutenConfiguration extends AnyFunSuite {
ConfigEntry.getAllEntries
.filter(_.isPublic)
+ .filter(!_.isExperimental)
+ .sortBy(_.key)
+ .foreach {
+ entry =>
+ val dft = entry.defaultValueString.replace("<", "<").replace(">",
">")
+ builder += Seq(s"${entry.key}", s"$dft", s"${entry.doc}")
+ .mkString("|")
+ }
+
+ builder ++=
+ s"""
+ |## Gluten *experimental* configurations
+ |
+ | Key | Default | Description
+ | --- | --- | ---
+ |"""
+
+ ConfigEntry.getAllEntries
+ .filter(_.isPublic)
+ .filter(_.isExperimental)
.sortBy(_.key)
.foreach {
entry =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
