This is an automated email from the ASF dual-hosted git repository.
rui-mo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/gluten.git
The following commit(s) were added to refs/heads/main by this push:
new cd1d1b9939 [VL] Add new config
`spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles` (#12759)
cd1d1b9939 is described below
commit cd1d1b9939c52d4e117e80a9c6072184f4039bf1
Author: Wechar Yu <[email protected]>
AuthorDate: Wed Aug 26 20:57:36 2026 +0800
[VL] Add new config
`spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles` (#12759)
---
.../src/main/scala/org/apache/gluten/config/VeloxConfig.scala | 9 +++++++++
cpp/velox/compute/WholeStageResultIterator.cc | 2 ++
cpp/velox/config/VeloxConfig.h | 1 +
docs/get-started/Velox.md | 1 +
docs/velox-configuration.md | 1 +
5 files changed, 14 insertions(+)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 802a47cadd..67c7192d95 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -286,6 +286,15 @@ object VeloxConfig extends ConfigRegistry {
.bytesConf(ByteUnit.BYTE)
.createWithDefaultString("1GB")
+ val COLUMNAR_VELOX_SPILL_NUM_MAX_MERGE_FILES =
+ buildConf("spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles")
+ .doc(
+ "The max number of files to merge at a time when merging sorted files
" +
+ "into a single ordered stream. 0 means unlimited.")
+ .intConf
+ .checkValue(_ >= 0, "must be non-negative")
+ .createWithDefault(0)
+
val COLUMNAR_VELOX_SPILL_FILE_SYSTEM =
buildConf("spark.gluten.sql.columnar.backend.velox.spillFileSystem")
.doc(
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index 4e680fb25b..409f288c17 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -580,6 +580,8 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
configs[velox::core::QueryConfig::kMaxSpillLevel] =
std::to_string(veloxCfg_->get<int32_t>(kMaxSpillLevel, 4));
configs[velox::core::QueryConfig::kMaxSpillFileSize] =
std::to_string(veloxCfg_->get<uint64_t>(kMaxSpillFileSize, 1L * 1024 *
1024 * 1024));
+ configs[velox::core::QueryConfig::kSpillNumMaxMergeFiles] =
+ std::to_string(veloxCfg_->get<uint32_t>(kSpillNumMaxMergeFiles, 0));
configs[velox::core::QueryConfig::kMaxSpillRunRows] =
std::to_string(veloxCfg_->get<uint64_t>(kMaxSpillRunRows, 3L * 1024 *
1024));
configs[velox::core::QueryConfig::kMaxSpillBytes] =
diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h
index c0ae4b3ff8..f9351560cf 100644
--- a/cpp/velox/config/VeloxConfig.h
+++ b/cpp/velox/config/VeloxConfig.h
@@ -35,6 +35,7 @@ const std::string kWindowSpillEnabled =
"spark.gluten.sql.columnar.backend.velox
//
https://github.com/facebookincubator/velox/blob/95f3e80e77d046c12fbc79dc529366be402e9c2b/velox/docs/configs.rst#spilling
const std::string kMaxSpillLevel =
"spark.gluten.sql.columnar.backend.velox.maxSpillLevel";
const std::string kMaxSpillFileSize =
"spark.gluten.sql.columnar.backend.velox.maxSpillFileSize";
+const std::string kSpillNumMaxMergeFiles =
"spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles";
const std::string kSpillStartPartitionBit =
"spark.gluten.sql.columnar.backend.velox.spillStartPartitionBit";
const std::string kSpillPartitionBits =
"spark.gluten.sql.columnar.backend.velox.spillPartitionBits";
const std::string kMaxSpillRunRows =
"spark.gluten.sql.columnar.backend.velox.MaxSpillRunRows";
diff --git a/docs/get-started/Velox.md b/docs/get-started/Velox.md
index 9eead57d0d..03aa02ef00 100644
--- a/docs/get-started/Velox.md
+++ b/docs/get-started/Velox.md
@@ -474,6 +474,7 @@ Using the following configuration options to customize
spilling:
| spark.gluten.sql.columnar.backend.velox.orderBySpillEnabled |
true | Whether spill is enabled on sorts
|
| spark.gluten.sql.columnar.backend.velox.maxSpillLevel | 4
| The max allowed spilling level with zero being the initial
spilling level
|
| spark.gluten.sql.columnar.backend.velox.maxSpillFileSize |
1GB | The max allowed spill file size. If it is zero, then there is
no limit
|
+| spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles | 0
| The max number of files to merge at a time when merging sorted
files into a single ordered stream. 0 means unlimited.
|
| spark.gluten.sql.columnar.backend.velox.spillStartPartitionBit |
48 | The start partition bit which is used with 'spillPartitionBits'
together to calculate the spilling partition number
|
| spark.gluten.sql.columnar.backend.velox.spillPartitionBits | 3
| The number of bits used to calculate the spilling partition
number. The number of spilling partitions will be power of two
|
| spark.gluten.sql.columnar.backend.velox.spillableReservationGrowthPct |
25 | The spillable memory reservation growth percentage of the
previous memory reservation size
|
diff --git a/docs/velox-configuration.md b/docs/velox-configuration.md
index 712260f984..a21373fe4a 100644
--- a/docs/velox-configuration.md
+++ b/docs/velox-configuration.md
@@ -78,6 +78,7 @@ nav_order: 16
| spark.gluten.sql.columnar.backend.velox.scan.bloomFilterPushdown.enabled
| ⚓ Static | false | Whether to push Bloom filters into
Velox scans.
[...]
| spark.gluten.sql.columnar.backend.velox.showTaskMetricsWhenFinished
| 🔄 Dynamic | false | Show velox full task metrics when
finished.
[...]
| spark.gluten.sql.columnar.backend.velox.spillFileSystem
| 🔄 Dynamic | local | The filesystem used to store spill
data. local: The local file system. heap-over-local: Write file to JVM heap if
having extra heap space. Otherwise write to local file system.
[...]
+| spark.gluten.sql.columnar.backend.velox.spillNumMaxMergeFiles
| 🔄 Dynamic | 0 | The max number of files to merge at a
time when merging sorted files into a single ordered stream. 0 means unlimited.
[...]
| spark.gluten.sql.columnar.backend.velox.spillStrategy
| 🔄 Dynamic | auto | none: Disable spill on Velox backend;
auto: Let Spark memory manager manage Velox's spilling
[...]
| spark.gluten.sql.columnar.backend.velox.ssdCacheIOThreads
| ⚓ Static | 4 | The number of IO threads for SSD
cache read/write operations
[...]
| spark.gluten.sql.columnar.backend.velox.ssdCachePath
| ⚓ Static | /tmp | The folder to store the cache files,
better on SSD
[...]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
