This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 254d62e72 [VL] Enable split preloading by default (#5456)
254d62e72 is described below
commit 254d62e7248c214591f8499ae2f99d1b669d9dfe
Author: Zhen Li <[email protected]>
AuthorDate: Wed May 8 00:53:57 2024 +0800
[VL] Enable split preloading by default (#5456)
Enable split preloading by default. use 1 I/O thread for each task thread
---
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 3c7ddf32c..06d72ec57 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -1230,9 +1230,14 @@ object GlutenConfig {
val COLUMNAR_VELOX_CONNECTOR_IO_THREADS =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.IOThreads")
.internal()
- .doc("The IO threads for connector split preloading")
+ .doc("The Size of the IO thread pool in the Connector. This thread pool
is used for split" +
+ " preloading and DirectBufferedInput.")
.intConf
- .createWithDefault(0)
+ .createWithDefaultFunction(
+ () =>
+ SQLConf.get.getConfString("spark.executor.cores", "1").toInt /
SQLConf.get
+ .getConfString("spark.task.cpus", "1")
+ .toInt)
val COLUMNAR_VELOX_ASYNC_TIMEOUT =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.asyncTimeoutOnTaskStopping")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
