This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c64ff69d24 [GLUTEN-5884][VL] change default load quantum to 8M for
local SSD cache (#8880)
c64ff69d24 is described below
commit c64ff69d24a5da9186f573e64d9b3254d65d2aa0
Author: Yuan <[email protected]>
AuthorDate: Tue Mar 4 18:14:16 2025 +0000
[GLUTEN-5884][VL] change default load quantum to 8M for local SSD cache
(#8880)
Velox is using 23 bits fro cache entry size
facebookincubator/velox#10242
---
.../src/main/scala/org/apache/gluten/config/VeloxConfig.scala | 4 ++--
cpp/velox/compute/VeloxBackend.cc | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index 9fd8064d6e..e1f1c2880e 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -406,10 +406,10 @@ object VeloxConfig {
val LOAD_QUANTUM =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.loadQuantum")
.internal()
- .doc("Set the load quantum for velox file scan, recommend to use the
default value (256MB) " +
+ .doc("Set the load quantum for velox file scan, recommend to use the
default value (8MB) " +
"for performance consideration. If Velox cache is enabled, it can be
8MB at most.")
.bytesConf(ByteUnit.BYTE)
- .createWithDefaultString("256MB")
+ .createWithDefaultString("8MB")
val MAX_COALESCED_DISTANCE_BYTES =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance")
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index c516a8460e..cd62da5c66 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -265,7 +265,7 @@ void VeloxBackend::initConnector() {
connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] =
backendConf_->get<std::string>(kPrefetchRowGroups, "1");
connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] =
- backendConf_->get<std::string>(kLoadQuantum, "268435456"); // 256M
+ backendConf_->get<std::string>(kLoadQuantum, "8388608"); // 8M
connectorConfMap[velox::connector::hive::HiveConfig::kFooterEstimatedSize] =
backendConf_->get<std::string>(kDirectorySizeGuess, "32768"); // 32K
connectorConfMap[velox::connector::hive::HiveConfig::kFilePreloadThreshold] =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
