This is an automated email from the ASF dual-hosted git repository.
felixybw pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 86d0174650 Revert "[GLUTEN-5884][VL] change default load quantum to 8M
for local SSD cac…" (#8900)
86d0174650 is described below
commit 86d0174650877a8cb81ad5b35114c3f72bb43286
Author: BInwei Yang <[email protected]>
AuthorDate: Tue Mar 4 22:28:57 2025 -0800
Revert "[GLUTEN-5884][VL] change default load quantum to 8M for local SSD
cac…" (#8900)
Revert since we already talked in #8197.
---
.../src/main/scala/org/apache/gluten/config/VeloxConfig.scala | 4 ++--
cpp/velox/compute/VeloxBackend.cc | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git
a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
index e1f1c2880e..9fd8064d6e 100644
--- a/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
+++ b/backends-velox/src/main/scala/org/apache/gluten/config/VeloxConfig.scala
@@ -406,10 +406,10 @@ object VeloxConfig {
val LOAD_QUANTUM =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.loadQuantum")
.internal()
- .doc("Set the load quantum for velox file scan, recommend to use the
default value (8MB) " +
+ .doc("Set the load quantum for velox file scan, recommend to use the
default value (256MB) " +
"for performance consideration. If Velox cache is enabled, it can be
8MB at most.")
.bytesConf(ByteUnit.BYTE)
- .createWithDefaultString("8MB")
+ .createWithDefaultString("256MB")
val MAX_COALESCED_DISTANCE_BYTES =
buildStaticConf("spark.gluten.sql.columnar.backend.velox.maxCoalescedDistance")
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index cd62da5c66..c516a8460e 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -265,7 +265,7 @@ void VeloxBackend::initConnector() {
connectorConfMap[velox::connector::hive::HiveConfig::kPrefetchRowGroups] =
backendConf_->get<std::string>(kPrefetchRowGroups, "1");
connectorConfMap[velox::connector::hive::HiveConfig::kLoadQuantum] =
- backendConf_->get<std::string>(kLoadQuantum, "8388608"); // 8M
+ backendConf_->get<std::string>(kLoadQuantum, "268435456"); // 256M
connectorConfMap[velox::connector::hive::HiveConfig::kFooterEstimatedSize] =
backendConf_->get<std::string>(kDirectorySizeGuess, "32768"); // 32K
connectorConfMap[velox::connector::hive::HiveConfig::kFilePreloadThreshold] =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
