This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 1df2e4454f [improvememt](file-cache) increase virtual node number to
make file cache more even (#24143)
1df2e4454f is described below
commit 1df2e4454f1a80609b7fb5b73a601efd474e27da
Author: Mingyu Chen <[email protected]>
AuthorDate: Sun Sep 10 19:56:53 2023 +0800
[improvememt](file-cache) increase virtual node number to make file cache
more even (#24143)
The origin virtual number is Math.max(Math.min(512 / backends.size(), 32),
2);, which is too small,
causing uneven cache distribution when enabling file cache.
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 8 ++++++++
.../apache/doris/planner/external/FederationBackendPolicy.java | 3 +--
2 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index a59a8e8d0a..54a0e814a4 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2157,4 +2157,12 @@ public class Config extends ConfigBase {
"是否禁止LocalDeployManager删除节点",
"Whether to disable LocalDeployManager drop node"})
public static boolean disable_local_deploy_manager_drop_node = true;
+
+ @ConfField(mutable = true, description = {
+ "开启 file cache 后，一致性哈希算法中，每个节点的虚拟节点数。"
+ + "该值越大，哈希算法的分布越均匀，但是会增加内存开销。",
+ "When file cache is enabled, the number of virtual nodes of each
node in the consistent hash algorithm. "
+ + "The larger the value, the more uniform the distribution
of the hash algorithm, "
+ + "but it will increase the memory overhead."})
+ public static int virtual_node_number = 2048;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
index eb2545affd..04c39fbf96 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/FederationBackendPolicy.java
@@ -96,9 +96,8 @@ public class FederationBackendPolicy {
throw new UserException("No available backends");
}
backendMap.putAll(backends.stream().collect(Collectors.groupingBy(Backend::getHost)));
- int virtualNumber = Math.max(Math.min(512 / backends.size(), 32), 2);
consistentHash = new ConsistentHash<>(Hashing.murmur3_128(), new
ScanRangeHash(),
- new BackendHash(), backends, virtualNumber);
+ new BackendHash(), backends, Config.virtual_node_number);
}
public Backend getNextBe() {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
