This is an automated email from the ASF dual-hosted git repository.
maobaolong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 78cb21bf1 [MINOR] improvement(server): Dynamic conf support of server
memory watermark (#2231)
78cb21bf1 is described below
commit 78cb21bf10b41ce7e023597a3386b939c1c0f80e
Author: maobaolong <[email protected]>
AuthorDate: Tue Nov 5 15:43:11 2024 +0800
[MINOR] improvement(server): Dynamic conf support of server memory
watermark (#2231)
### What changes were proposed in this pull request?
### Why are the changes needed?
It can be easy to tune the server online without restart leverage this.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Tested Locally.
```Console
➜ ~ curl -X POST http://localhost:19948/api/shuffleServer/confOps/update \
-H "Content-Type: application/json" \
-d '{"update":{"rss.server.memory.shuffle.highWaterMark.percentage": "0.5",
"rss.server.memory.shuffle.lowWaterMark.percentage": "0.1"}}'
temporarily effective until restart: Update successfully
```
server.log
```
[2024-11-03 21:29:20.660] [Jetty-5] [INFO] ConfOpsResource - Dynamic
updating ConfVO{update={rss.server.memory.shuffle.highWaterMark.percentage=0.5,
rss.server.memory.shuffle.lowWaterMark.percentage=0.1}, delete=[]}
```
<img width="1116" alt="snapshot_b6b93cf5-bf97-4efb-9f20-f106c9278b27"
src="https://github.com/user-attachments/assets/f2d03ac3-16a9-4927-8fe9-12e44a7fa231";>
---
.../server/buffer/ShuffleBufferManager.java | 28 ++++++++++++++++++++++
1 file changed, 28 insertions(+)
diff --git
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
index b4c9b9ef3..7d7b98ac9 100644
---
a/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
+++
b/server/src/main/java/org/apache/uniffle/server/buffer/ShuffleBufferManager.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.uniffle.common.ReconfigurableConfManager;
+import org.apache.uniffle.common.ReconfigurableRegistry;
import org.apache.uniffle.common.ShuffleDataResult;
import org.apache.uniffle.common.ShufflePartitionedData;
import org.apache.uniffle.common.rpc.StatusCode;
@@ -176,6 +177,33 @@ public class ShuffleBufferManager {
ShuffleServerMetrics.addLabeledGauge(
SHUFFLE_COUNT_IN_BUFFER_POOL,
() -> bufferPool.values().stream().mapToLong(innerMap ->
innerMap.size()).sum());
+ ReconfigurableRegistry.register(
+ Sets.newHashSet(
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_HIGHWATERMARK_PERCENTAGE.key(),
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_LOWWATERMARK_PERCENTAGE.key()),
+ (theConf, changedProperties) -> {
+ if (changedProperties == null) {
+ return;
+ }
+ if (changedProperties.contains(
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_HIGHWATERMARK_PERCENTAGE.key())) {
+ this.highWaterMark =
+ (long)
+ (capacity
+ / 100.0
+ * conf.get(
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_HIGHWATERMARK_PERCENTAGE));
+ }
+ if (changedProperties.contains(
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_LOWWATERMARK_PERCENTAGE.key())) {
+ this.lowWaterMark =
+ (long)
+ (capacity
+ / 100.0
+ * conf.get(
+
ShuffleServerConf.SERVER_MEMORY_SHUFFLE_LOWWATERMARK_PERCENTAGE));
+ }
+ });
}
public void setShuffleTaskManager(ShuffleTaskManager taskManager) {
