(incubator-uniffle) branch master updated: improvement(spark): Record the partition choosing sort time for analysis (#2364)

[zuston]

This is an automated email from the ASF dual-hosted git repository.

zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new e5cfc4a02 improvement(spark): Record the partition choosing sort time 
for analysis (#2364)
e5cfc4a02 is described below

commit e5cfc4a02feef00918472ab149e8faa19ea6c417
Author: Junfan Zhang <zus...@apache.org>
AuthorDate: Fri Feb 7 17:58:30 2025 +0800

    improvement(spark): Record the partition choosing sort time for analysis 
(#2364)
    
    ### What changes were proposed in this pull request?
    
    Record the partition choosing sort time for analysis
    
    ### Why are the changes needed?
    
    For better optimize write duration
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Needn't
    
    Co-authored-by: Junfan Zhang <zhangjun...@qiyi.com>
---
 .../java/org/apache/spark/shuffle/writer/WriteBufferManager.java     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git 
a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
 
b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
index e91441f1c..7e4e893e3 100644
--- 
a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
+++ 
b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
@@ -85,6 +85,7 @@ public class WriteBufferManager extends MemoryConsumer {
   private long copyTime = 0;
   private long serializeTime = 0;
   private long compressTime = 0;
+  private long sortTime = 0;
   private long writeTime = 0;
   private long estimateTime = 0;
   private long requireMemoryTime = 0;
@@ -372,9 +373,11 @@ public class WriteBufferManager extends MemoryConsumer {
     bufferSpillRatio = Math.max(0.1, Math.min(1.0, bufferSpillRatio));
     List<Integer> partitionList = new ArrayList(buffers.keySet());
     if (Double.compare(bufferSpillRatio, 1.0) < 0) {
+      long start = System.currentTimeMillis();
       partitionList.sort(
           Comparator.comparingInt(o -> buffers.get(o) == null ? 0 : 
buffers.get(o).getMemoryUsed())
               .reversed());
+      sortTime += start;
       targetSpillSize = (long) ((getUsedBytes() - getInSendListBytes()) * 
bufferSpillRatio);
     }
 
@@ -643,6 +646,8 @@ public class WriteBufferManager extends MemoryConsumer {
         + writeTime
         + "], serializeTime["
         + serializeTime
+        + "], sortTime["
+        + sortTime
         + "], compressTime["
         + compressTime
         + "], estimateTime["