(incubator-uniffle) branch master updated: [#2362] feat(client): Log more compression infos (#2367)

[zuston]

This is an automated email from the ASF dual-hosted git repository.

zuston pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new 608471a12 [#2362] feat(client): Log more compression infos (#2367)
608471a12 is described below

commit 608471a1270098d058a10563322dae2259d7042a
Author: Junfan Zhang <zus...@apache.org>
AuthorDate: Tue Feb 11 14:12:46 2025 +0800

    [#2362] feat(client): Log more compression infos (#2367)
    
    ### What changes were proposed in this pull request?
    
     Followup to log more compression infos like data len and compression ratio
    
    ### Why are the changes needed?
    
    followup #2362
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Needn't
    
    ---------
    
    Co-authored-by: Junfan Zhang <zhangjun...@qiyi.com>
---
 .../org/apache/spark/shuffle/writer/WriteBufferManager.java    | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git 
a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
 
b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
index 7e4e893e3..cf4b4bc51 100644
--- 
a/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
+++ 
b/client-spark/common/src/main/java/org/apache/spark/shuffle/writer/WriteBufferManager.java
@@ -92,6 +92,7 @@ public class WriteBufferManager extends MemoryConsumer {
   private SerializationStream serializeStream;
   private WrappedByteArrayOutputStream arrayOutputStream;
   private long uncompressedDataLen = 0;
+  private long compressedDataLen = 0;
   private long requireMemoryInterval;
   private int requireMemoryRetryMax;
   private Optional<Codec> codec;
@@ -430,6 +431,7 @@ public class WriteBufferManager extends MemoryConsumer {
         blockIdLayout.getBlockId(getNextSeqNo(partitionId), partitionId, 
taskAttemptId);
     blockCounter.incrementAndGet();
     uncompressedDataLen += data.length;
+    compressedDataLen += compressed.length;
     shuffleWriteMetrics.incBytesWritten(compressed.length);
     // add memory to indicate bytes which will be sent to shuffle server
     inSendListBytes.addAndGet(wb.getMemoryUsed());
@@ -648,14 +650,18 @@ public class WriteBufferManager extends MemoryConsumer {
         + serializeTime
         + "], sortTime["
         + sortTime
-        + "], compressTime["
-        + compressTime
         + "], estimateTime["
         + estimateTime
         + "], requireMemoryTime["
         + requireMemoryTime
         + "], uncompressedDataLen["
         + uncompressedDataLen
+        + "], compressedDataLen["
+        + compressedDataLen
+        + "], compressTime["
+        + compressTime
+        + "], compressRatio["
+        + (compressedDataLen == 0 ? 0 : (float) uncompressedDataLen / 
compressedDataLen)
         + "]";
   }