[hudi] branch master updated: [HUDI-1792] flink-client query error when processing files larger than 128mb (#2814)

[vinoyang]

This is an automated email from the ASF dual-hosted git repository.

vinoyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git


The following commit(s) were added to refs/heads/master by this push:
     new 62b8a34  [HUDI-1792] flink-client query error when processing files 
larger than 128mb (#2814)
62b8a34 is described below

commit 62b8a341ddae0ab80195c41c7a44b84c1fe23d31
Author: hj2016 <hj3245...@163.com>
AuthorDate: Fri Apr 16 13:59:19 2021 +0800

    [HUDI-1792] flink-client query error when processing files larger than 
128mb (#2814)
    
    Co-authored-by: huangjing <huangj...@clinbrain.com>
---
 .../org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git 
a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
 
b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
index 77f3e2a..477f54b 100644
--- 
a/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
+++ 
b/hudi-flink/src/main/java/org/apache/hudi/table/format/cow/CopyOnWriteInputFormat.java
@@ -40,6 +40,7 @@ import org.slf4j.LoggerFactory;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Comparator;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -209,7 +210,13 @@ public class CopyOnWriteInputFormat extends 
FileInputFormat<RowData> {
 
         // get the block locations and make sure they are in order with 
respect to their offset
         final BlockLocation[] blocks = fs.getFileBlockLocations(file, 0, len);
-        Arrays.sort(blocks);
+        Arrays.sort(blocks, new Comparator<BlockLocation>() {
+          @Override
+          public int compare(BlockLocation o1, BlockLocation o2) {
+            long diff = o1.getLength() - o2.getOffset();
+            return diff < 0L ? -1 : (diff > 0L ? 1 : 0);
+          }
+        });
 
         long bytesUnassigned = len;
         long position = 0;