[iotdb] branch rel/1.1 updated: [IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking whether the tsfile data loaded locally is in the same time partition during the loading process & LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10284)

[rong]

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch rel/1.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/rel/1.1 by this push:
     new dafd53c8a85 [IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking 
whether the tsfile data loaded locally is in the same time partition during the 
loading process & LoadTsFilePieceNode error when loading tsfile with empty 
value chunks (#10284)
dafd53c8a85 is described below

commit dafd53c8a85246f8dff9e5c5536bf5eb051a9427
Author: Steve Yurong Su <r...@apache.org>
AuthorDate: Sat Jun 24 01:32:19 2023 +0800

    [IOTDB-6015][IOTDB-6023] load TsFile bugs: Not checking whether the tsfile 
data loaded locally is in the same time partition during the loading process & 
LoadTsFilePieceNode error when loading tsfile with empty value chunks (#10284)
    
    * [IOTDB-6015] Pipe: Not checking whether the tsfile data loaded locally is 
in the same time partition during the loading process (#10249)
    
    When determining whether a tsfile should be loaded locally without 
splitting, you should also check that all the data in this tsfile is in the 
same dataregion and the same time division, without previously checking the 
latter
    
    (cherry picked from commit 56faa8ef61b2cae499d1ef42fe994a8c4c9bc9c2)
    
    * [IOTDB-6023] Pipe: LoadTsFilePieceNode error when loading tsfile with 
empty value chunks (#10281)
    
    (cherry picked from commit 11755a10f5f237ba5dbdc4c4562222117e06dc9c)
    
    ---------
    
    Co-authored-by: yschengzi <87161145+yschen...@users.noreply.github.com>
---
 .../java/org/apache/iotdb/db/engine/load/TsFileSplitter.java |  8 ++++++--
 .../plan/planner/plan/node/load/LoadSingleTsFileNode.java    | 12 +++++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git 
a/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java 
b/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
index e3010f0eeb2..919c372c631 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/load/TsFileSplitter.java
@@ -229,7 +229,7 @@ public class TsFileSplitter {
             chunkMetadata = offset2ChunkMetadata.get(chunkOffset - Byte.BYTES);
             header = reader.readChunkHeader(marker);
             if (header.getDataSize() == 0) {
-              handleEmptyValueChunk(header, pageIndex2ChunkData);
+              handleEmptyValueChunk(header, pageIndex2ChunkData, 
chunkMetadata);
               break;
             }
 
@@ -421,12 +421,16 @@ public class TsFileSplitter {
   }
 
   private void handleEmptyValueChunk(
-      ChunkHeader header, Map<Integer, List<AlignedChunkData>> 
pageIndex2ChunkData) {
+      ChunkHeader header,
+      Map<Integer, List<AlignedChunkData>> pageIndex2ChunkData,
+      IChunkMetadata chunkMetadata)
+      throws IOException {
     Set<ChunkData> allChunkData = new HashSet<>();
     for (Map.Entry<Integer, List<AlignedChunkData>> entry : 
pageIndex2ChunkData.entrySet()) {
       for (AlignedChunkData alignedChunkData : entry.getValue()) {
         if (!allChunkData.contains(alignedChunkData)) {
           alignedChunkData.addValueChunk(header);
+          alignedChunkData.writeEntireChunk(ByteBuffer.allocate(0), 
chunkMetadata);
           allChunkData.add(alignedChunkData);
         }
       }
diff --git 
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
 
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
index d68184b04d3..18627c7be19 100644
--- 
a/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
+++ 
b/server/src/main/java/org/apache/iotdb/db/mpp/plan/planner/plan/node/load/LoadSingleTsFileNode.java
@@ -86,7 +86,17 @@ public class LoadSingleTsFileNode extends WritePlanNode {
               slotList.add(
                   new Pair<>(o, 
TimePartitionUtils.getTimePartition(resource.getEndTime(o))));
             });
-    needDecodeTsFile = !isDispatchedToLocal(new 
HashSet<>(partitionFetcher.apply(slotList)));
+
+    if (slotList.isEmpty()) {
+      throw new IllegalStateException(
+          String.format("Devices in TsFile %s is empty, this should not happen 
here.", tsFile));
+    } else if (slotList.stream()
+        .anyMatch(slotPair -> 
!slotPair.getRight().equals(slotList.get(0).right))) {
+      needDecodeTsFile = true;
+    } else {
+      needDecodeTsFile = !isDispatchedToLocal(new 
HashSet<>(partitionFetcher.apply(slotList)));
+    }
+
     if (!needDecodeTsFile && !resource.resourceFileExists()) {
       resource.serialize();
     }