This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch fix-cmt-slow in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 843ade31bd51083a644d41e14187c58775d66b70 Author: Liu Xuxin <[email protected]> AuthorDate: Mon Sep 26 16:28:04 2022 +0800 reset currentChunkSize when flush out chunk metadata --- .../org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java | 9 +++++---- .../org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java | 9 +++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index 9ee1f7f566..76ead0f1a7 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -285,10 +285,11 @@ public class ChunkMetadata implements IChunkMetadata { } public long calculateRamSize() { - return CHUNK_METADATA_FIXED_RAM_SIZE - + RamUsageEstimator.sizeOf(tsFilePrefixPath) - + RamUsageEstimator.sizeOf(measurementUid) - + statistics.calculateRamSize(); + long memSize = CHUNK_METADATA_FIXED_RAM_SIZE; + memSize += RamUsageEstimator.sizeOf(tsFilePrefixPath); + memSize += RamUsageEstimator.sizeOf(measurementUid); + memSize += statistics.calculateRamSize(); + return memSize; } public static long calculateRamSize(String measurementId, TSDataType dataType) { diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java index 6cf00f8b84..2a3b2afbb6 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java @@ -117,6 +117,7 @@ public class TsFileIOWriter implements AutoCloseable { protected boolean enableMemoryControl = false; private Path lastSerializePath = null; protected LinkedList<Long> endPosInCMTForDevice = new LinkedList<>(); + private volatile int chunkMetadataCount = 0; public static final String CHUNK_METADATA_TEMP_FILE_SUFFIX = ".meta"; /** empty construct function. */ @@ -287,6 +288,7 @@ public class TsFileIOWriter implements AutoCloseable { if (enableMemoryControl) { this.currentChunkMetadataSize += currentChunkMetadata.calculateRamSize(); } + chunkMetadataCount++; chunkMetadataList.add(currentChunkMetadata); currentChunkMetadata = null; } @@ -609,7 +611,14 @@ public class TsFileIOWriter implements AutoCloseable { // This function should be called after all data of an aligned device has been written if (enableMemoryControl && currentChunkMetadataSize > maxMetadataSize) { try { + logger.debug( + "Flushing chunk metadata, total size is {}, count is {}, avg size is {}", + currentChunkMetadataSize, + chunkMetadataCount, + currentChunkMetadataSize / chunkMetadataCount); sortAndFlushChunkMetadata(); + chunkMetadataCount = 0; + currentChunkMetadataSize = 0; } catch (IOException e) { logger.error("Meets exception when flushing metadata to temp file for {}", file, e); throw e;
