This is an automated email from the ASF dual-hosted git repository. leirui pushed a commit to branch research/M4-visualization in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 84bd6f1e356cb92922f359873fe3cc4b207a12c5 Author: Lei Rui <[email protected]> AuthorDate: Fri Jan 27 23:27:31 2023 +0800 update modifyChunkMetadata optimize the double-loop for chunkmetadata and deletions --- .../resources/conf/iotdb-engine.properties | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 8 ++++---- .../java/org/apache/iotdb/db/utils/QueryUtils.java | 24 ++++++++++++++-------- .../iotdb/tsfile/file/metadata/ChunkMetadata.java | 7 ++++++- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/server/src/assembly/resources/conf/iotdb-engine.properties b/server/src/assembly/resources/conf/iotdb-engine.properties index 209e4fc389..4a9b6799ad 100644 --- a/server/src/assembly/resources/conf/iotdb-engine.properties +++ b/server/src/assembly/resources/conf/iotdb-engine.properties @@ -20,7 +20,7 @@ #################### ### enable CPV #################### -enable_CPV=false +enable_CPV=true #################### ### Web Page Configuration diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index e9d333f0e8..f45461a069 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -266,10 +266,10 @@ public class IoTDBConfig { private String indexRootFolder = "data" + File.separator + "index"; /** When a unSequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */ - private long unSeqTsFileSize = 1L; + private long unSeqTsFileSize = 1073741824; /** When a sequence TsFile's file size (in byte) exceed this, the TsFile is forced closed. */ - private long seqTsFileSize = 1L; + private long seqTsFileSize = 1073741824; /** When a memTable's size (in byte) exceeds this, the memtable is flushed to disk. */ private long memtableSizeThreshold = 1024 * 1024 * 1024L; @@ -328,7 +328,7 @@ public class IoTDBConfig { private int mergePagePointNumberThreshold = 100; /** LEVEL_COMPACTION, NO_COMPACTION */ - private CompactionStrategy compactionStrategy = CompactionStrategy.LEVEL_COMPACTION; + private CompactionStrategy compactionStrategy = CompactionStrategy.NO_COMPACTION; /** * Works when the compaction_strategy is LEVEL_COMPACTION. Whether to merge unseq files into seq @@ -431,7 +431,7 @@ public class IoTDBConfig { /** Is performance tracing enable. */ private boolean enablePerformanceTracing = false; - private boolean enableCPV = false; + private boolean enableCPV = true; /** The display of stat performance interval in ms. */ private long performanceStatDisplayInterval = 60000; diff --git a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java index 8da3273068..9863716ba0 100644 --- a/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/utils/QueryUtils.java @@ -54,18 +54,24 @@ public class QueryUtils { List<Deletion> deletions = modifications.stream().map(e -> (Deletion) e).sorted().collect(Collectors.toList()); + // sort chunkMetadatas by startTime + List<ChunkMetadata> sortedChunkMetadata = + chunkMetaData.stream().map(e -> (ChunkMetadata) e).sorted().collect(Collectors.toList()); + + // TODO add startPos for chunk to iterate deletes to speed up + int deleteStartIdx = 0; // 对于每个chunkMetadata，先过滤掉时间范围上不重叠的删除操作，然后再根据版本高低判断是否应用mod - for (int metaIndex = 0; metaIndex < chunkMetaData.size(); metaIndex++) { - ChunkMetadata metaData = chunkMetaData.get(metaIndex); - long startTime = metaData.getStartTime(); - long endTime = metaData.getEndTime(); - for (Deletion deletion : deletions) { - long deleteStartTime = deletion.getStartTime(); - long deleteEndTime = deletion.getEndTime(); - if (deleteStartTime > endTime) { + for (int metaIndex = 0; metaIndex < sortedChunkMetadata.size(); metaIndex++) { + ChunkMetadata metaData = sortedChunkMetadata.get(metaIndex); + // for (Deletion deletion : deletions) { + for (int j = deleteStartIdx; j < deletions.size(); j++) { + // TODO use deleteStartIdx to avoid iterate from the first delete + Deletion deletion = deletions.get(j); + if (deletion.getStartTime() > metaData.getEndTime()) { break; } - if (startTime > deleteEndTime) { + if (metaData.getStartTime() > deletion.getEndTime()) { + deleteStartIdx = j + 1; // TODO update startPos for chunk to iterate deletes to speed up continue; } // then deals with deletes that overlap in time with the chunk diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java index 4ce0d92e9b..60494e223a 100644 --- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java +++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/ChunkMetadata.java @@ -34,7 +34,7 @@ import java.util.List; import java.util.Objects; /** Metadata of one chunk. */ -public class ChunkMetadata { +public class ChunkMetadata implements Comparable<ChunkMetadata> { private String measurementUid; @@ -117,6 +117,11 @@ public class ChunkMetadata { // this.version = chunkMetadata.version; // } + @Override + public int compareTo(ChunkMetadata chunkMetadata) { + return Long.compare(statistics.getStartTime(), chunkMetadata.getStartTime()); + } + @Override public String toString() { return String.format(
