This is an automated email from the ASF dual-hosted git repository. marklau99 pushed a commit to branch IOTDB-5092 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e512a9a7c062eb30388c1316cb2540f6f2e7cfa8 Author: Liu Xuxin <[email protected]> AuthorDate: Wed Nov 30 18:41:06 2022 +0800 set cross_compaction to true and add compaction validation --- docs/UserGuide/Reference/Common-Config-Manual.md | 10 ++++++ .../zh/UserGuide/Reference/Common-Config-Manual.md | 9 +++++ .../resources/conf/iotdb-common.properties | 7 ++-- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++++ .../db/engine/compaction/CompactionUtils.java | 40 ++++++++++++++++++++++ .../compaction/cross/CrossSpaceCompactionTask.java | 12 +++++++ .../compaction/inner/InnerSpaceCompactionTask.java | 11 ++++++ 8 files changed, 103 insertions(+), 2 deletions(-) diff --git a/docs/UserGuide/Reference/Common-Config-Manual.md b/docs/UserGuide/Reference/Common-Config-Manual.md index 8356e196c6..504824ccdc 100644 --- a/docs/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/UserGuide/Reference/Common-Config-Manual.md @@ -1008,6 +1008,16 @@ Different configuration parameters take effect in the following three ways: |Default| 4 | |Effective| After restart system | +* enable\_compaction\_validation + +|Name| enable\_compaction\_validation | +|:---:|:----------------------------------------------------------------| +|Description| Enable the check of sequence tsfile time range after compaction | +|Type| Boolean | +|Default| true | +|Effective| After restart system | + + ### Write Ahead Log Configuration * wal\_mode diff --git a/docs/zh/UserGuide/Reference/Common-Config-Manual.md b/docs/zh/UserGuide/Reference/Common-Config-Manual.md index 0066cb1e43..3fa5556310 100644 --- a/docs/zh/UserGuide/Reference/Common-Config-Manual.md +++ b/docs/zh/UserGuide/Reference/Common-Config-Manual.md @@ -1063,6 +1063,15 @@ IoTDB ConfigNode 和 DataNode 的公共配置参数位于 `conf` 目录下。 |默认值| 4 | |改后生效方式| 重启服务生效| +* enable\_compaction\_validation + +|名字| enable\_compaction\_validation | +|:---:|:--| +|描述| 开启合并结束后对顺序文件时间范围的检查 | +|类型| Boolean | +|默认值| true | +|改后生效方式| 重启服务生效| + ### 写前日志配置 * wal\_mode diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties index deb66da80d..930ab46ba2 100644 --- a/node-commons/src/assembly/resources/conf/iotdb-common.properties +++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties @@ -532,9 +532,8 @@ # enable_unseq_space_compaction=true # cross space compaction: compact the unsequence files into the overlapped sequence files -# It is recommended to disable this in version 1.0.0, and can enable it in version 1.0.1 # Datatype: boolean -enable_cross_space_compaction=false +# enable_cross_space_compaction=true # the selector of cross space compaction task # Options: rewrite @@ -630,6 +629,10 @@ enable_cross_space_compaction=false # Datatype: int # sub_compaction_thread_count=4 +# Enable the check of sequence tsfile time range after compaction +# Datatype: boolean +# enable_compaction_validation=true + #################### ### Write Ahead Log Configuration #################### diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 6a1f46f760..0f923aef2e 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -487,6 +487,8 @@ public class IoTDBConfig { */ private int subCompactionTaskNum = 4; + private boolean enableCompactionValidation = true; + /** whether to cache meta data(ChunkMetaData and TsFileMetaData) or not. */ private boolean metaDataCacheEnable = true; @@ -3577,4 +3579,12 @@ public class IoTDBConfig { public void setSchemaRatisLogMaxMB(long schemaRatisLogMaxMB) { this.schemaRatisLogMaxMB = schemaRatisLogMaxMB; } + + public boolean isEnableCompactionValidation() { + return enableCompactionValidation; + } + + public void setEnableCompactionValidation(boolean enableCompactionValidation) { + this.enableCompactionValidation = enableCompactionValidation; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 5373b4d7b0..bba0dd92c5 100644 --- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -673,6 +673,12 @@ public class IoTDBDescriptor { "compaction_write_throughput_mb_per_sec", Integer.toString(conf.getCompactionWriteThroughputMbPerSec())))); + conf.setEnableCompactionValidation( + Boolean.parseBoolean( + properties.getProperty( + "enable_compaction_validation", + Boolean.toString(conf.isEnableCompactionValidation())))); + conf.setEnablePartialInsert( Boolean.parseBoolean( properties.getProperty( diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java index bb114710f5..d4b5a33e60 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/CompactionUtils.java @@ -21,6 +21,7 @@ package org.apache.iotdb.db.engine.compaction; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.engine.modification.Modification; import org.apache.iotdb.db.engine.modification.ModificationFile; +import org.apache.iotdb.db.engine.storagegroup.TsFileManager; import org.apache.iotdb.db.engine.storagegroup.TsFileResource; import org.apache.iotdb.db.query.control.FileReaderManager; import org.apache.iotdb.tsfile.common.constant.TsFileConstant; @@ -35,8 +36,10 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -256,4 +259,41 @@ public class CompactionUtils { } } } + + public static boolean validateTsFileResources( + TsFileManager manager, String storageGroupName, long timePartition) { + List<TsFileResource> resources = + manager.getSequenceListByTimePartition(timePartition).getArrayList(); + resources.sort( + (f1, f2) -> + Long.compareUnsigned( + Long.parseLong(f1.getTsFile().getName().split("-")[0]), + Long.parseLong(f2.getTsFile().getName().split("-")[0]))); + Map<String, Long> lastEndTimeMap = new HashMap<>(); + TsFileResource prevTsFileResource = null; + for (TsFileResource resource : resources) { + Set<String> devices = resource.getDevices(); + for (String device : devices) { + long currentStartTime = resource.getStartTime(device); + long currentEndTime = resource.getEndTime(device); + long lastEndTime = lastEndTimeMap.computeIfAbsent(device, x -> Long.MIN_VALUE); + if (lastEndTime >= currentStartTime) { + logger.error( + "{} Device {} is overlapped between {} and {}, end time in {} is {}, start time in {} is {}", + storageGroupName, + device, + prevTsFileResource, + resource, + prevTsFileResource, + lastEndTime, + resource, + currentStartTime); + return false; + } + lastEndTimeMap.put(device, currentEndTime); + } + prevTsFileResource = resource; + } + return true; + } } diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java index 960073aea3..4a555525d3 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/cross/CrossSpaceCompactionTask.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.compaction.cross; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.TsFileMetricManager; import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; @@ -169,6 +170,17 @@ public class CrossSpaceCompactionTask extends AbstractCompactionTask { timePartition, true); + if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation() + && !CompactionUtils.validateTsFileResources( + tsFileManager, storageGroupName, timePartition)) { + LOGGER.error( + "Failed to pass compaction validation, source sequence files is: {}, unsequence files is {}, target files is {}", + selectedSequenceFiles, + selectedUnsequenceFiles, + targetTsfileResourceList); + throw new RuntimeException("Failed to pass compaction validation"); + } + releaseReadAndLockWrite(selectedSequenceFiles); releaseReadAndLockWrite(selectedUnsequenceFiles); diff --git a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java index 51d8554940..ca78d5d694 100644 --- a/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java +++ b/server/src/main/java/org/apache/iotdb/db/engine/compaction/inner/InnerSpaceCompactionTask.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.engine.compaction.inner; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.engine.TsFileMetricManager; import org.apache.iotdb.db.engine.compaction.CompactionExceptionHandler; import org.apache.iotdb.db.engine.compaction.CompactionUtils; @@ -177,6 +178,16 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { false); } + if (IoTDBDescriptor.getInstance().getConfig().isEnableCompactionValidation() + && !CompactionUtils.validateTsFileResources( + tsFileManager, storageGroupName, timePartition)) { + LOGGER.error( + "Failed to pass compaction validation, source files is: {}, target files is {}", + selectedTsFileResourceList, + targetTsFileList); + throw new RuntimeException("Failed to pass compaction validation"); + } + LOGGER.info( "{}-{} [Compaction] Compacted target files, try to get the write lock of source files", storageGroupName,
