This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch rc/1.3.1 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a06ff5968946e316647560f6b10953cd0450e39b Author: shuwenwei <[email protected]> AuthorDate: Thu Apr 11 10:52:09 2024 +0800 Do not set read only in compaction module (#12311) --- .../execute/exception/CompactionExceptionHandler.java | 15 +++++---------- .../execute/recover/CompactionRecoverTask.java | 18 ++++-------------- .../execute/task/AbstractCompactionTask.java | 10 +++++----- .../execute/task/InnerSpaceCompactionTask.java | 2 +- .../impl/RewriteCrossSpaceCompactionSelector.java | 2 +- .../inner/InnerSpaceCompactionExceptionTest.java | 5 +---- 6 files changed, 17 insertions(+), 35 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java index 12039324339..5c663e37955 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java @@ -102,23 +102,19 @@ public class CompactionExceptionHandler { if (!handleSuccess) { LOGGER.error( "[Compaction][ExceptionHandler] Fail to handle {} space compaction exception, " - + "set allowCompaction to false in {}", + + "storage group is {}", compactionType, fullStorageGroupName); - tsFileManager.setAllowCompaction(false); } else { FileUtils.delete(logFile); } } catch (IOException e) { // catch throwable when handling exception - // set the allowCompaction to false LOGGER.error( "[Compaction][ExceptionHandler] exception occurs when handling exception in {} space compaction. " - + "Set allowCompaction to false in {}", + + "storage group is {}", compactionType, - fullStorageGroupName, - e); - tsFileManager.setAllowCompaction(false); + fullStorageGroupName); } } @@ -209,7 +205,7 @@ public class CompactionExceptionHandler { /** * Some source files are lost, check if all target files are complete. If all target files are * complete, delete the remaining source files and compaction mods files. If some target files are - * not complete, set the allowCompaction in tsFileManager to false and print some error logs. + * not complete, print some error logs. * * @throws IOException if the io operations on file fails */ @@ -252,8 +248,7 @@ public class CompactionExceptionHandler { if (!TsFileUtils.isTsFileComplete(targetResource.getTsFile())) { LOGGER.error( "{} [Compaction][ExceptionHandler] target file {} is not complete, " - + "and some source files {} is lost, do nothing. " - + "Set allowCompaction to false", + + "and some source files {} is lost, do nothing.", fullStorageGroupName, targetResource, lostSourceResources); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java index 7f238ba02f7..a504df6e7cb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover; -import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils; @@ -113,10 +111,7 @@ public class CompactionRecoverTask { LOGGER.error("Recover compaction error", e); } finally { if (!recoverSuccess) { - LOGGER.error( - "{} [Compaction][Recover] Failed to recover compaction, set allowCompaction to false", - fullStorageGroupName); - tsFileManager.setAllowCompaction(false); + LOGGER.error("{} [Compaction][Recover] Failed to recover compaction", fullStorageGroupName); } else { if (compactionLogFile.exists()) { try { @@ -127,12 +122,10 @@ public class CompactionRecoverTask { FileUtils.delete(compactionLogFile); } catch (IOException e) { LOGGER.error( - "{} [Compaction][Recover] Exception occurs while deleting log file {}, " - + "set allowCompaction to false", + "{} [Compaction][Recover] Exception occurs while deleting log file {}", fullStorageGroupName, compactionLogFile, e); - tsFileManager.setAllowCompaction(false); } } } @@ -190,8 +183,7 @@ public class CompactionRecoverTask { CompactionUtils.deleteCompactionModsFile(sourceTsFileResourceList, Collections.emptyList()); } catch (IOException e) { LOGGER.error( - "{} [Compaction][Recover] Exception occurs while deleting compaction mods file, " - + "set allowCompaction to false", + "{} [Compaction][Recover] Exception occurs while deleting compaction mods file", fullStorageGroupName, e); return false; @@ -265,11 +257,9 @@ public class CompactionRecoverTask { || !TsFileUtils.isTsFileComplete(new TsFileResource(targetFile).getTsFile())) { LOGGER.error( "{} [Compaction][ExceptionHandler] target file {} is not complete, " - + "and some source files is lost, do nothing. " - + "Set allowCompaction to false", + + "and some source files is lost, do nothing.", fullStorageGroupName, targetFileIdentifier.getFilePath()); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java index 7f33d7eaf3f..209680b5a1a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java @@ -19,8 +19,6 @@ package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task; -import org.apache.iotdb.commons.cluster.NodeStatus; -import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.conf.IoTDBConstant; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.service.metrics.CompactionMetrics; @@ -319,9 +317,11 @@ public abstract class AbstractCompactionTask { dataRegionId, this, e); - tsFileManager.setAllowCompaction(false); - LOGGER.error("stop compaction because of exception during recovering"); - CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly); + // Do not set allow compaction to false here. To keep the error environment, mark all source + // files in memory to avoid compaction. + for (TsFileResource sourceTsFileResource : getAllSourceTsFiles()) { + sourceTsFileResource.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR); + } } protected void insertFilesToTsFileManager(List<TsFileResource> tsFiles) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java index 7fcc419db46..9de3917a9ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java @@ -172,7 +172,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask { recoverMemoryStatus = true; LOGGER.info( "{}-{} [Compaction] {} InnerSpaceCompaction task starts with {} files, " - + "total file size is {} MB, memory cost is {} MB", + + "total file size is {} MB, estimated memory cost is {} MB", storageGroupName, dataRegionId, sequence ? "Sequence" : "Unsequence", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java index fdc1ea8fbf4..64807c0c913 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java @@ -362,7 +362,7 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector "{} [{}] Total source files: {} seqFiles, {} unseqFiles. " + "Candidate source files: {} seqFiles, {} unseqFiles. " + "Selected source files: {} seqFiles, " - + "{} unseqFiles, total memory cost {} MB, " + + "{} unseqFiles, estimated memory cost {} MB, " + "total selected file size is {} MB, " + "total selected seq file size is {} MB, " + "total selected unseq file size is {} MB, " diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java index aa54be5766a..1087c56863b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java @@ -215,8 +215,7 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact } /** - * Test some source files are lost and target file is not complete. System should be set to read - * only at this time. + * Test some source files are lost and target file is not complete. * * @throws Exception */ @@ -265,8 +264,6 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact Assert.assertTrue(resource.resourceFileExists()); Assert.assertTrue(resource.getTsFile().exists()); } - - Assert.assertFalse(tsFileManager.isAllowCompaction()); } /**
