This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 32872277746 fix compaction log is not deleted after exception (#11855)
32872277746 is described below
commit 32872277746c5269d263d6d4fc7f91d96d2a9a9d
Author: shuwenwei <[email protected]>
AuthorDate: Fri Jan 5 19:03:50 2024 +0800
fix compaction log is not deleted after exception (#11855)
---
.../execute/task/CrossSpaceCompactionTask.java | 3 +-
.../execute/task/InnerSpaceCompactionTask.java | 3 +-
.../compaction/ReadChunkInnerCompactionTest.java | 49 ++++++++++++++
.../cross/CrossSpaceCompactionExceptionTest.java | 79 ++++++++++++++++++++++
4 files changed, 132 insertions(+), 2 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
index 3f3ae1bc8cc..3c9cdee87ae 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/CrossSpaceCompactionTask.java
@@ -271,8 +271,9 @@ public class CrossSpaceCompactionTask extends
AbstractCompactionTask {
"%.2f",
(selectedSeqFileSize + selectedUnseqFileSize) / 1024.0d /
1024.0d / costTime),
summary);
+ } finally {
+ Files.deleteIfExists(logFile.toPath());
}
- Files.deleteIfExists(logFile.toPath());
} catch (Exception e) {
isSuccess = false;
printLogWhenException(LOGGER, e);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 4628c9e8f46..3d27622cf56 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -299,8 +299,9 @@ public class InnerSpaceCompactionTask extends
AbstractCompactionTask {
String.format("%.2f", costTime),
String.format("%.2f", selectedFileSize / 1024.0d / 1024.0d /
costTime),
summary);
+ } finally {
+ Files.deleteIfExists(logFile.toPath());
}
- Files.deleteIfExists(logFile.toPath());
} catch (Exception e) {
isSuccess = false;
printLogWhenException(LOGGER, e);
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
index 9c8a129be45..e87596c1c55 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/ReadChunkInnerCompactionTest.java
@@ -24,13 +24,16 @@ import org.apache.iotdb.commons.path.MeasurementPath;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.StorageEngineException;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadChunkCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.InnerSpaceCompactionTask;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
import org.apache.iotdb.db.storageengine.dataregion.modification.Deletion;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -47,11 +50,13 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createChunkWriter;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.TsFileGeneratorUtils.createCompressionType;
@@ -660,4 +665,48 @@ public class ReadChunkInnerCompactionTest extends
AbstractCompactionTest {
Assert.assertTrue(task.start());
Assert.assertEquals(1, tsFileManager.getTsFileList(true).size());
}
+
+ @Test
+ public void testCompactionLogIsDeletedAfterException() throws IOException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][][] {
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12), new
TimeRange(3, 12)}}
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1", new TimeRange[] {new TimeRange(1, 9)}, TSEncoding.PLAIN,
CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ tsFileManager.addAll(seqResources, true);
+
+ InnerSpaceCompactionTask task =
+ new InnerSpaceCompactionTask(
+ 0, tsFileManager, seqResources, true, new
FastCompactionPerformer(false), 0);
+ Assert.assertFalse(task.start());
+ Assert.assertFalse(
+ new File(
+ seqResource1
+ .getTsFile()
+ .getAbsolutePath()
+ .replace(
+ "0-0" + TsFileConstant.TSFILE_SUFFIX,
+ "1-0.inner" +
CompactionLogger.INNER_COMPACTION_LOG_NAME_SUFFIX))
+ .exists());
+ Assert.assertEquals(
+ 4,
Objects.requireNonNull(seqResource1.getTsFile().getParentFile().listFiles()).length);
+ }
}
diff --git
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
index 557155f047c..8a2df9b8a7a 100644
---
a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
+++
b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/cross/CrossSpaceCompactionExceptionTest.java
@@ -29,17 +29,22 @@ import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.FastCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.performer.impl.ReadPointCompactionPerformer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CompactionTaskSummary;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.subtask.FastCompactionTaskSummary;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionConfigRestorer;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionFileGeneratorUtils;
+import
org.apache.iotdb.db.storageengine.dataregion.compaction.utils.CompactionTestFileWriter;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.common.TimeRange;
import org.apache.iotdb.tsfile.utils.Pair;
import org.junit.After;
@@ -52,6 +57,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger.STR_SOURCE_FILES;
import static
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.log.CompactionLogger.STR_TARGET_FILES;
@@ -746,4 +752,77 @@ public class CrossSpaceCompactionExceptionTest extends
AbstractCompactionTest {
.exists());
}
}
+
+ @Test
+ public void testCompactionLogIsDeletedAfterException() throws IOException {
+ TsFileResource seqResource1 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][][] {
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12), new
TimeRange(3, 12)}}
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource seqResource2 = createEmptyFileAndResource(true);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(seqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1", new TimeRange[] {new TimeRange(1, 9)}, TSEncoding.PLAIN,
CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ seqResources.add(seqResource1);
+ seqResources.add(seqResource2);
+ tsFileManager.addAll(seqResources, true);
+ TsFileResource unseqResource1 = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(unseqResource1)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1",
+ new TimeRange[][][] {
+ new TimeRange[][] {new TimeRange[] {new TimeRange(10, 12), new
TimeRange(3, 12)}}
+ },
+ TSEncoding.PLAIN,
+ CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ TsFileResource unseqResource2 = createEmptyFileAndResource(false);
+ try (CompactionTestFileWriter writer = new
CompactionTestFileWriter(unseqResource2)) {
+ writer.startChunkGroup("d1");
+ writer.generateSimpleNonAlignedSeriesToCurrentDevice(
+ "s1", new TimeRange[] {new TimeRange(35, 40)}, TSEncoding.PLAIN,
CompressionType.LZ4);
+ writer.endChunkGroup();
+ writer.endFile();
+ }
+ unseqResources.add(unseqResource1);
+ unseqResources.add(unseqResource2);
+ tsFileManager.addAll(unseqResources, false);
+ CrossSpaceCompactionTask task =
+ new CrossSpaceCompactionTask(
+ 0,
+ tsFileManager,
+ seqResources,
+ unseqResources,
+ new FastCompactionPerformer(true),
+ 0,
+ 0);
+ Assert.assertFalse(task.start());
+ Assert.assertFalse(
+ new File(
+ seqResource1
+ .getTsFile()
+ .getAbsolutePath()
+ .replace(
+ "0-0" + TsFileConstant.TSFILE_SUFFIX,
+ "0-1.cross" +
CompactionLogger.CROSS_COMPACTION_LOG_NAME_SUFFIX))
+ .exists());
+ Assert.assertEquals(
+ 4,
Objects.requireNonNull(seqResource1.getTsFile().getParentFile().listFiles()).length);
+ }
}
