This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch adapt_wal_1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 4003a3448cb1bf9c5991145bffa7d923f7d85ff3 Author: HTHou <[email protected]> AuthorDate: Wed Aug 14 12:13:53 2024 +0800 add more tests and fix wal with broken v3 tsfile --- .../file/AbstractTsFileRecoverPerformer.java | 5 ++++ .../wal/recover/WALRecoverManagerTest.java | 32 +++++++++++++++++---- .../src/test/resources/oldwal/1723544967972-1-0-0 | Bin 0 -> 237 bytes .../resources/oldwal/{2147483647 => }/_0-0-0.wal | Bin .../oldwal/{2147483647 => }/_0.checkpoint | Bin .../resources/oldwal/{2147483647 => }/_1-0-0.wal | Bin .../resources/oldwal/{2147483647 => }/_2-0-0.wal | Bin .../resources/oldwal/{2147483647 => }/_3-0-1.wal | Bin 8 files changed, 31 insertions(+), 6 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java index 0880697b02c..cec93e0ead4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/AbstractTsFileRecoverPerformer.java @@ -35,6 +35,7 @@ import org.slf4j.LoggerFactory; import java.io.Closeable; import java.io.File; import java.io.IOException; +import java.nio.file.Files; /** This class is used to help recover TsFile. */ public abstract class AbstractTsFileRecoverPerformer implements Closeable { @@ -85,6 +86,10 @@ public abstract class AbstractTsFileRecoverPerformer implements Closeable { boolean result = tsFile.delete(); logger.warn( "TsFile {} is incompatible. Try to delete it and delete result is {}", tsFile, result); + // if the broken TsFile is v3, we can recover the all data from wal + // to support it, we can regenerate an empty file here + Files.createFile(tsFile.toPath()); + writer = new RestorableTsFileIOWriter(tsFile); throw new DataRegionException(e); } catch (IOException e) { throw new DataRegionException(e); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index b2d462e10f3..50233122779 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -354,19 +354,30 @@ public class WALRecoverManagerTest { } @Test - public void testMemTableSnapshotFromOldVersion() throws Exception { + public void testRecoverOldWalWithEmptyTsFile() throws Exception { // old version of wal is generated by prepareCheckpointAndWALFileForSnapshot() in v1.3 - String oldWalPathStr = - this.getClass().getClassLoader().getResource("oldwal/2147483647").getFile(); + String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile(); File oldWalFileDir = new File(oldWalPathStr); FileUtils.copyDir(oldWalFileDir, new File(WAL_NODE_FOLDER)); walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER); checkpointManager = walBuffer.getCheckpointManager(); WALRecoverManager.getInstance().clear(); - recoverFromOldWalAndCheck(); + recoverFromOldWalAndCheck(false); } - private void recoverFromOldWalAndCheck() throws Exception { + @Test + public void testRecoverOldWalWithBrokenTsFile() throws Exception { + // old version of wal is generated by prepareCheckpointAndWALFileForSnapshot() in v1.3 + String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile(); + File oldWalFileDir = new File(oldWalPathStr); + FileUtils.copyDir(oldWalFileDir, new File(WAL_NODE_FOLDER)); + walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER); + checkpointManager = walBuffer.getCheckpointManager(); + WALRecoverManager.getInstance().clear(); + recoverFromOldWalAndCheck(true); + } + + private void recoverFromOldWalAndCheck(boolean withBrokenTsFile) throws Exception { // prepare tsFiles List<WALRecoverListener> recoverListeners = new ArrayList<>(); @@ -374,7 +385,16 @@ public class WALRecoverManagerTest { File fileWithWALDir = new File(FILE_WITH_WAL_NAME).getParentFile(); Files.createDirectory(fileWithWALDir.toPath()); File fileWithWAL = new File(fileWithWALDir, "1723544967972-1-0-0.tsfile"); - Files.createFile(fileWithWAL.toPath()); + ; + if (withBrokenTsFile) { + // copy a broken TsFileV3 + String oldWalPathStr = this.getClass().getClassLoader().getResource("oldwal").getFile(); + File oldWalFileDir = new File(oldWalPathStr); + FileUtils.copyFile(new File(oldWalFileDir, "1723544967972-1-0-0"), fileWithWAL); + } else { + // create an empty tsfile + Files.createFile(fileWithWAL.toPath()); + } tsFileWithWALResource = new TsFileResource(fileWithWAL); UnsealedTsFileRecoverPerformer recoverPerformer = new UnsealedTsFileRecoverPerformer( diff --git a/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0 b/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0 new file mode 100644 index 00000000000..f41e64de04b Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/1723544967972-1-0-0 differ diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_0-0-0.wal similarity index 100% rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal rename to iotdb-core/datanode/src/test/resources/oldwal/_0-0-0.wal diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint b/iotdb-core/datanode/src/test/resources/oldwal/_0.checkpoint similarity index 100% rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint rename to iotdb-core/datanode/src/test/resources/oldwal/_0.checkpoint diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_1-0-0.wal similarity index 100% rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal rename to iotdb-core/datanode/src/test/resources/oldwal/_1-0-0.wal diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/_2-0-0.wal similarity index 100% rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal rename to iotdb-core/datanode/src/test/resources/oldwal/_2-0-0.wal diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal b/iotdb-core/datanode/src/test/resources/oldwal/_3-0-1.wal similarity index 100% rename from iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal rename to iotdb-core/datanode/src/test/resources/oldwal/_3-0-1.wal
