This is an automated email from the ASF dual-hosted git repository. haonan pushed a commit to branch adapt_wal_1.3 in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a4ac1fc4a4d969aab07b0b766cce74071d3bdd48 Author: HTHou <[email protected]> AuthorDate: Wed Aug 14 11:42:04 2024 +0800 Add old wal UT --- .../wal/recover/WALRecoverManagerTest.java | 67 +++++++++++++++++++++ .../test/resources/oldwal/2147483647/_0-0-0.wal | Bin 0 -> 265150 bytes .../test/resources/oldwal/2147483647/_0.checkpoint | Bin 0 -> 533 bytes .../test/resources/oldwal/2147483647/_1-0-0.wal | Bin 0 -> 259747 bytes .../test/resources/oldwal/2147483647/_2-0-0.wal | Bin 0 -> 36948 bytes .../test/resources/oldwal/2147483647/_3-0-1.wal | Bin 0 -> 513 bytes 6 files changed, 67 insertions(+) diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index dace1a99ba6..b2d462e10f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.conf.CommonDescriptor; import org.apache.iotdb.commons.exception.IllegalPathException; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.query.QueryProcessException; @@ -73,6 +74,7 @@ import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.nio.channels.FileChannel; +import java.nio.file.Files; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -351,6 +353,71 @@ public class WALRecoverManagerTest { // endregion } + @Test + public void testMemTableSnapshotFromOldVersion() throws Exception { + // old version of wal is generated by prepareCheckpointAndWALFileForSnapshot() in v1.3 + String oldWalPathStr = + this.getClass().getClassLoader().getResource("oldwal/2147483647").getFile(); + File oldWalFileDir = new File(oldWalPathStr); + FileUtils.copyDir(oldWalFileDir, new File(WAL_NODE_FOLDER)); + walBuffer = new WALBuffer(WAL_NODE_IDENTIFIER, WAL_NODE_FOLDER); + checkpointManager = walBuffer.getCheckpointManager(); + WALRecoverManager.getInstance().clear(); + recoverFromOldWalAndCheck(); + } + + private void recoverFromOldWalAndCheck() throws Exception { + // prepare tsFiles + List<WALRecoverListener> recoverListeners = new ArrayList<>(); + + // prepare file with wal + File fileWithWALDir = new File(FILE_WITH_WAL_NAME).getParentFile(); + Files.createDirectory(fileWithWALDir.toPath()); + File fileWithWAL = new File(fileWithWALDir, "1723544967972-1-0-0.tsfile"); + Files.createFile(fileWithWAL.toPath()); + tsFileWithWALResource = new TsFileResource(fileWithWAL); + UnsealedTsFileRecoverPerformer recoverPerformer = + new UnsealedTsFileRecoverPerformer( + tsFileWithWALResource, true, performer -> assertFalse(performer.canWrite())); + recoverManager.addRecoverPerformer(recoverPerformer); + recoverListeners.add(recoverPerformer.getRecoverListener()); + // recover + recoverManager.setAllDataRegionScannedLatch(new ExceptionalCountDownLatch(0)); + recoverManager.recover(); + // check recover listeners + try { + for (WALRecoverListener recoverListener : recoverListeners) { + assertEquals(WALRecoverListener.Status.SUCCESS, recoverListener.waitForResult()); + } + } catch (NullPointerException e) { + // ignore + } + // region check file with wal + // check file content + TsFileSequenceReader reader = new TsFileSequenceReader(fileWithWAL.getPath()); + List<ChunkMetadata> chunkMetadataList = + reader.getChunkMetadataList(new Path(DEVICE1_NAME, "s1", true)); + assertNotNull(chunkMetadataList); + chunkMetadataList = reader.getChunkMetadataList(new Path(DEVICE1_NAME, "s2", true)); + assertNotNull(chunkMetadataList); + chunkMetadataList = reader.getChunkMetadataList(new Path(DEVICE2_NAME, "s1", true)); + assertNotNull(chunkMetadataList); + chunkMetadataList = reader.getChunkMetadataList(new Path(DEVICE2_NAME, "s2", true)); + assertNotNull(chunkMetadataList); + assertEquals(1, chunkMetadataList.size()); + Chunk chunk = reader.readMemChunk(chunkMetadataList.get(0)); + assertEquals(15, chunk.getChunkStatistic().getEndTime()); + reader.close(); + // check .resource file in memory + assertEquals(4, tsFileWithWALResource.getStartTime(DEVICE2_NAME)); + assertEquals(15, tsFileWithWALResource.getEndTime(DEVICE2_NAME)); + // check file existence + assertTrue(fileWithWAL.exists()); + assertTrue(new File(fileWithWAL.getPath().concat(TsFileResource.RESOURCE_SUFFIX)).exists()); + // endregion + + } + private InsertRowNode getInsertRowNode(String devicePath, long time) throws MetadataException, QueryProcessException { TSDataType[] dataTypes = new TSDataType[] {TSDataType.FLOAT, TSDataType.DOUBLE}; diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal new file mode 100644 index 00000000000..49e504082d7 Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0-0-0.wal differ diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint new file mode 100644 index 00000000000..6d1291c2867 Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_0.checkpoint differ diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal new file mode 100644 index 00000000000..a092cf71d5e Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_1-0-0.wal differ diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal new file mode 100644 index 00000000000..71c6581cad6 Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_2-0-0.wal differ diff --git a/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal new file mode 100644 index 00000000000..00041c3d9fc Binary files /dev/null and b/iotdb-core/datanode/src/test/resources/oldwal/2147483647/_3-0-1.wal differ
