This is an automated email from the ASF dual-hosted git repository. jiangtian pushed a commit to branch add_keep_same_disk_when_loading_snapshot in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit b5ec2cafebe73129f8d776bed400d98b718f976e Author: Tian Jiang <[email protected]> AuthorDate: Wed Oct 15 19:39:53 2025 +0800 Add config "keep_same_disk_when_loading_snapshot" --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 10 ++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 6 ++ .../dataregion/snapshot/SnapshotLoader.java | 99 ++++++++++-------- .../storageengine/rescon/disk/FolderManager.java | 22 ++++ .../dataregion/snapshot/IoTDBSnapshotTest.java | 111 ++++++++++++++++++--- .../conf/iotdb-system.properties.template | 6 ++ 6 files changed, 199 insertions(+), 55 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 74a966eacea..11e3610e1ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1176,6 +1176,8 @@ public class IoTDBConfig { private boolean includeNullValueInWriteThroughputMetric = false; + private boolean keepSameDiskWhenLoadingSnapshot = false; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -4225,4 +4227,12 @@ public class IoTDBConfig { public void setPasswordLockTimeMinutes(int passwordLockTimeMinutes) { this.passwordLockTimeMinutes = passwordLockTimeMinutes; } + + public boolean isKeepSameDiskWhenLoadingSnapshot() { + return keepSameDiskWhenLoadingSnapshot; + } + + public void setKeepSameDiskWhenLoadingSnapshot(boolean keepSameDiskWhenLoadingSnapshot) { + this.keepSameDiskWhenLoadingSnapshot = keepSameDiskWhenLoadingSnapshot; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 0fa4aaf90e7..cbb92d22959 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1171,6 +1171,12 @@ public class IoTDBDescriptor { "region_migration_speed_limit_bytes_per_second", ConfigurationFileUtils.getConfigurationDefaultValue( "region_migration_speed_limit_bytes_per_second")))); + conf.setKeepSameDiskWhenLoadingSnapshot( + Boolean.parseBoolean( + properties.getProperty( + "keep_same_disk_when_loading_snapshot", + ConfigurationFileUtils.getConfigurationDefaultValue( + "keep_same_disk_when_loading_snapshot")))); } private void loadIoTConsensusV2Props(TrimProperties properties) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java index daa64274b12..ffcee4f51bd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotLoader.java @@ -314,56 +314,69 @@ public class SnapshotLoader { } } + private File createLinksFromSnapshotToSourceDir( + String targetSuffix, + File file, + Map<String, String> fileTarget, + String fileKey, + String finalDir) + throws IOException { + File targetFile = + new File(finalDir + File.separator + targetSuffix + File.separator + file.getName()); + + try { + if (!targetFile.getParentFile().exists() && !targetFile.getParentFile().mkdirs()) { + throw new IOException( + String.format( + "Cannot create directory %s", targetFile.getParentFile().getAbsolutePath())); + } + + try { + Files.createLink(targetFile.toPath(), file.toPath()); + LOGGER.debug("Created hard link from {} to {}", file, targetFile); + fileTarget.put(fileKey, finalDir); + return targetFile; + } catch (IOException e) { + LOGGER.info("Cannot create link from {} to {}, fallback to copy", file, targetFile); + } + + Files.copy(file.toPath(), targetFile.toPath()); + fileTarget.put(fileKey, finalDir); + return targetFile; + } catch (Exception e) { + LOGGER.warn( + "Failed to process file {} in dir {}: {}", file.getName(), finalDir, e.getMessage(), e); + throw e; + } + } + private void createLinksFromSnapshotToSourceDir( - String targetSuffix, File[] files, FolderManager folderManager) - throws DiskSpaceInsufficientException, IOException { + String targetSuffix, File[] files, FolderManager folderManager) throws IOException { Map<String, String> fileTarget = new HashMap<>(); for (File file : files) { String fileKey = file.getName().split("\\.")[0]; String dataDir = fileTarget.get(fileKey); + if (dataDir != null) { + createLinksFromSnapshotToSourceDir(targetSuffix, file, fileTarget, fileKey, dataDir); + continue; + } + try { - folderManager.getNextWithRetry( - currentDataDir -> { - String effectiveDir = (dataDir != null) ? dataDir : currentDataDir; - File targetFile = - new File( - effectiveDir - + File.separator - + targetSuffix - + File.separator - + file.getName()); - - try { - if (!targetFile.getParentFile().exists() && !targetFile.getParentFile().mkdirs()) { - throw new IOException( - String.format( - "Cannot create directory %s", - targetFile.getParentFile().getAbsolutePath())); - } - - try { - Files.createLink(targetFile.toPath(), file.toPath()); - LOGGER.debug("Created hard link from {} to {}", file, targetFile); - return targetFile; - } catch (IOException e) { - LOGGER.info( - "Cannot create link from {} to {}, fallback to copy", file, targetFile); - } - - Files.copy(file.toPath(), targetFile.toPath()); - fileTarget.put(fileKey, effectiveDir); - return targetFile; - } catch (Exception e) { - LOGGER.warn( - "Failed to process file {} in dir {}: {}", - file.getName(), - effectiveDir, - e.getMessage(), - e); - throw e; - } - }); + String firstFolderOfSameDisk = + IoTDBDescriptor.getInstance().getConfig().isKeepSameDiskWhenLoadingSnapshot() + ? folderManager.getFirstFolderOfSameDisk(file.getAbsolutePath()) + : null; + + if (firstFolderOfSameDisk != null) { + createLinksFromSnapshotToSourceDir( + targetSuffix, file, fileTarget, fileKey, firstFolderOfSameDisk); + } else { + folderManager.getNextWithRetry( + currentDataDir -> + createLinksFromSnapshotToSourceDir( + targetSuffix, file, fileTarget, fileKey, currentDataDir)); + } } catch (Exception e) { throw new IOException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java index e90292853f1..8d3b15d72b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/FolderManager.java @@ -32,6 +32,11 @@ import org.apache.iotdb.db.storageengine.rescon.disk.strategy.SequenceStrategy; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -146,4 +151,21 @@ public class FolderManager { public List<String> getFolders() { return folders; } + + public String getFirstFolderOfSameDisk(String pathStr) { + Path path = Paths.get(pathStr); + try { + FileStore fileStore = Files.getFileStore(path); + for (String folder : folders) { + Path folderPath = Paths.get(folder); + FileStore folderFileStore = Files.getFileStore(folderPath); + if (folderFileStore.equals(fileStore)) { + return folder; + } + } + } catch (IOException e) { + logger.warn("Failed to read file store path '" + pathStr + "'", e); + } + return null; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java index 6bec0df050b..926352646aa 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/IoTDBSnapshotTest.java @@ -38,15 +38,21 @@ import org.apache.tsfile.utils.TsFileGeneratorUtils; import org.junit.After; import org.junit.Assert; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.mockito.Mockito; import java.io.File; import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.List; +import static org.apache.iotdb.consensus.iot.IoTConsensusServerImpl.SNAPSHOT_DIR_NAME; import static org.apache.tsfile.common.constant.TsFileConstant.PATH_SEPARATOR; +import static org.junit.Assert.assertEquals; public class IoTDBSnapshotTest { private String[][] testDataDirs = @@ -65,11 +71,12 @@ public class IoTDBSnapshotTest { FileUtils.recursivelyDeleteFolder("target" + File.separator + "tmp"); } - private List<TsFileResource> writeTsFiles() throws IOException, WriteProcessException { + private List<TsFileResource> writeTsFiles(String[] dataDirs) + throws IOException, WriteProcessException { List<TsFileResource> resources = new ArrayList<>(); for (int i = 0; i < 100; i++) { String filePath = - testDataDirs[0][i % 3] + dataDirs[i % dataDirs.length] + File.separator + "sequence" + File.separator @@ -108,7 +115,7 @@ public class IoTDBSnapshotTest { IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs); TierManager.getInstance().resetFolders(); try { - List<TsFileResource> resources = writeTsFiles(); + List<TsFileResource> resources = writeTsFiles(testDataDirs[0]); DataRegion region = new DataRegion(testSgName, "0"); region.getTsFileManager().addAll(resources, true); File snapshotDir = new File("target" + File.separator + "snapshot"); @@ -117,12 +124,12 @@ public class IoTDBSnapshotTest { new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true); File[] files = snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME)); - Assert.assertEquals(1, files.length); + assertEquals(1, files.length); SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(files[0]); Assert.assertTrue(analyzer.isSnapshotComplete()); int cnt = analyzer.getTotalFileCountInSnapshot(); analyzer.close(); - Assert.assertEquals(200, cnt); + assertEquals(200, cnt); for (TsFileResource resource : resources) { Assert.assertTrue(resource.tryWriteLock()); } @@ -142,7 +149,7 @@ public class IoTDBSnapshotTest { IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs); TierManager.getInstance().resetFolders(); try { - List<TsFileResource> resources = writeTsFiles(); + List<TsFileResource> resources = writeTsFiles(testDataDirs[0]); resources.subList(50, 100).forEach(x -> x.setStatusForTest(TsFileResourceStatus.UNCLOSED)); DataRegion region = new DataRegion(testSgName, "0"); region.setAllowCompaction(false); @@ -153,13 +160,13 @@ public class IoTDBSnapshotTest { new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true); File[] files = snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME)); - Assert.assertEquals(1, files.length); + assertEquals(1, files.length); SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(files[0]); int cnt = 0; Assert.assertTrue(analyzer.isSnapshotComplete()); cnt = analyzer.getTotalFileCountInSnapshot(); analyzer.close(); - Assert.assertEquals(100, cnt); + assertEquals(100, cnt); for (TsFileResource resource : resources) { Assert.assertTrue(resource.tryWriteLock()); } @@ -179,7 +186,7 @@ public class IoTDBSnapshotTest { IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(testDataDirs); TierManager.getInstance().resetFolders(); try { - List<TsFileResource> resources = writeTsFiles(); + List<TsFileResource> resources = writeTsFiles(testDataDirs[0]); DataRegion region = new DataRegion(testSgName, "0"); CompressionRatio.getInstance().updateRatio(100, 100, "0"); region.getTsFileManager().addAll(resources, true); @@ -195,8 +202,8 @@ public class IoTDBSnapshotTest { .loadSnapshotForStateMachine(); Assert.assertNotNull(dataRegion); List<TsFileResource> resource = dataRegion.getTsFileManager().getTsFileList(true); - Assert.assertEquals(100, resource.size()); - Assert.assertEquals( + assertEquals(100, resource.size()); + assertEquals( new Pair<>(100L, 100L), CompressionRatio.getInstance().getDataRegionRatioMap().get("0")); } finally { @@ -208,6 +215,86 @@ public class IoTDBSnapshotTest { } } + @Ignore("Need manual execution to specify different disks") + @Test + public void testLoadSnapshotNoHardLink() + throws IOException, WriteProcessException, DirectoryNotLegalException { + IoTDBDescriptor.getInstance().getConfig().setKeepSameDiskWhenLoadingSnapshot(true); + // initialize dirs + String[][] dataDirsForDB = new String[][] {{"C://snapshot_test", "D://snapshot_test"}}; + File snapshotDir = new File("D://snapshot_store//"); + if (snapshotDir.exists()) { + FileUtils.recursivelyDeleteFolder(snapshotDir.getAbsolutePath()); + } + for (String[] dirs : dataDirsForDB) { + for (String dir : dirs) { + if (new File(dir).exists()) { + FileUtils.recursivelyDeleteFolder(dir); + } + } + } + IoTDBDescriptor.getInstance().getConfig().setTierDataDirs(dataDirsForDB); + TierManager.getInstance().resetFolders(); + + // prepare files, files should be written into two folders + List<TsFileResource> resources = writeTsFiles(dataDirsForDB[0]); + DataRegion region = new DataRegion(testSgName, "0"); + region.getTsFileManager().addAll(resources, true); + + // take a snapshot into one disk + Assert.assertTrue(snapshotDir.exists() || snapshotDir.mkdirs()); + try { + Assert.assertTrue( + new SnapshotTaker(region).takeFullSnapshot(snapshotDir.getAbsolutePath(), true)); + File[] files = + snapshotDir.listFiles((dir, name) -> name.equals(SnapshotLogger.SNAPSHOT_LOG_NAME)); + // use loadWithoutLog + if (files != null && files.length > 0) { + files[0].delete(); + } + // move files to snapshot store (simulate snapshot transfer) + for (String dir : dataDirsForDB[0]) { + File internalSnapshotDir = new File(dir, SNAPSHOT_DIR_NAME); + if (internalSnapshotDir.exists()) { + for (File file : FileUtils.listFilesRecursively(internalSnapshotDir, f -> true)) { + if (file.isFile()) { + String absolutePath = file.getAbsolutePath(); + int snapshotIdIndex = absolutePath.indexOf("snapshot_store"); + int suffixIndex = snapshotIdIndex + "snapshot_store".length(); + String suffix = absolutePath.substring(suffixIndex); + File snapshotFile = new File(snapshotDir, suffix); + FileUtils.copyFile(file, snapshotFile); + } + } + } + } + + // load the snapshot + DataRegion dataRegion = + new SnapshotLoader(snapshotDir.getAbsolutePath(), testSgName, "0") + .loadSnapshotForStateMachine(); + Assert.assertNotNull(dataRegion); + resources = dataRegion.getTsFileManager().getTsFileList(true); + assertEquals(100, resources.size()); + + // files should not be moved to another disk + Path snapshotDirPath = snapshotDir.toPath(); + FileStore snapshotFileStore = Files.getFileStore(snapshotDirPath); + for (TsFileResource tsFileResource : resources) { + Path tsfilePath = tsFileResource.getTsFile().toPath(); + FileStore tsFileFileStore = Files.getFileStore(tsfilePath); + assertEquals(snapshotFileStore, tsFileFileStore); + } + } finally { + FileUtils.recursivelyDeleteFolder(snapshotDir.getAbsolutePath()); + for (String[] dirs : dataDirsForDB) { + for (String dir : dirs) { + FileUtils.recursivelyDeleteFolder(dir); + } + } + } + } + @Test public void testGetSnapshotFile() throws IOException { File tsFile = @@ -228,7 +315,7 @@ public class IoTDBSnapshotTest { Mockito.when(region.getDataRegionId()).thenReturn("0"); File snapshotFile = new SnapshotTaker(region).getSnapshotFilePathForTsFile(tsFile, "test-snapshotId"); - Assert.assertEquals( + assertEquals( new File( IoTDBDescriptor.getInstance().getConfig().getLocalDataDirs()[0] + File.separator diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index 207c0507093..eb55c021d7a 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1607,6 +1607,12 @@ data_region_iot_max_memory_ratio_for_queue = 0.6 # Datatype: long region_migration_speed_limit_bytes_per_second = 50331648 +# When loading snapshot, try keeping TsFiles in the same disk as the snapshot dir. +# This may reduce file copies but may also result in a worse disk load-balance +# effectiveMode: hot_reload +# Datatype: boolean +keep_same_disk_when_loading_snapshot=false + #################### ### Blob Allocator Configuration ####################
