This is an automated email from the ASF dual-hosted git repository.
marklau99 pushed a commit to branch IOTDB-4257
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/IOTDB-4257 by this push:
new 65d859b4ca refactor snapshot
65d859b4ca is described below
commit 65d859b4caea6f9737e7f36a74f4733e873fa23b
Author: Liu Xuxin <[email protected]>
AuthorDate: Thu Oct 6 17:41:54 2022 +0800
refactor snapshot
---
.../apache/iotdb/commons/conf/IoTDBConstant.java | 1 +
.../iotdb/db/engine/snapshot/SnapshotLoader.java | 343 +++++++++++++--------
.../db/engine/snapshot/SnapshotLogAnalyzer.java | 53 +++-
.../iotdb/db/engine/snapshot/SnapshotLogger.java | 55 +++-
.../iotdb/db/engine/snapshot/SnapshotTaker.java | 14 +-
.../db/engine/snapshot/IoTDBSnapshotTest.java | 2 +-
6 files changed, 329 insertions(+), 139 deletions(-)
diff --git
a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
index 66e8c157df..60900789e1 100644
---
a/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
+++
b/node-commons/src/main/java/org/apache/iotdb/commons/conf/IoTDBConstant.java
@@ -186,6 +186,7 @@ public class IoTDBConstant {
public static final String FILE_NAME_SEPARATOR = "-";
public static final String UPGRADE_FOLDER_NAME = "upgrade";
public static final String CONSENSUS_FOLDER_NAME = "consensus";
+ public static final String SNAPSHOT_FOLDER_NAME = "snapshot";
// system folder name
public static final String SYSTEM_FOLDER_NAME = "system";
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java
index cff5886f38..d30edf80db 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLoader.java
@@ -21,11 +21,11 @@ package org.apache.iotdb.db.engine.snapshot;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.conf.directories.DirectoryManager;
+import org.apache.iotdb.db.conf.directories.FolderManager;
+import org.apache.iotdb.db.conf.directories.strategy.DirectoryStrategyType;
import org.apache.iotdb.db.engine.StorageEngineV2;
import org.apache.iotdb.db.engine.storagegroup.DataRegion;
import org.apache.iotdb.db.exception.DiskSpaceInsufficientException;
-import org.apache.iotdb.tsfile.utils.Pair;
import org.apache.commons.io.FileUtils;
import org.slf4j.Logger;
@@ -33,14 +33,12 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
-import java.nio.file.FileSystemException;
import java.nio.file.Files;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.LinkedList;
import java.util.List;
-import java.util.Objects;
+import java.util.Queue;
public class SnapshotLoader {
private Logger LOGGER = LoggerFactory.getLogger(SnapshotLoader.class);
@@ -48,11 +46,13 @@ public class SnapshotLoader {
private String snapshotPath;
private String dataRegionId;
private SnapshotLogAnalyzer logAnalyzer;
+ private FolderManager folderManager;
public SnapshotLoader(String snapshotPath, String storageGroupName, String
dataRegionId) {
this.snapshotPath = snapshotPath;
this.storageGroupName = storageGroupName;
this.dataRegionId = dataRegionId;
+ this.folderManager = null;
}
private DataRegion loadSnapshot() {
@@ -125,23 +125,31 @@ public class SnapshotLoader {
}
private DataRegion loadSnapshotWithLog(File logFile) {
+ boolean snapshotComplete = false;
try {
logAnalyzer = new SnapshotLogAnalyzer(logFile);
+ snapshotComplete = logAnalyzer.isSnapshotComplete();
} catch (Exception e) {
LOGGER.error("Exception occurs when reading snapshot file", e);
return null;
}
+ if (!snapshotComplete) {
+ // Do not load this snapshot because it's not complete.
+ LOGGER.error("This snapshot is not complete, cannot load it");
+ return null;
+ }
+
try {
try {
deleteAllFilesInDataDirs();
LOGGER.info("Remove all data files in original data dir");
+ createLinksFromSnapshotDirToDataDirWithLog();
+ return loadSnapshot();
} catch (IOException e) {
+ LOGGER.error("IOException occurs when creating links from snapshot dir
to data dir", e);
return null;
}
-
- createLinksFromSnapshotDirToDataDirWithLog();
- return loadSnapshot();
} finally {
logAnalyzer.close();
}
@@ -203,134 +211,195 @@ public class SnapshotLoader {
private void createLinksFromSnapshotDirToDataDirWithoutLog(File sourceDir)
throws IOException, DiskSpaceInsufficientException {
- File seqFileDir = new File(sourceDir, "sequence" + File.separator +
storageGroupName);
- File unseqFileDir = new File(sourceDir, "unsequence" + File.separator +
storageGroupName);
+ File seqFileDir =
+ new File(
+ sourceDir,
+ IoTDBConstant.SEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId);
+ File unseqFileDir =
+ new File(
+ sourceDir,
+ IoTDBConstant.UNSEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId);
if (!seqFileDir.exists() && !unseqFileDir.exists()) {
throw new IOException(
String.format(
"Cannot find %s or %s",
seqFileDir.getAbsolutePath(), unseqFileDir.getAbsolutePath()));
}
-
- File[] seqRegionDirs = seqFileDir.listFiles();
- if (seqRegionDirs != null && seqRegionDirs.length > 0) {
- for (File seqRegionDir : seqRegionDirs) {
- if (!seqRegionDir.isDirectory()) {
- LOGGER.info("Skip {}, because it is not a directory", seqRegionDir);
+ folderManager =
+ new FolderManager(
+
Arrays.asList(IoTDBDescriptor.getInstance().getConfig().getDataDirs()),
+ DirectoryStrategyType.SEQUENCE_STRATEGY);
+ File[] timePartitionFolders = seqFileDir.listFiles();
+ if (timePartitionFolders != null) {
+ for (File timePartitionFolder : timePartitionFolders) {
+ File[] files = timePartitionFolder.listFiles();
+ if (files == null || files.length == 0) {
continue;
}
- File[] seqPartitionDirs = seqRegionDir.listFiles();
- if (seqPartitionDirs != null && seqPartitionDirs.length > 0) {
- for (File seqPartitionDir : seqPartitionDirs) {
- String[] splitPath =
- seqPartitionDir
- .getAbsolutePath()
- .split(File.separator.equals("\\") ? "\\\\" :
File.separator);
- long timePartition = Long.parseLong(splitPath[splitPath.length -
1]);
- File[] files = seqPartitionDir.listFiles();
- if (files != null && files.length > 0) {
- Arrays.sort(files, Comparator.comparing(File::getName));
- String currDir =
DirectoryManager.getInstance().getNextFolderForSequenceFile();
- for (File file : files) {
- if (file.getName().endsWith(".tsfile")) {
- currDir =
DirectoryManager.getInstance().getNextFolderForSequenceFile();
- }
- File targetFile =
- new File(
- currDir,
- storageGroupName
- + File.separator
- + dataRegionId
- + File.separator
- + timePartition
- + File.separator
- + file.getName());
- if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
- throw new IOException(
- String.format("Failed to create dir %s",
targetFile.getParent()));
- }
- try {
- Files.createLink(targetFile.toPath(), file.toPath());
- } catch (FileSystemException e) {
- // cannot create link between two directories in different fs
- // copy it
- Files.copy(file.toPath(), targetFile.toPath());
- }
- }
- }
- }
- }
+ String targetSuffix =
+ IoTDBConstant.SEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId
+ + File.separator
+ + timePartitionFolder.getName();
+ createLinksFromSnapshotToSourceDir(targetSuffix, files);
}
}
- File[] unseqRegionDirs = unseqFileDir.listFiles();
- if (unseqRegionDirs != null && unseqRegionDirs.length > 0) {
- for (File unseqRegionDir : unseqRegionDirs) {
- if (!unseqRegionDir.isDirectory()) {
- LOGGER.info("Skip {}, because it is not a directory",
unseqRegionDir);
+ timePartitionFolders = unseqFileDir.listFiles();
+ if (timePartitionFolders != null) {
+ for (File timePartitionFolder : timePartitionFolders) {
+ File[] files = timePartitionFolder.listFiles();
+ if (files == null || files.length == 0) {
continue;
}
- File[] unseqPartitionDirs = unseqRegionDir.listFiles();
- if (unseqPartitionDirs != null && unseqPartitionDirs.length > 0) {
- for (File unseqPartitionDir : unseqPartitionDirs) {
- String[] splitPath =
- unseqPartitionDir
- .getAbsolutePath()
- .split(File.separator.equals("\\") ? "\\\\" :
File.separator);
- long timePartition = Long.parseLong(splitPath[splitPath.length -
1]);
- File[] files = unseqPartitionDir.listFiles();
- if (files != null && files.length > 0) {
- Arrays.sort(files, Comparator.comparing(File::getName));
- String currDir =
DirectoryManager.getInstance().getNextFolderForUnSequenceFile();
- for (File file : files) {
- if (file.getName().endsWith(".tsfile")) {
- currDir =
DirectoryManager.getInstance().getNextFolderForUnSequenceFile();
- }
- File targetFile =
- new File(
- currDir,
- storageGroupName
- + File.separator
- + dataRegionId
- + File.separator
- + timePartition
- + File.separator
- + file.getName());
- if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
- throw new IOException(
- String.format("Failed to create dir %s",
targetFile.getParent()));
- }
- try {
- Files.createLink(targetFile.toPath(), file.toPath());
- } catch (FileSystemException e) {
- Files.copy(file.toPath(), targetFile.toPath());
- }
- }
- }
- }
- }
+ String targetSuffix =
+ IoTDBConstant.UNSEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId
+ + File.separator
+ + timePartitionFolder.getName();
+ createLinksFromSnapshotToSourceDir(targetSuffix, files);
}
}
}
- private void createLinksFromSnapshotDirToDataDirWithLog() {
- while (logAnalyzer.hasNext()) {
- Pair<String, String> filesPath = logAnalyzer.getNextPairs();
- File sourceFile = new File(filesPath.left);
- File linkedFile = new File(filesPath.right);
- if (!linkedFile.exists()) {
- LOGGER.warn("Snapshot file {} does not exist, skip it", linkedFile);
- continue;
- }
- if (!sourceFile.getParentFile().exists() &&
!sourceFile.getParentFile().mkdirs()) {
- LOGGER.error("Failed to create folder {}", sourceFile.getParentFile());
- continue;
+ private void createLinksFromSnapshotToSourceDir(String targetSuffix, File[]
files)
+ throws DiskSpaceInsufficientException, IOException {
+ for (File file : files) {
+ String dataDir = folderManager.getNextFolder();
+ File targetFile =
+ new File(dataDir + File.separator + targetSuffix + File.separator +
file.getName());
+ if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "Cannot create directory %s",
targetFile.getParentFile().getAbsolutePath()));
}
try {
- Files.createLink(sourceFile.toPath(), linkedFile.toPath());
+ Files.createLink(targetFile.toPath(), file.toPath());
+ continue;
} catch (IOException e) {
- LOGGER.error("Failed to create link from {} to {}", linkedFile,
sourceFile, e);
+ LOGGER.info("Cannot create link from {} to {}, try to copy it", file,
targetFile);
}
+
+ Files.copy(file.toPath(), targetFile.toPath());
+ }
+ }
+
+ private void createLinksFromSnapshotDirToDataDirWithLog() throws IOException
{
+ String snapshotId = logAnalyzer.getSnapshotId();
+ int fileNum = logAnalyzer.getTotalFileCountInSnapshot();
+ String[] dataDirs =
IoTDBDescriptor.getInstance().getConfig().getDataDirs();
+ int fileCnt = 0;
+ for (String dataDir : dataDirs) {
+ String snapshotDir =
+ dataDir
+ + File.separator
+ + IoTDBConstant.SNAPSHOT_FOLDER_NAME
+ + File.separator
+ + snapshotId;
+ fileCnt += takeHardLinksFromSnapshotToDataDir(dataDir, new
File(snapshotDir));
+ }
+ if (fileCnt != fileNum) {
+ throw new IOException(
+ String.format(
+ "The file num in log is %d, while file num in disk is %d",
fileNum, fileCnt));
+ }
+ }
+
+ private int takeHardLinksFromSnapshotToDataDir(String dataDir, File
snapshotFolder)
+ throws IOException {
+ int cnt = 0;
+ File sequenceTimePartitionFolders =
+ new File(
+ snapshotFolder.getAbsolutePath()
+ + File.separator
+ + IoTDBConstant.SEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId);
+ File[] timePartitionFolders = sequenceTimePartitionFolders.listFiles();
+ if (timePartitionFolders != null) {
+ for (File timePartitionFolder : timePartitionFolders) {
+ String timePartition = timePartitionFolder.getName();
+ File[] sourceFiles = timePartitionFolder.listFiles();
+ if (sourceFiles == null) {
+ continue;
+ }
+ File targetDir =
+ new File(
+ dataDir
+ + File.separator
+ + IoTDBConstant.SEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId
+ + File.separator
+ + timePartition);
+ createLinksFromSourceToTarget(targetDir, sourceFiles);
+ cnt += sourceFiles.length;
+ }
+ }
+
+ File unsequenceTimePartitionFolders =
+ new File(
+ snapshotFolder.getAbsolutePath()
+ + File.separator
+ + IoTDBConstant.UNSEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId);
+ timePartitionFolders = unsequenceTimePartitionFolders.listFiles();
+ if (timePartitionFolders != null) {
+ for (File timePartitionFolder : timePartitionFolders) {
+ String timePartition = timePartitionFolder.getName();
+ File[] sourceFiles = timePartitionFolder.listFiles();
+ if (sourceFiles == null) {
+ continue;
+ }
+ File targetDir =
+ new File(
+ dataDir
+ + File.separator
+ + IoTDBConstant.UNSEQUENCE_FLODER_NAME
+ + File.separator
+ + storageGroupName
+ + File.separator
+ + dataRegionId
+ + File.separator
+ + timePartition);
+ createLinksFromSourceToTarget(targetDir, sourceFiles);
+ cnt += sourceFiles.length;
+ }
+ }
+
+ return cnt;
+ }
+
+ private void createLinksFromSourceToTarget(File targetDir, File[] files)
throws IOException {
+ for (File file : files) {
+ File targetFile = new File(targetDir, file.getName());
+ if (!targetFile.getParentFile().exists() &&
!targetFile.getParentFile().mkdirs()) {
+ throw new IOException(
+ String.format(
+ "Cannot create directory %s",
targetFile.getParentFile().getAbsolutePath()));
+ }
+ Files.createLink(targetFile.toPath(), file.toPath());
}
}
@@ -338,7 +407,7 @@ public class SnapshotLoader {
File snapshotLogFile = getSnapshotLogFile();
if (snapshotLogFile == null) {
- return getSnapshotFileWithoutLog();
+ return searchDataFilesRecursively(snapshotPath);
} else {
return getSnapshotFileWithLog(snapshotLogFile);
}
@@ -347,24 +416,44 @@ public class SnapshotLoader {
private List<File> getSnapshotFileWithLog(File logFile) throws IOException {
SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(logFile);
try {
-
+ String snapshotId = analyzer.getSnapshotId();
+ String[] dataDirs =
IoTDBDescriptor.getInstance().getConfig().getDataDirs();
List<File> fileList = new LinkedList<>();
-
- while (analyzer.hasNext()) {
- fileList.add(new File(analyzer.getNextPairs().right));
+ for (String dataDir : dataDirs) {
+ String snapshotDir =
+ dataDir
+ + File.separator
+ + IoTDBConstant.SNAPSHOT_FOLDER_NAME
+ + File.separator
+ + snapshotId;
+ fileList.addAll(searchDataFilesRecursively(snapshotDir));
}
-
return fileList;
} finally {
analyzer.close();
}
}
- private List<File> getSnapshotFileWithoutLog() {
- return new LinkedList<>(
- Arrays.asList(
- Objects.requireNonNull(
- new File(snapshotPath)
- .listFiles((dir, name) -> SnapshotFileSet.isDataFile(new
File(dir, name))))));
+ /**
+ * Search all data files in one directory recursively.
+ *
+ * @return
+ */
+ private List<File> searchDataFilesRecursively(String dir) {
+ LinkedList<File> fileList = new LinkedList<>();
+ Queue<File> queueToSearch = new LinkedList<>();
+ queueToSearch.add(new File(dir));
+ while (!queueToSearch.isEmpty()) {
+ File f = queueToSearch.poll();
+ if (f.isDirectory()) {
+ File[] files = f.listFiles();
+ if (files != null && files.length != 0) {
+ queueToSearch.addAll(Arrays.asList(files));
+ }
+ } else if (SnapshotFileSet.isDataFile(f)) {
+ fileList.add(f);
+ }
+ }
+ return fileList;
}
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogAnalyzer.java
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogAnalyzer.java
index 9291a99e3d..9fa935b969 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogAnalyzer.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogAnalyzer.java
@@ -25,18 +25,25 @@ import org.slf4j.LoggerFactory;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FileNotFoundException;
+import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
+import java.nio.charset.StandardCharsets;
public class SnapshotLogAnalyzer {
private static final Logger LOGGER =
LoggerFactory.getLogger(SnapshotLogAnalyzer.class);
private File snapshotLogFile;
private BufferedReader reader;
+ private FileReader fileReader;
+ private FileInputStream fileInputStream;
+ String snapshotId;
- public SnapshotLogAnalyzer(File snapshotLogFile) throws
FileNotFoundException {
+ public SnapshotLogAnalyzer(File snapshotLogFile) throws IOException {
this.snapshotLogFile = snapshotLogFile;
- this.reader = new BufferedReader(new FileReader(snapshotLogFile));
+ this.fileReader = new FileReader(snapshotLogFile);
+ this.fileInputStream = new FileInputStream(snapshotLogFile);
+ this.reader = new BufferedReader(fileReader);
+ this.snapshotId = reader.readLine();
}
public void close() {
@@ -59,7 +66,7 @@ public class SnapshotLogAnalyzer {
* @return The next pair of files recorded in the log. The left one is the
path of source file,
* the right one is the path of target file
*/
- public Pair<String, String> getNextPairs() {
+ public Pair<String, String> getTotalFileCount() {
if (reader == null) {
return null;
}
@@ -76,4 +83,42 @@ public class SnapshotLogAnalyzer {
return null;
}
}
+
+ public String getSnapshotId() {
+ return snapshotId;
+ }
+
+ /**
+ * Return the total num of file in this snapshot.
+ *
+ * @return
+ */
+ public int getTotalFileCountInSnapshot() throws IOException {
+ reader.reset();
+ String currLine;
+ int cnt = 0;
+ while ((currLine = reader.readLine()) != null) {
+ cnt++;
+ }
+ return cnt;
+ }
+
+ /**
+ * Read the tail of the log file to see if the snapshot is complete.
+ *
+ * @return
+ */
+ public boolean isSnapshotComplete() throws IOException {
+ char[] endFlagInChar = new char[SnapshotLogger.END_FLAG.length()];
+ long fileLength = snapshotLogFile.length();
+ int endFlagLength =
SnapshotLogger.END_FLAG.getBytes(StandardCharsets.UTF_8).length;
+ if (fileLength < endFlagLength) {
+ // this snapshot cannot be complete
+ return false;
+ }
+ reader.mark((int) fileLength);
+ reader.read(endFlagInChar, (int) (fileLength - endFlagLength),
endFlagLength);
+ String fileEndStr = new String(endFlagInChar);
+ return fileEndStr.equals(SnapshotLogger.END_FLAG);
+ }
}
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogger.java
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogger.java
index b2b8c06ba5..c067a0c3c3 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogger.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotLogger.java
@@ -27,7 +27,13 @@ import java.nio.file.Files;
public class SnapshotLogger implements AutoCloseable {
public static final String SNAPSHOT_LOG_NAME = "snapshot.log";
- public static final String SPLIT_CHAR = "#";
+ public static final String SPLIT_CHAR = " ";
+ public static final String END_FLAG = "END";
+ public static final int FILE_NAME_OFFSET = 1;
+ public static final int TIME_PARTITION_OFFSET = 2;
+ public static final int DATA_REGION_OFFSET = 3;
+ public static final int STORAGE_GROUP_OFFSET = 4;
+ public static final int SEQUENCE_OFFSET = 5;
private File logFile;
private BufferedOutputStream os;
@@ -48,14 +54,55 @@ public class SnapshotLogger implements AutoCloseable {
os.close();
}
- public void logFile(String sourceFile, String linkFile) throws IOException {
- os.write(sourceFile.getBytes(StandardCharsets.UTF_8));
+ /**
+ * Log the logical info for the link file, including its file name, time
partition, data region
+ * id, storage group name, sequence or not.
+ *
+ * @param sourceFile
+ * @throws IOException
+ */
+ public void logFile(File sourceFile) throws IOException {
+ String[] splitInfo =
+ sourceFile.getAbsolutePath().split(File.separator.equals("\\") ?
"\\\\" : "/");
+ int length = splitInfo.length;
+ String fileName = splitInfo[length - FILE_NAME_OFFSET];
+ String timePartition = splitInfo[length - TIME_PARTITION_OFFSET];
+ String dataRegion = splitInfo[length - DATA_REGION_OFFSET];
+ String storageGroup = splitInfo[length - STORAGE_GROUP_OFFSET];
+ String sequence = splitInfo[length - SEQUENCE_OFFSET];
+ os.write(fileName.getBytes(StandardCharsets.UTF_8));
os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
- os.write(linkFile.getBytes(StandardCharsets.UTF_8));
+ os.write(timePartition.getBytes(StandardCharsets.UTF_8));
+ os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
+ os.write(dataRegion.getBytes(StandardCharsets.UTF_8));
+ os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
+ os.write(storageGroup.getBytes(StandardCharsets.UTF_8));
+ os.write(SPLIT_CHAR.getBytes(StandardCharsets.UTF_8));
+ os.write(sequence.getBytes(StandardCharsets.UTF_8));
os.write("\n".getBytes(StandardCharsets.UTF_8));
os.flush();
}
+ /**
+ * Log the snapshot id to identify this snapshot.
+ *
+ * @param id
+ * @throws IOException
+ */
+ public void logSnapshotId(String id) throws IOException {
+ os.write(id.getBytes(StandardCharsets.UTF_8));
+ os.write("\n".getBytes(StandardCharsets.UTF_8));
+ }
+
+ /**
+ * Log the end of the snapshot.
+ *
+ * @throws IOException
+ */
+ public void logEnd() throws IOException {
+ os.write(END_FLAG.getBytes(StandardCharsets.UTF_8));
+ }
+
public void cleanUpWhenFailed() throws IOException {
os.close();
Files.delete(logFile.toPath());
diff --git
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java
index 004793bfa2..bde1e083d1 100644
---
a/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java
+++
b/server/src/main/java/org/apache/iotdb/db/engine/snapshot/SnapshotTaker.java
@@ -19,6 +19,7 @@
package org.apache.iotdb.db.engine.snapshot;
+import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.engine.modification.ModificationFile;
@@ -76,6 +77,7 @@ public class SnapshotTaker {
try {
snapshotLogger = new SnapshotLogger(snapshotLog);
boolean success = true;
+ snapshotLogger.logSnapshotId(snapshotDir.getName());
readLockTheFile();
try {
@@ -92,6 +94,7 @@ public class SnapshotTaker {
dataRegion.getDataRegionId());
cleanUpWhenFail(snapshotDir.getName());
} else {
+ snapshotLogger.logEnd();
LOGGER.info(
"Successfully take snapshot for {}-{}, snapshot directory is {}",
dataRegion.getStorageGroupName(),
@@ -167,7 +170,7 @@ public class SnapshotTaker {
private void createHardLink(File target, File source) throws IOException {
Files.createLink(target.toPath(), source.toPath());
- snapshotLogger.logFile(source.getAbsolutePath(), target.getAbsolutePath());
+ snapshotLogger.logFile(source);
}
/**
@@ -193,7 +196,7 @@ public class SnapshotTaker {
stringBuilder.append(splittedPath[i]);
stringBuilder.append(File.separator);
}
- stringBuilder.append("snapshot");
+ stringBuilder.append(IoTDBConstant.SNAPSHOT_FOLDER_NAME);
stringBuilder.append(File.separator);
stringBuilder.append(snapshotId);
stringBuilder.append(File.separator);
@@ -215,7 +218,12 @@ public class SnapshotTaker {
private void cleanUpWhenFail(String snapshotId) {
for (String dataDir :
IoTDBDescriptor.getInstance().getConfig().getDataDirs()) {
File dataDirForThisSnapshot =
- new File(dataDir + File.separator + "snapshot" + File.separator +
snapshotId);
+ new File(
+ dataDir
+ + File.separator
+ + IoTDBConstant.SNAPSHOT_FOLDER_NAME
+ + File.separator
+ + snapshotId);
if (dataDirForThisSnapshot.exists()) {
try {
FileUtils.recursiveDeleteFolder(dataDirForThisSnapshot.getAbsolutePath());
diff --git
a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
index 804ed6ef94..49c8560ef3 100644
---
a/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
+++
b/server/src/test/java/org/apache/iotdb/db/engine/snapshot/IoTDBSnapshotTest.java
@@ -110,7 +110,7 @@ public class IoTDBSnapshotTest {
SnapshotLogAnalyzer analyzer = new SnapshotLogAnalyzer(files[0]);
int cnt = 0;
while (analyzer.hasNext()) {
- analyzer.getNextPairs();
+ analyzer.getTotalFileCount();
cnt++;
}
analyzer.close();
