This is an automated email from the ASF dual-hosted git repository.
xingtanzjr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new ebca20e5d83 Validate flushed tsfile without holding closeQueryLock
(#11508)
ebca20e5d83 is described below
commit ebca20e5d83ff9f57cab631f02c562e2ebe9c6df
Author: shuwenwei <[email protected]>
AuthorDate: Mon Nov 13 21:36:38 2023 +0800
Validate flushed tsfile without holding closeQueryLock (#11508)
---
.../db/storageengine/dataregion/DataRegion.java | 46 +++++++++++++++-------
.../dataregion/tsfile/TsFileResource.java | 1 +
2 files changed, 32 insertions(+), 15 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index 4aeafd10203..0321fe0d595 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -124,6 +124,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -153,6 +154,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR;
import static
org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet.SEQUENCE_TSFILE;
import static
org.apache.iotdb.db.queryengine.metric.QueryResourceMetricSet.UNSEQUENCE_TSFILE;
+import static
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource.BROKEN_SUFFIX;
+import static
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource.RESOURCE_SUFFIX;
import static
org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource.TEMP_SUFFIX;
import static
org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX;
@@ -2086,12 +2089,17 @@ public class DataRegion implements IDataRegionForQuery {
// TODO please consider concurrency with read and insert method.
private void closeUnsealedTsFileProcessorCallBack(TsFileProcessor
tsFileProcessor)
throws TsFileProcessorException {
+ boolean tsFileNotValid =
+ tsFileProcessor.isEmpty()
+ ||
!TsFileValidator.getInstance().validateTsFile(tsFileProcessor.getTsFileResource());
closeQueryLock.writeLock().lock();
try {
tsFileProcessor.close();
- if (tsFileProcessor.isEmpty()
- ||
!TsFileValidator.getInstance().validateTsFile(tsFileProcessor.getTsFileResource()))
{
- tsFileProcessor.getTsFileResource().remove();
+ if (tsFileNotValid) {
+ String tsFilePath =
tsFileProcessor.getTsFileResource().getTsFile().getAbsolutePath();
+ renameAndHandleError(tsFilePath, tsFilePath + BROKEN_SUFFIX);
+ renameAndHandleError(
+ tsFilePath + RESOURCE_SUFFIX, tsFilePath + RESOURCE_SUFFIX +
BROKEN_SUFFIX);
tsFileManager.remove(tsFileProcessor.getTsFileResource(),
tsFileProcessor.isSequence());
} else {
tsFileResourceManager.registerSealedTsFileResource(tsFileProcessor.getTsFileResource());
@@ -2108,14 +2116,16 @@ public class DataRegion implements IDataRegionForQuery {
synchronized (closeStorageGroupCondition) {
closeStorageGroupCondition.notifyAll();
}
- TsFileResource tsFileResource = tsFileProcessor.getTsFileResource();
- FileMetrics.getInstance()
- .addTsFile(
- tsFileResource.getDatabaseName(),
- tsFileResource.getDataRegionId(),
- tsFileResource.getTsFileSize(),
- tsFileProcessor.isSequence(),
- tsFileResource.getTsFile().getName());
+ if (!tsFileNotValid) {
+ TsFileResource tsFileResource = tsFileProcessor.getTsFileResource();
+ FileMetrics.getInstance()
+ .addTsFile(
+ tsFileResource.getDatabaseName(),
+ tsFileResource.getDataRegionId(),
+ tsFileResource.getTsFileSize(),
+ tsFileProcessor.isSequence(),
+ tsFileResource.getTsFile().getName());
+ }
logger.info("signal closing database condition in {}", databaseName + "-"
+ dataRegionId);
}
@@ -2483,10 +2493,8 @@ public class DataRegion implements IDataRegionForQuery {
tsFileToLoad.getAbsolutePath(), targetFile.getAbsolutePath(),
e.getMessage()));
}
- File resourceFileToLoad =
- fsFactory.getFile(tsFileToLoad.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
- File targetResourceFile =
- fsFactory.getFile(targetFile.getAbsolutePath() +
TsFileResource.RESOURCE_SUFFIX);
+ File resourceFileToLoad = fsFactory.getFile(tsFileToLoad.getAbsolutePath()
+ RESOURCE_SUFFIX);
+ File targetResourceFile = fsFactory.getFile(targetFile.getAbsolutePath() +
RESOURCE_SUFFIX);
try {
if (deleteOriginFile) {
FileUtils.moveFile(resourceFileToLoad, targetResourceFile);
@@ -3036,6 +3044,14 @@ public class DataRegion implements IDataRegionForQuery {
return dataRegionInfo.getMemCost();
}
+ private void renameAndHandleError(String originFileName, String newFileName)
{
+ try {
+ Files.move(Paths.get(originFileName), Paths.get(newFileName));
+ } catch (IOException e) {
+ logger.error("Failed to rename {} to {},", originFileName, newFileName,
e);
+ }
+ }
+
@Override
public long getDataTTL() {
return dataTTL;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
index 0e228cbc7e0..99c6aa710f6 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java
@@ -87,6 +87,7 @@ public class TsFileResource {
public static final String RESOURCE_SUFFIX = ".resource";
public static final String TEMP_SUFFIX = ".temp";
+ public static final String BROKEN_SUFFIX = ".broken";
/** version number */
public static final byte VERSION_NUMBER = 1;
