This is an automated email from the ASF dual-hosted git repository.
JackieTien97 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 7a38e7e225f Add DataNodeExceptionMetrics with suspected disk exception
counter (#18554)
7a38e7e225f is described below
commit 7a38e7e225fdd58a970710018e70d5614400719c
Author: shuwenwei <[email protected]>
AuthorDate: Mon Aug 31 19:16:54 2026 +0800
Add DataNodeExceptionMetrics with suspected disk exception counter (#18554)
---
.../process/copyto/TableCopyToOperator.java | 2 +
.../execution/operator/source/FileLoaderUtils.java | 4 ++
.../read_tsfile/ExternalTsFileQueryResource.java | 3 +
.../schemaregion/impl/SchemaRegionMemoryImpl.java | 2 +
.../schemaregion/impl/SchemaRegionPBTreeImpl.java | 17 ++++-
.../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 2 +
.../impl/pbtree/flush/PBTreeFlushExecutor.java | 2 +
.../mtree/impl/pbtree/schemafile/SchemaFile.java | 2 +
.../service/metrics/DataNodeExceptionMetrics.java | 72 ++++++++++++++++++++++
.../db/service/metrics/DataNodeMetricsHelper.java | 1 +
.../db/storageengine/dataregion/DataRegion.java | 2 +
.../execute/task/AbstractCompactionTask.java | 2 +
.../compaction/schedule/CompactionScheduler.java | 2 +
.../dataregion/flush/MemTableFlushTask.java | 3 +
.../dataregion/memtable/TsFileProcessor.java | 10 +++
.../utils/TableDiskUsageStatisticUtil.java | 2 +
.../tableDiskUsageIndex/TableDiskUsageIndex.java | 2 +
.../tsfile/TsFileTableDiskUsageIndexWriter.java | 4 ++
.../tsfile/TsFileTableSizeIndexFileWriter.java | 4 +-
.../tsfile/TsFileTableSizeIndexReader.java | 2 +
.../dataregion/wal/buffer/WALBuffer.java | 5 ++
.../wal/checkpoint/CheckpointManager.java | 6 ++
.../dataregion/wal/recover/WALNodeRecoverTask.java | 2 +
.../recover/file/SealedTsFileRecoverPerformer.java | 2 +
.../file/UnsealedTsFileRecoverPerformer.java | 3 +
.../db/storageengine/load/LoadTsFileManager.java | 4 ++
.../db/storageengine/rescon/disk/TierManager.java | 4 ++
.../iotdb/commons/service/metric/enums/Metric.java | 1 +
28 files changed, 164 insertions(+), 3 deletions(-)
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/TableCopyToOperator.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/TableCopyToOperator.java
index 030a5ca4ae6..f43fbd2b730 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/TableCopyToOperator.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/process/copyto/TableCopyToOperator.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import org.apache.iotdb.db.queryengine.execution.operator.OperatorContext;
import
org.apache.iotdb.db.queryengine.execution.operator.process.copyto.tsfile.CopyToTsFileOptions;
import
org.apache.iotdb.db.queryengine.execution.operator.process.copyto.tsfile.TsFileFormatCopyToWriter;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.rescon.disk.TierManager;
import org.apache.iotdb.rpc.TSStatusCode;
@@ -106,6 +107,7 @@ public class TableCopyToOperator implements ProcessOperator
{
}
private IoTDBRuntimeException newCopyToWriteError(IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
LOGGER.warn(
DataNodeQueryMessages.LOG_FAILED_TO_WRITE_THE_TARGET_FILE_DURING_COPY_TO_EE25EF37,
e);
return new IoTDBRuntimeException(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
index 023dd69fd57..ab034a46806 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java
@@ -28,6 +28,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages;
import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext;
import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache;
import
org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey;
import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry;
@@ -172,6 +173,7 @@ public class FileLoaderUtils {
if (loadFromMem) {
throw e;
}
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new CorruptedTsFileException(
resource.getTsFile(),
CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA,
@@ -276,6 +278,7 @@ public class FileLoaderUtils {
if (loadFromMem) {
throw e;
}
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new CorruptedTsFileException(
resource.getTsFile(),
CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA,
@@ -533,6 +536,7 @@ public class FileLoaderUtils {
if (tsFile == null) {
throw e;
}
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new CorruptedTsFileException(
tsFile,
CorruptedTsFileException.Stage.READ_CHUNK_DATA_OR_LOAD_PAGE_READER,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/tvf/read_tsfile/ExternalTsFileQueryResource.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/tvf/read_tsfile/ExternalTsFileQueryResource.java
index 3333d99e933..441418519ba 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/tvf/read_tsfile/ExternalTsFileQueryResource.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/function/tvf/read_tsfile/ExternalTsFileQueryResource.java
@@ -35,6 +35,7 @@ import org.apache.iotdb.db.queryengine.common.QueryId;
import
org.apache.iotdb.db.queryengine.plan.planner.memory.NotThreadSafeMemoryReservationManager;
import
org.apache.iotdb.db.queryengine.plan.relational.metadata.AlignedDeviceEntry;
import org.apache.iotdb.db.queryengine.plan.relational.metadata.DeviceEntry;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSource;
import
org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -179,6 +180,7 @@ public class ExternalTsFileQueryResource {
? new SequentialDeviceTaskRunCursorManager(partition)
: new PriorityDeviceTaskRunCursorManager(partition));
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new RuntimeException(
DataNodeQueryMessages.FAILED_TO_CREATE_EXTERNAL_TSFILE_DEVICE_TASK_RUN_READER,
e);
}
@@ -329,6 +331,7 @@ public class ExternalTsFileQueryResource {
writeDeviceTaskRun(
queryTempRoot.resolve(planNodeId.getId()), runFiles.size(),
pendingDeviceTasks));
} catch (IOException e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new RuntimeException(
DataNodeQueryMessages.FAILED_TO_FLUSH_EXTERNAL_TSFILE_DEVICE_TASK_PARTITION, e);
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
index c9841112df6..184c66cfc21 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java
@@ -127,6 +127,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IDeleteLogic
import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IPreDeleteLogicalViewPlan;
import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IRollbackPreDeleteLogicalViewPlan;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.SchemaUtils;
@@ -749,6 +750,7 @@ public class SchemaRegionMemoryImpl implements
ISchemaRegion {
}
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new MetadataException(e);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
index 8fc6a95bbd2..502afed74ea 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java
@@ -101,6 +101,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateAligne
import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSeriesPlanImpl;
import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IAlterLogicalViewPlan;
import
org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.ICreateLogicalViewPlan;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo;
import org.apache.iotdb.db.utils.SchemaUtils;
@@ -249,6 +250,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
isRecovering = false;
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.CANNOT_RECOVER_ALL_MTREE,
storageGroupFullPath, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
initialized = true;
}
@@ -294,6 +296,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
SchemaConstant.METADATA_LOG_DESCRIPTION,
e.getMessage(),
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
}
@@ -341,8 +344,13 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
public void writeToMLog(ISchemaRegionPlan schemaRegionPlan) throws
IOException {
if (usingMLog && !isRecovering) {
- logWriter.write(schemaRegionPlan);
- regionStatistics.setMLogLength(logWriter.position());
+ try {
+ logWriter.write(schemaRegionPlan);
+ regionStatistics.setMLogLength(logWriter.position());
+ } catch (IOException e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
+ throw e;
+ }
}
}
@@ -359,6 +367,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
}
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.CANNOT_FORCE_MLOG, schemaRegionId,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
}
@@ -394,6 +403,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
logger.info(DataNodeSchemaMessages.MLOG_RECOVERY_CHECK_POINT,
mLogOffset);
} catch (IOException e) {
logger.warn(DataNodeSchemaMessages.CANNOT_GET_MLOG_CHECKPOINT,
e.getMessage());
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
try (SchemaLogReader<ISchemaRegionPlan> mLogReader =
new SchemaLogReader<>(
@@ -428,6 +438,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
mLogReader.skip(offset);
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.FAILED_TO_SKIP_MLOG, offset,
schemaRegionDirPath, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
while (mLogReader.hasNext()) {
plan = mLogReader.next();
@@ -468,6 +479,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
initialized = false;
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.CANNOT_CLOSE_METADATA_LOG_WRITER, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
isClearing = false;
}
@@ -585,6 +597,7 @@ public class SchemaRegionPBTreeImpl implements
ISchemaRegion {
} catch (IOException | MetadataException e) {
logger.error(
DataNodeSchemaMessages.FAILED_TO_LOAD_SNAPSHOT, schemaRegionId,
e.getMessage(), e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
try {
initialized = false;
isRecovering = true;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java
index 96933bdcae1..06f1adafaae 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java
@@ -37,6 +37,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStor
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.info.TableDeviceInfo;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.apache.tsfile.write.schema.MeasurementSchema;
@@ -364,6 +365,7 @@ public class MemMTreeSnapshotUtil {
}
} catch (IOException e) {
logger.error(SERIALIZE_ERROR_INFO, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
return false;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java
index 066116790f1..50295f6ea8a 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java
@@ -27,6 +27,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.IM
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.container.ICachedMNodeContainer;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafile.ISchemaFile;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -124,6 +125,7 @@ public class PBTreeFlushExecutor {
DataNodeSchemaMessages.IO_EXCEPTION_UPDATING_SG_MNODE,
updatedStorageGroupMNode.getFullPath(),
e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw e;
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java
index 968a4792e9a..9a2371b8d8f 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java
@@ -38,6 +38,7 @@ import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafil
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafile.pagemgr.IPageManager;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.schemafile.pagemgr.PageManager;
import
org.apache.iotdb.db.schemaengine.schemaregion.mtree.loader.MNodeFactoryLoader;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.tsfile.utils.ReadWriteIOUtils;
import org.slf4j.Logger;
@@ -483,6 +484,7 @@ public class SchemaFile implements ISchemaFile {
return true;
} catch (IOException e) {
logger.error(DataNodeSchemaMessages.FAILED_TO_CREATE_SCHEMA_FILE_SNAPSHOT,
e.getMessage(), e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
FileUtils.deleteFileIfExist(schemaFileSnapshot);
return false;
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java
new file mode 100644
index 00000000000..42035ca0c09
--- /dev/null
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.service.metrics;
+
+import org.apache.iotdb.commons.service.metric.enums.Metric;
+import org.apache.iotdb.metrics.AbstractMetricService;
+import org.apache.iotdb.metrics.impl.DoNothingMetricManager;
+import org.apache.iotdb.metrics.metricsets.IMetricSet;
+import org.apache.iotdb.metrics.type.Counter;
+import org.apache.iotdb.metrics.utils.MetricLevel;
+import org.apache.iotdb.metrics.utils.MetricType;
+
+import java.nio.file.FileSystemException;
+
+/** Records DataNode exceptions whose cause chain contains a {@link
FileSystemException}. */
+public class DataNodeExceptionMetrics implements IMetricSet {
+
+ private static final DataNodeExceptionMetrics INSTANCE = new
DataNodeExceptionMetrics();
+
+ private Counter suspiciousDiskExceptionCounter =
DoNothingMetricManager.DO_NOTHING_COUNTER;
+
+ private DataNodeExceptionMetrics() {
+ // singleton
+ }
+
+ @Override
+ public void bindTo(AbstractMetricService metricService) {
+ suspiciousDiskExceptionCounter =
+ metricService.getOrCreateCounter(
+ Metric.SUSPICIOUS_DISK_EXCEPTION_COUNT.toString(),
MetricLevel.IMPORTANT);
+ }
+
+ @Override
+ public void unbindFrom(AbstractMetricService metricService) {
+ suspiciousDiskExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER;
+ metricService.remove(MetricType.COUNTER,
Metric.SUSPICIOUS_DISK_EXCEPTION_COUNT.toString());
+ }
+
+ public void recordSuspiciousDiskException(Throwable throwable) {
+ // Uses exact type matching instead of instanceof because subclasses of
+ // FileSystemException (e.g. NoSuchFileException, NotDirectoryException,
+ // AccessDeniedException) usually indicate logical file-state errors rather
+ // than real disk failures, so they should not be counted here.
+ for (Throwable current = throwable; current != null; current =
current.getCause()) {
+ if (current.getClass() == FileSystemException.class) {
+ suspiciousDiskExceptionCounter.inc();
+ return;
+ }
+ }
+ }
+
+ public static DataNodeExceptionMetrics getInstance() {
+ return INSTANCE;
+ }
+}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
index e2204e8cf0b..9f6f4e57201 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java
@@ -69,6 +69,7 @@ public class DataNodeMetricsHelper {
metricService.addMetricSet(CompactionMetrics.getInstance());
metricService.addMetricSet(new ProcessMetrics());
metricService.addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE));
+ metricService.addMetricSet(DataNodeExceptionMetrics.getInstance());
metricService.addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE));
metricService.addMetricSet(ClientManagerMetrics.getInstance());
metricService.addMetricSet(RPCServiceThriftHandlerMetrics.getInstance());
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
index cfdc63edbdf..17b7e5b426d 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java
@@ -101,6 +101,7 @@ import
org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Tr
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
import org.apache.iotdb.db.service.SettleService;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.FileMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
@@ -2156,6 +2157,7 @@ public class DataRegion implements IDataRegionForQuery {
StorageEngineMessages
.STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
throw new WriteProcessException(
String.format(
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index 6959d3cbf03..9650196c5cf 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.exception.ChunkTypeInconsistentException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionFileCountExceededException;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException;
@@ -185,6 +186,7 @@ public abstract class AbstractCompactionTask {
chunkTypeInconsistentException.measurement,
chunkTypeInconsistentException.offsetOfChunkHeader);
} else {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
logger.error(
StorageEngineMessages.STORAGE_LOG_COMPACTION_TASK_MEETS_ERROR_1002C659,
storageGroupName,
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
index 77f13d3fe90..e80d45a8abf 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask;
import
org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask;
@@ -119,6 +120,7 @@ public class CompactionScheduler {
} catch (InterruptedException e) {
throw e;
} catch (Throwable e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
LOGGER.error(StorageEngineMessages.MEET_ERROR_IN_COMPACTION_SCHEDULE, e);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java
index 888c184f573..68727f03aa7 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java
@@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import
org.apache.iotdb.db.storageengine.dataregion.flush.pool.FlushSubTaskPoolManager;
@@ -213,6 +214,7 @@ public class MemTableFlushTask {
WritingMetrics.WRITE_PLAN_INDICES,
System.currentTimeMillis() - writePlanIndicesStartTime);
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new ExecutionException(e);
}
@@ -363,6 +365,7 @@ public class MemTableFlushTask {
storageGroup,
memTable,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
return;
}
long subTaskTime = System.currentTimeMillis() - starTime;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index 759b58d4966..7cfbb666c8c 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -50,6 +50,7 @@ import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTablet
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode;
import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo;
@@ -1867,6 +1868,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getAbsolutePath(),
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
try {
logger.error(
@@ -1882,6 +1884,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getAbsolutePath(),
e1);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e1);
}
// Release resource
try {
@@ -1929,6 +1932,7 @@ public class TsFileProcessor {
.STORAGE_LOG_MEET_ERROR_WHEN_WRITING_INTO_MODIFICATIONFILE_FILE_OF_63B5E24A,
tsFileResource.getTsFile().getAbsolutePath(),
e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
} finally {
flushQueryLock.writeLock().unlock();
}
@@ -1947,6 +1951,7 @@ public class TsFileProcessor {
writer.getTsFileOutput().force();
} catch (IOException e) {
logger.error(StorageEngineMessages.FSYNC_MEMTABLE_TO_DISK_ERROR, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
// Call flushed listener after memtable is released safely
@@ -1975,6 +1980,7 @@ public class TsFileProcessor {
logger.debug(StorageEngineMessages.FLUSHING_MEMTABLES_CLEAR,
dataRegionName);
}
} catch (Exception e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
logger.error(
StorageEngineMessages.STORAGE_LOG_MARKING_OR_ENDING_FILE_MEET_ERROR_5653B904,
dataRegionName,
@@ -1992,6 +1998,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getAbsolutePath(),
e1);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e1);
}
// Retry or set read-only
if (retryCnt < 3) {
@@ -2051,6 +2058,7 @@ public class TsFileProcessor {
dataRegionName,
tsFileResource.getTsFile().getName(),
e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
@@ -2129,6 +2137,7 @@ public class TsFileProcessor {
// When closing resource file, its corresponding mod file is also closed.
tsFileResource.closeWithoutSettingStatus();
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new TsFileProcessorException(e);
}
}
@@ -2595,6 +2604,7 @@ public class TsFileProcessor {
try {
writer.close();
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new TsFileProcessorException(e);
}
tsFileProcessorInfo.clear();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
index a695d0cad7c..414a1bcbe54 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/TableDiskUsageStatisticUtil.java
@@ -21,6 +21,7 @@ package org.apache.iotdb.db.storageengine.dataregion.utils;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import
org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
@@ -249,6 +250,7 @@ public class TableDiskUsageStatisticUtil extends
DiskUsageStatisticUtil {
timeSeriesMetadataCountRecorder,
timeSeriesMetadataIoSizeRecorder);
} catch (IOException e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new RuntimeException(e);
}
});
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
index f5b965f4786..ea578316511 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/TableDiskUsageIndex.java
@@ -23,6 +23,7 @@ import
org.apache.iotdb.commons.concurrent.IoTDBThreadPoolFactory;
import org.apache.iotdb.commons.concurrent.ThreadName;
import org.apache.iotdb.commons.utils.TestOnly;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
@@ -118,6 +119,7 @@ public class TableDiskUsageIndex {
writer.tsFileIndexWriter.syncIfNecessary();
} catch (IOException e) {
LOGGER.warn(StorageEngineMessages.FAILED_SYNC_TABLE_SIZE_INDEX, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableDiskUsageIndexWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableDiskUsageIndexWriter.java
index bc74d6ccb46..815441f5987 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableDiskUsageIndexWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableDiskUsageIndexWriter.java
@@ -22,6 +22,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageIndex.t
import org.apache.iotdb.commons.consensus.DataRegionId;
import org.apache.iotdb.commons.utils.FileUtils;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
@@ -98,6 +99,7 @@ public class TsFileTableDiskUsageIndexWriter extends
AbstractTableSizeIndexWrite
Files.move(tempValueFile.toPath(), valueFile.toPath());
} catch (IOException e) {
logger.warn(StorageEngineMessages.FAILED_TO_MOVE_FILE,
tempValueFile, valueFile, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
continue;
}
valueFiles.add(valueFile);
@@ -125,6 +127,7 @@ public class TsFileTableDiskUsageIndexWriter extends
AbstractTableSizeIndexWrite
new TsFileTableSizeIndexFileWriter(
regionId, currentKeyIndexFile, currentValueIndexFile,
needRecover);
} catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
failedToRecover(e);
}
}
@@ -192,6 +195,7 @@ public class TsFileTableDiskUsageIndexWriter extends
AbstractTableSizeIndexWrite
}
} catch (IOException e) {
logger.error(StorageEngineMessages.FAILED_TO_READ_KEY_FILE_DURING_COMPACTION,
e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
return;
} finally {
indexFileReader.closeCurrentFile();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexFileWriter.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexFileWriter.java
index bcfef97bdf0..0365526af63 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexFileWriter.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexFileWriter.java
@@ -19,6 +19,7 @@
package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageIndex.tsfile;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import org.apache.tsfile.utils.Pair;
@@ -178,7 +179,8 @@ public class TsFileTableSizeIndexFileWriter {
public void close() {
try {
sync();
- } catch (IOException ignored) {
+ } catch (IOException e) {
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
try {
if (valueBufferedOutputStream != null) {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexReader.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexReader.java
index 655e0b4e19d..85e19c69d65 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexReader.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/utils/tableDiskUsageIndex/tsfile/TsFileTableSizeIndexReader.java
@@ -21,6 +21,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageIndex.t
import org.apache.iotdb.commons.exception.IoTDBRuntimeException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
import
org.apache.iotdb.db.storageengine.dataregion.utils.tableDiskUsageIndex.DataRegionTableSizeQueryContext;
import org.apache.iotdb.db.utils.MmapUtil;
@@ -164,6 +165,7 @@ public class TsFileTableSizeIndexReader {
}
} catch (IOException e) {
closeCurrentFile();
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw e;
}
} while (System.nanoTime() - startTime < maxRunTime);
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
index 6921abecf75..d67cd88a4c4 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint;
import
org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager;
@@ -641,6 +642,7 @@ public class WALBuffer extends AbstractWALBuffer {
if (info.rollWALFileWriterListener != null) {
info.rollWALFileWriterListener.fail(e);
}
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
}
} else if (forceFlag) { // force os cache to the storage device, avoid
force twice by judging
@@ -654,6 +656,7 @@ public class WALBuffer extends AbstractWALBuffer {
.STORAGE_LOG_FAIL_TO_FSYNC_WAL_NODE_S_LOG_WRITER_CHANGE_SYSTEM_MODE_TO_7930160B,
identifier,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
for (WALFlushListener fsyncListener : info.fsyncListeners) {
fsyncListener.fail(e);
}
@@ -769,6 +772,7 @@ public class WALBuffer extends AbstractWALBuffer {
currentWALFileWriter.close();
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_CLOSE_WAL_LOG_WRITER,
identifier, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
checkpointManager.close();
@@ -844,6 +848,7 @@ public class WALBuffer extends AbstractWALBuffer {
id,
identifier,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
return Collections.emptySet();
});
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
index 1fc84166e05..a1bba3bad40 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java
@@ -24,6 +24,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.service.metrics.WritingMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor;
import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter;
@@ -110,6 +111,7 @@ public class CheckpointManager implements AutoCloseable {
currentLogWriter.write(tmpBuffer);
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_LOG_MAX_MEMTABLE_ID,
maxMemTableId, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
// log global memTables' info
makeGlobalInfoCP();
@@ -200,6 +202,7 @@ public class CheckpointManager implements AutoCloseable {
currentLogWriter.write(cachedByteBuffer);
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_MAKE_CHECKPOINT, checkpoint,
e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
} finally {
cachedByteBuffer.clear();
}
@@ -218,6 +221,7 @@ public class CheckpointManager implements AutoCloseable {
.STORAGE_LOG_FAIL_TO_FSYNC_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_6E1EE226,
identifier,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
}
@@ -238,6 +242,7 @@ public class CheckpointManager implements AutoCloseable {
.STORAGE_LOG_FAIL_TO_ROLL_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_791DDAB7,
identifier,
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
CommonDescriptor.getInstance().getConfig().handleUnrecoverableError();
}
} finally {
@@ -329,6 +334,7 @@ public class CheckpointManager implements AutoCloseable {
currentLogWriter.close();
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_CLOSE_WAL_CHECKPOINT_WRITER,
identifier, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
} finally {
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java
index af221813e7d..30d628dd45e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.memtable.AbstractMemTable;
import org.apache.iotdb.db.storageengine.dataregion.wal.WALManager;
import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry;
@@ -208,6 +209,7 @@ public class WALNodeRecoverTask implements Runnable {
walRepairWriter.repair(metaData);
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_RECOVER_WAL_METADATA,
walFile, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java
index bd021f6d74f..036a1d42070 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java
@@ -21,6 +21,7 @@ package
org.apache.iotdb.db.storageengine.dataregion.wal.recover.file;
import org.apache.iotdb.db.exception.DataRegionException;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
import java.io.IOException;
@@ -46,6 +47,7 @@ public class SealedTsFileRecoverPerformer extends
AbstractTsFileRecoverPerformer
try {
reconstructResourceFile();
} catch (IOException e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new DataRegionException(
String.format(
StorageEngineMessages
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
index f9696cc2617..d8617234190 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java
@@ -29,6 +29,7 @@ import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.flush.CompressionRatio;
import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
import org.apache.iotdb.db.storageengine.dataregion.memtable.IMemTable;
@@ -305,6 +306,7 @@ public class UnsealedTsFileRecoverPerformer extends
AbstractTsFileRecoverPerform
databaseName,
tsFileResource.getTsFile().getName(),
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
// if we put following codes in the 'if' clause above, this file can
be continued writing
@@ -314,6 +316,7 @@ public class UnsealedTsFileRecoverPerformer extends
AbstractTsFileRecoverPerform
tsFileResource.serialize();
FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(tsFileResource);
} catch (IOException | ExecutionException e) {
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
throw new WALRecoverException(e);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
index 220be0cd0c6..1be41760e8e 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java
@@ -46,6 +46,7 @@ import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent;
import
org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode;
import
org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand;
import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.db.storageengine.dataregion.flush.MemTableFlushTask;
import
org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile;
@@ -757,6 +758,7 @@ public class LoadTsFileManager {
StorageEngineMessages.CLOSE_TSFILE_IO_WRITER_ERROR,
entry.getValue().getFile().getPath(),
e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
}
@@ -777,6 +779,7 @@ public class LoadTsFileManager {
} catch (IOException e) {
LOGGER.warn(
StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR,
entry.getValue().getFile(), e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
}
@@ -790,6 +793,7 @@ public class LoadTsFileManager {
LOGGER.info(StorageEngineMessages.TASK_DIR_NOT_EMPTY_SKIP_DELETE,
taskDir.getPath());
} catch (IOException e) {
LOGGER.warn(MESSAGE_DELETE_FAIL, taskDir.getPath(), e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
dataPartition2Writer = null;
dataPartition2Resource = null;
diff --git
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
index 87c5cf00273..59bc5877979 100644
---
a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
+++
b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java
@@ -26,6 +26,7 @@ import
org.apache.iotdb.commons.exception.DiskSpaceInsufficientException;
import org.apache.iotdb.db.conf.IoTDBConfig;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.i18n.StorageEngineMessages;
+import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics;
import org.apache.iotdb.metrics.utils.FileStoreUtils;
import com.google.common.io.BaseEncoding;
@@ -117,6 +118,7 @@ public class TierManager {
tierDirs[i][j] = new File(tierDirs[i][j]).getCanonicalPath();
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_GET_CANONICAL_PATH,
tierDirs[i][j], e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
break;
case OBJECT_STORAGE:
@@ -367,6 +369,7 @@ public class TierManager {
filePath = file.getCanonicalFile().toPath();
} catch (IOException e) {
logger.error(StorageEngineMessages.FAIL_TO_GET_CANONICAL_PATH, file, e);
+ DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
filePath = file.toPath();
}
@@ -418,6 +421,7 @@ public class TierManager {
}
} catch (IOException e) {
logger.error(StorageEngineMessages.FAILED_TO_STATISTIC_SIZE,
fileStore, e);
+
DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e);
}
}
}
diff --git
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
index b41d8fb53ac..b35d2bc3575 100644
---
a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
+++
b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java
@@ -94,6 +94,7 @@ public enum Metric {
PAM_ALLOCATED_FAILURE_COUNT("primitive_array_manager_allocated_failure_count"),
PAM_RELEASED_FAILURE_COUNT("primitive_array_manager_released_failure_count"),
TABLE_DISK_USAGE_CACHE("table_disk_usage_cache"),
+ SUSPICIOUS_DISK_EXCEPTION_COUNT("suspicious_disk_exception_count"),
// compaction related
DATA_WRITTEN("data_written"),
