This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch diskExceptionCounter in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 637b3ba84ef202ff09fc6ebc430a85c4feb7e3d2 Author: shuwenwei <[email protected]> AuthorDate: Mon Aug 31 17:59:30 2026 +0800 Add DataNodeExceptionMetrics with suspected disk exception counter for DataNode Add a new DataNodeExceptionMetrics class with a counter that tracks suspicious disk exceptions. The counter only increments when an exception in the cause chain is exactly java.nio.file.FileSystemException (not subclasses), because subclasses like NoSuchFileException or NotDirectoryException typically indicate logical file-state errors rather than real disk failures. Integration points cover: - Write path: TsFileProcessor, MemTableFlushTask, WALBuffer, CheckpointManager - Compaction: AbstractCompactionTask, CompactionScheduler - WAL recovery: WALNodeRecoverTask, Unsealed/SealedTsFileRecoverPerformer - Schema engine: SchemaRegionPBTreeImpl (writeToMLog), SchemaRegionMemoryImpl, MemMTreeSnapshotUtil, PBTreeFlushExecutor, SchemaFile - Load: LoadTsFileManager, FileLoaderUtils - Disk management: TierManager --- .../execution/operator/source/FileLoaderUtils.java | 4 ++++ .../schemaregion/impl/SchemaRegionMemoryImpl.java | 2 ++ .../schemaregion/impl/SchemaRegionPBTreeImpl.java | 17 +++++++++++++++-- .../impl/mem/snapshot/MemMTreeSnapshotUtil.java | 2 ++ .../mtree/impl/pbtree/flush/PBTreeFlushExecutor.java | 2 ++ .../mtree/impl/pbtree/schemafile/SchemaFile.java | 2 ++ .../db/service/metrics/DataNodeExceptionMetrics.java | 20 ++++++++++++-------- .../db/storageengine/dataregion/DataRegion.java | 2 +- .../execute/task/AbstractCompactionTask.java | 2 ++ .../compaction/schedule/CompactionScheduler.java | 2 ++ .../dataregion/flush/MemTableFlushTask.java | 3 +++ .../dataregion/memtable/TsFileProcessor.java | 11 +++++++++-- .../dataregion/wal/buffer/WALBuffer.java | 5 ++++- .../dataregion/wal/checkpoint/CheckpointManager.java | 6 +++++- .../dataregion/wal/recover/WALNodeRecoverTask.java | 2 ++ .../recover/file/SealedTsFileRecoverPerformer.java | 2 ++ .../recover/file/UnsealedTsFileRecoverPerformer.java | 3 +++ .../db/storageengine/load/LoadTsFileManager.java | 4 ++++ .../db/storageengine/rescon/disk/TierManager.java | 4 ++++ .../iotdb/commons/service/metric/enums/Metric.java | 2 +- 20 files changed, 81 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java index 023dd69fd57..ab034a46806 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/source/FileLoaderUtils.java @@ -28,6 +28,7 @@ import org.apache.iotdb.db.i18n.DataNodeQueryMessages; import org.apache.iotdb.db.queryengine.execution.fragment.FragmentInstanceContext; import org.apache.iotdb.db.queryengine.execution.fragment.QueryContext; import org.apache.iotdb.db.queryengine.metric.SeriesScanCostMetricSet; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache; import org.apache.iotdb.db.storageengine.buffer.TimeSeriesMetadataCache.TimeSeriesMetadataCacheKey; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; @@ -172,6 +173,7 @@ public class FileLoaderUtils { if (loadFromMem) { throw e; } + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new CorruptedTsFileException( resource.getTsFile(), CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, @@ -276,6 +278,7 @@ public class FileLoaderUtils { if (loadFromMem) { throw e; } + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new CorruptedTsFileException( resource.getTsFile(), CorruptedTsFileException.Stage.READ_TIMESERIES_METADATA, @@ -533,6 +536,7 @@ public class FileLoaderUtils { if (tsFile == null) { throw e; } + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new CorruptedTsFileException( tsFile, CorruptedTsFileException.Stage.READ_CHUNK_DATA_OR_LOAD_PAGE_READER, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java index c9841112df6..4f749d78134 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionMemoryImpl.java @@ -48,6 +48,7 @@ import org.apache.iotdb.commons.utils.RegionMigrationFileRemoveRateLimiter; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.exception.metadata.PathNotExistException; import org.apache.iotdb.db.exception.metadata.SchemaDirCreationFailureException; import org.apache.iotdb.db.exception.metadata.SchemaQuotaExceededException; @@ -749,6 +750,7 @@ public class SchemaRegionMemoryImpl implements ISchemaRegion { } } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new MetadataException(e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java index 8fc6a95bbd2..9cf3251d5ba 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/impl/SchemaRegionPBTreeImpl.java @@ -102,6 +102,7 @@ import org.apache.iotdb.db.schemaengine.schemaregion.write.req.impl.CreateTimeSe import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.IAlterLogicalViewPlan; import org.apache.iotdb.db.schemaengine.schemaregion.write.req.view.ICreateLogicalViewPlan; import org.apache.iotdb.db.storageengine.rescon.memory.SystemInfo; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.utils.SchemaUtils; import org.apache.tsfile.enums.TSDataType; @@ -249,6 +250,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { isRecovering = false; } catch (IOException e) { logger.error(DataNodeSchemaMessages.CANNOT_RECOVER_ALL_MTREE, storageGroupFullPath, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } initialized = true; } @@ -294,6 +296,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { SchemaConstant.METADATA_LOG_DESCRIPTION, e.getMessage(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } @@ -341,8 +344,13 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { public void writeToMLog(ISchemaRegionPlan schemaRegionPlan) throws IOException { if (usingMLog && !isRecovering) { - logWriter.write(schemaRegionPlan); - regionStatistics.setMLogLength(logWriter.position()); + try { + logWriter.write(schemaRegionPlan); + regionStatistics.setMLogLength(logWriter.position()); + } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); + throw e; + } } } @@ -359,6 +367,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { } } catch (IOException e) { logger.error(DataNodeSchemaMessages.CANNOT_FORCE_MLOG, schemaRegionId, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } @@ -394,6 +403,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { logger.info(DataNodeSchemaMessages.MLOG_RECOVERY_CHECK_POINT, mLogOffset); } catch (IOException e) { logger.warn(DataNodeSchemaMessages.CANNOT_GET_MLOG_CHECKPOINT, e.getMessage()); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } try (SchemaLogReader<ISchemaRegionPlan> mLogReader = new SchemaLogReader<>( @@ -428,6 +438,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { mLogReader.skip(offset); } catch (IOException e) { logger.error(DataNodeSchemaMessages.FAILED_TO_SKIP_MLOG, offset, schemaRegionDirPath, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } while (mLogReader.hasNext()) { plan = mLogReader.next(); @@ -468,6 +479,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { initialized = false; } catch (IOException e) { logger.error(DataNodeSchemaMessages.CANNOT_CLOSE_METADATA_LOG_WRITER, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } isClearing = false; } @@ -585,6 +597,7 @@ public class SchemaRegionPBTreeImpl implements ISchemaRegion { } catch (IOException | MetadataException e) { logger.error( DataNodeSchemaMessages.FAILED_TO_LOAD_SNAPSHOT, schemaRegionId, e.getMessage(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); try { initialized = false; isRecovering = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java index 96933bdcae1..3f9eb2c1bb3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/mem/snapshot/MemMTreeSnapshotUtil.java @@ -32,6 +32,7 @@ import org.apache.iotdb.commons.schema.node.visitor.MNodeVisitor; import org.apache.iotdb.commons.schema.view.LogicalViewSchema; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.schemaengine.rescon.MemSchemaRegionStatistics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.MemMTreeStore; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.mem.mnode.IMemMNode; @@ -364,6 +365,7 @@ public class MemMTreeSnapshotUtil { } } catch (IOException e) { logger.error(SERIALIZE_ERROR_INFO, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); return false; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java index 066116790f1..c621736ce23 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/flush/PBTreeFlushExecutor.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.flush; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.commons.schema.node.role.IDatabaseMNode; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.lock.LockManager; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.memory.IMemoryManager; import org.apache.iotdb.db.schemaengine.schemaregion.mtree.impl.pbtree.mnode.ICachedMNode; @@ -124,6 +125,7 @@ public class PBTreeFlushExecutor { DataNodeSchemaMessages.IO_EXCEPTION_UPDATING_SG_MNODE, updatedStorageGroupMNode.getFullPath(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw e; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java index 968a4792e9a..818c9a73109 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/schemaengine/schemaregion/mtree/impl/pbtree/schemafile/SchemaFile.java @@ -29,6 +29,7 @@ import org.apache.iotdb.commons.utils.PathUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.consensus.ConsensusFactory; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.exception.metadata.schemafile.SchemaFileNotExists; import org.apache.iotdb.db.i18n.DataNodeSchemaMessages; import org.apache.iotdb.db.schemaengine.metric.SchemaRegionCachedMetric; @@ -483,6 +484,7 @@ public class SchemaFile implements ISchemaFile { return true; } catch (IOException e) { logger.error(DataNodeSchemaMessages.FAILED_TO_CREATE_SCHEMA_FILE_SNAPSHOT, e.getMessage(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); FileUtils.deleteFileIfExist(schemaFileSnapshot); return false; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java index af2a867c339..42035ca0c09 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java @@ -34,7 +34,7 @@ public class DataNodeExceptionMetrics implements IMetricSet { private static final DataNodeExceptionMetrics INSTANCE = new DataNodeExceptionMetrics(); - private Counter fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + private Counter suspiciousDiskExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; private DataNodeExceptionMetrics() { // singleton @@ -42,21 +42,25 @@ public class DataNodeExceptionMetrics implements IMetricSet { @Override public void bindTo(AbstractMetricService metricService) { - fileSystemExceptionCounter = + suspiciousDiskExceptionCounter = metricService.getOrCreateCounter( - Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString(), MetricLevel.IMPORTANT); + Metric.SUSPICIOUS_DISK_EXCEPTION_COUNT.toString(), MetricLevel.IMPORTANT); } @Override public void unbindFrom(AbstractMetricService metricService) { - fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; - metricService.remove(MetricType.COUNTER, Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString()); + suspiciousDiskExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + metricService.remove(MetricType.COUNTER, Metric.SUSPICIOUS_DISK_EXCEPTION_COUNT.toString()); } - public void recordFileSystemException(Throwable throwable) { + public void recordSuspiciousDiskException(Throwable throwable) { + // Uses exact type matching instead of instanceof because subclasses of + // FileSystemException (e.g. NoSuchFileException, NotDirectoryException, + // AccessDeniedException) usually indicate logical file-state errors rather + // than real disk failures, so they should not be counted here. for (Throwable current = throwable; current != null; current = current.getCause()) { - if (current instanceof FileSystemException) { - fileSystemExceptionCounter.inc(); + if (current.getClass() == FileSystemException.class) { + suspiciousDiskExceptionCounter.inc(); return; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index fc06e7bc532..17b7e5b426d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -2157,7 +2157,7 @@ public class DataRegion implements IDataRegionForQuery { StorageEngineMessages .STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729, e); - DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); throw new WriteProcessException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java index 6959d3cbf03..9650196c5cf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.ChunkTypeInconsistentException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.service.metrics.CompactionMetrics; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionFileCountExceededException; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.exception.CompactionLastTimeCheckFailedException; @@ -185,6 +186,7 @@ public abstract class AbstractCompactionTask { chunkTypeInconsistentException.measurement, chunkTypeInconsistentException.offsetOfChunkHeader); } else { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); logger.error( StorageEngineMessages.STORAGE_LOG_COMPACTION_TASK_MEETS_ERROR_1002C659, storageGroupName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java index 77f13d3fe90..e80d45a8abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.service.metrics.CompactionMetrics; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.storageengine.dataregion.compaction.constant.CompactionTaskType; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.AbstractCompactionTask; import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task.CrossSpaceCompactionTask; @@ -119,6 +120,7 @@ public class CompactionScheduler { } catch (InterruptedException e) { throw e; } catch (Throwable e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); LOGGER.error(StorageEngineMessages.MEET_ERROR_IN_COMPACTION_SCHEDULE, e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 888c184f573..68727f03aa7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.flush.pool.FlushSubTaskPoolManager; @@ -213,6 +214,7 @@ public class MemTableFlushTask { WritingMetrics.WRITE_PLAN_INDICES, System.currentTimeMillis() - writePlanIndicesStartTime); } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new ExecutionException(e); } @@ -363,6 +365,7 @@ public class MemTableFlushTask { storageGroup, memTable, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); return; } long subTaskTime = System.currentTimeMillis() - starTime; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index caed4732cb1..7cfbb666c8c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -1868,7 +1868,7 @@ public class TsFileProcessor { dataRegionName, tsFileResource.getTsFile().getAbsolutePath(), e); - DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); try { logger.error( @@ -1884,6 +1884,7 @@ public class TsFileProcessor { dataRegionName, tsFileResource.getTsFile().getAbsolutePath(), e1); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e1); } // Release resource try { @@ -1931,6 +1932,7 @@ public class TsFileProcessor { .STORAGE_LOG_MEET_ERROR_WHEN_WRITING_INTO_MODIFICATIONFILE_FILE_OF_63B5E24A, tsFileResource.getTsFile().getAbsolutePath(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } finally { flushQueryLock.writeLock().unlock(); } @@ -1949,6 +1951,7 @@ public class TsFileProcessor { writer.getTsFileOutput().force(); } catch (IOException e) { logger.error(StorageEngineMessages.FSYNC_MEMTABLE_TO_DISK_ERROR, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } // Call flushed listener after memtable is released safely @@ -1977,7 +1980,7 @@ public class TsFileProcessor { logger.debug(StorageEngineMessages.FLUSHING_MEMTABLES_CLEAR, dataRegionName); } } catch (Exception e) { - DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); logger.error( StorageEngineMessages.STORAGE_LOG_MARKING_OR_ENDING_FILE_MEET_ERROR_5653B904, dataRegionName, @@ -1995,6 +1998,7 @@ public class TsFileProcessor { dataRegionName, tsFileResource.getTsFile().getAbsolutePath(), e1); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e1); } // Retry or set read-only if (retryCnt < 3) { @@ -2054,6 +2058,7 @@ public class TsFileProcessor { dataRegionName, tsFileResource.getTsFile().getName(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } @@ -2132,6 +2137,7 @@ public class TsFileProcessor { // When closing resource file, its corresponding mod file is also closed. tsFileResource.closeWithoutSettingStatus(); } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new TsFileProcessorException(e); } } @@ -2598,6 +2604,7 @@ public class TsFileProcessor { try { writer.close(); } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new TsFileProcessorException(e); } tsFileProcessorInfo.clear(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 097025e2e7d..d67cd88a4c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -642,7 +642,7 @@ public class WALBuffer extends AbstractWALBuffer { if (info.rollWALFileWriterListener != null) { info.rollWALFileWriterListener.fail(e); } - DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); } } else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging @@ -656,6 +656,7 @@ public class WALBuffer extends AbstractWALBuffer { .STORAGE_LOG_FAIL_TO_FSYNC_WAL_NODE_S_LOG_WRITER_CHANGE_SYSTEM_MODE_TO_7930160B, identifier, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); for (WALFlushListener fsyncListener : info.fsyncListeners) { fsyncListener.fail(e); } @@ -771,6 +772,7 @@ public class WALBuffer extends AbstractWALBuffer { currentWALFileWriter.close(); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_CLOSE_WAL_LOG_WRITER, identifier, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } checkpointManager.close(); @@ -846,6 +848,7 @@ public class WALBuffer extends AbstractWALBuffer { id, identifier, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } return Collections.emptySet(); }); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index a8245627763..a1bba3bad40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -111,6 +111,7 @@ public class CheckpointManager implements AutoCloseable { currentLogWriter.write(tmpBuffer); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_LOG_MAX_MEMTABLE_ID, maxMemTableId, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } // log global memTables' info makeGlobalInfoCP(); @@ -201,6 +202,7 @@ public class CheckpointManager implements AutoCloseable { currentLogWriter.write(cachedByteBuffer); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_MAKE_CHECKPOINT, checkpoint, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } finally { cachedByteBuffer.clear(); } @@ -219,6 +221,7 @@ public class CheckpointManager implements AutoCloseable { .STORAGE_LOG_FAIL_TO_FSYNC_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_6E1EE226, identifier, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); } @@ -239,7 +242,7 @@ public class CheckpointManager implements AutoCloseable { .STORAGE_LOG_FAIL_TO_ROLL_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_791DDAB7, identifier, e); - DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); } } finally { @@ -331,6 +334,7 @@ public class CheckpointManager implements AutoCloseable { currentLogWriter.close(); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_CLOSE_WAL_CHECKPOINT_WRITER, identifier, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } finally { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index af221813e7d..afa71a3fccd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -38,6 +38,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.UnsealedTsF import org.apache.iotdb.db.storageengine.dataregion.wal.utils.CheckpointFileUtils; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -208,6 +209,7 @@ public class WALNodeRecoverTask implements Runnable { walRepairWriter.repair(metaData); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_RECOVER_WAL_METADATA, walFile, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java index bd021f6d74f..f35326f40ca 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/SealedTsFileRecoverPerformer.java @@ -22,6 +22,7 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.recover.file; import org.apache.iotdb.db.exception.DataRegionException; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import java.io.IOException; @@ -46,6 +47,7 @@ public class SealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerformer try { reconstructResourceFile(); } catch (IOException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new DataRegionException( String.format( StorageEngineMessages diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java index f9696cc2617..7647a419aec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/file/UnsealedTsFileRecoverPerformer.java @@ -44,6 +44,7 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeInd import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALRecoverException; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.listener.WALRecoverListener; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.tsfile.file.metadata.ChunkMetadata; import org.apache.tsfile.file.metadata.IDeviceID; @@ -305,6 +306,7 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform databaseName, tsFileResource.getTsFile().getName(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } // if we put following codes in the 'if' clause above, this file can be continued writing @@ -314,6 +316,7 @@ public class UnsealedTsFileRecoverPerformer extends AbstractTsFileRecoverPerform tsFileResource.serialize(); FileTimeIndexCacheRecorder.getInstance().logFileTimeIndex(tsFileResource); } catch (IOException | ExecutionException e) { + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); throw new WALRecoverException(e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index 220be0cd0c6..b5141f54c9c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.consensus.DataRegionConsensusImpl; import org.apache.iotdb.db.exception.load.LoadFileException; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.pipe.agent.PipeDataNodeAgent; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.load.LoadTsFilePieceNode; import org.apache.iotdb.db.queryengine.plan.scheduler.load.LoadTsFileScheduler.LoadCommand; @@ -757,6 +758,7 @@ public class LoadTsFileManager { StorageEngineMessages.CLOSE_TSFILE_IO_WRITER_ERROR, entry.getValue().getFile().getPath(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } @@ -777,6 +779,7 @@ public class LoadTsFileManager { } catch (IOException e) { LOGGER.warn( StorageEngineMessages.CLOSE_MODIFICATION_FILE_ERROR, entry.getValue().getFile(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } @@ -790,6 +793,7 @@ public class LoadTsFileManager { LOGGER.info(StorageEngineMessages.TASK_DIR_NOT_EMPTY_SKIP_DELETE, taskDir.getPath()); } catch (IOException e) { LOGGER.warn(MESSAGE_DELETE_FAIL, taskDir.getPath(), e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } dataPartition2Writer = null; dataPartition2Resource = null; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java index 87c5cf00273..4ce7cc33f0d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/disk/TierManager.java @@ -25,6 +25,7 @@ import org.apache.iotdb.commons.disk.strategy.DirectoryStrategyType; import org.apache.iotdb.commons.exception.DiskSpaceInsufficientException; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.metrics.utils.FileStoreUtils; @@ -117,6 +118,7 @@ public class TierManager { tierDirs[i][j] = new File(tierDirs[i][j]).getCanonicalPath(); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_GET_CANONICAL_PATH, tierDirs[i][j], e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } break; case OBJECT_STORAGE: @@ -367,6 +369,7 @@ public class TierManager { filePath = file.getCanonicalFile().toPath(); } catch (IOException e) { logger.error(StorageEngineMessages.FAIL_TO_GET_CANONICAL_PATH, file, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); filePath = file.toPath(); } @@ -418,6 +421,7 @@ public class TierManager { } } catch (IOException e) { logger.error(StorageEngineMessages.FAILED_TO_STATISTIC_SIZE, fileStore, e); + DataNodeExceptionMetrics.getInstance().recordSuspiciousDiskException(e); } } } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index 5e6eab5701b..b35d2bc3575 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -94,7 +94,7 @@ public enum Metric { PAM_ALLOCATED_FAILURE_COUNT("primitive_array_manager_allocated_failure_count"), PAM_RELEASED_FAILURE_COUNT("primitive_array_manager_released_failure_count"), TABLE_DISK_USAGE_CACHE("table_disk_usage_cache"), - FILE_SYSTEM_EXCEPTION_COUNT("file_system_exception_count"), + SUSPICIOUS_DISK_EXCEPTION_COUNT("suspicious_disk_exception_count"), // compaction related DATA_WRITTEN("data_written"),
