This is an automated email from the ASF dual-hosted git repository. shuwenwei pushed a commit to branch diskExceptionCounter in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 66df54bcaf89ef86d4447a36bc6d15c334e365e5 Author: shuwenwei <[email protected]> AuthorDate: Mon Aug 31 12:26:21 2026 +0800 feat(metrics): record DataNode file system exceptions --- .../service/metrics/DataNodeExceptionMetrics.java | 68 ++++++++++++++++++++++ .../db/service/metrics/DataNodeMetricsHelper.java | 1 + .../db/storageengine/dataregion/DataRegion.java | 2 + .../dataregion/memtable/TsFileProcessor.java | 3 + .../dataregion/wal/buffer/WALBuffer.java | 2 + .../wal/checkpoint/CheckpointManager.java | 2 + .../iotdb/commons/service/metric/enums/Metric.java | 1 + 7 files changed, 79 insertions(+) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java new file mode 100644 index 00000000000..af2a867c339 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeExceptionMetrics.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.service.metrics; + +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.metrics.AbstractMetricService; +import org.apache.iotdb.metrics.impl.DoNothingMetricManager; +import org.apache.iotdb.metrics.metricsets.IMetricSet; +import org.apache.iotdb.metrics.type.Counter; +import org.apache.iotdb.metrics.utils.MetricLevel; +import org.apache.iotdb.metrics.utils.MetricType; + +import java.nio.file.FileSystemException; + +/** Records DataNode exceptions whose cause chain contains a {@link FileSystemException}. */ +public class DataNodeExceptionMetrics implements IMetricSet { + + private static final DataNodeExceptionMetrics INSTANCE = new DataNodeExceptionMetrics(); + + private Counter fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + + private DataNodeExceptionMetrics() { + // singleton + } + + @Override + public void bindTo(AbstractMetricService metricService) { + fileSystemExceptionCounter = + metricService.getOrCreateCounter( + Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString(), MetricLevel.IMPORTANT); + } + + @Override + public void unbindFrom(AbstractMetricService metricService) { + fileSystemExceptionCounter = DoNothingMetricManager.DO_NOTHING_COUNTER; + metricService.remove(MetricType.COUNTER, Metric.FILE_SYSTEM_EXCEPTION_COUNT.toString()); + } + + public void recordFileSystemException(Throwable throwable) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current instanceof FileSystemException) { + fileSystemExceptionCounter.inc(); + return; + } + } + } + + public static DataNodeExceptionMetrics getInstance() { + return INSTANCE; + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java index e2204e8cf0b..9f6f4e57201 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/service/metrics/DataNodeMetricsHelper.java @@ -69,6 +69,7 @@ public class DataNodeMetricsHelper { metricService.addMetricSet(CompactionMetrics.getInstance()); metricService.addMetricSet(new ProcessMetrics()); metricService.addMetricSet(new DiskMetrics(IoTDBConstant.DN_ROLE)); + metricService.addMetricSet(DataNodeExceptionMetrics.getInstance()); metricService.addMetricSet(new NetMetrics(IoTDBConstant.DN_ROLE)); metricService.addMetricSet(ClientManagerMetrics.getInstance()); metricService.addMetricSet(RPCServiceThriftHandlerMetrics.getInstance()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index cfdc63edbdf..fc06e7bc532 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -101,6 +101,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.fetcher.cache.Tr import org.apache.iotdb.db.schemaengine.table.DataNodeTableCache; import org.apache.iotdb.db.service.SettleService; import org.apache.iotdb.db.service.metrics.CompactionMetrics; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.service.metrics.FileMetrics; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.StorageEngine; @@ -2156,6 +2157,7 @@ public class DataRegion implements IDataRegionForQuery { StorageEngineMessages .STORAGE_LOG_MEET_IOEXCEPTION_WHEN_CREATING_TSFILEPROCESSOR_CHANGE_SYSTEM_4337F729, e); + DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); throw new WriteProcessException( String.format( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java index 759b58d4966..caed4732cb1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java @@ -50,6 +50,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTablet import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalDeleteDataNode; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.RelationalInsertTabletNode; import org.apache.iotdb.db.schemaengine.schemaregion.utils.ResourceByPathUtils; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.db.storageengine.dataregion.DataRegionInfo; @@ -1867,6 +1868,7 @@ public class TsFileProcessor { dataRegionName, tsFileResource.getTsFile().getAbsolutePath(), e); + DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); try { logger.error( @@ -1975,6 +1977,7 @@ public class TsFileProcessor { logger.debug(StorageEngineMessages.FLUSHING_MEMTABLES_CLEAR, dataRegionName); } } catch (Exception e) { + DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); logger.error( StorageEngineMessages.STORAGE_LOG_MARKING_OR_ENDING_FILE_MEET_ERROR_5653B904, dataRegionName, diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 6921abecf75..097025e2e7d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -27,6 +27,7 @@ import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.SearchNode; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.CheckpointManager; @@ -641,6 +642,7 @@ public class WALBuffer extends AbstractWALBuffer { if (info.rollWALFileWriterListener != null) { info.rollWALFileWriterListener.fail(e); } + DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); } } else if (forceFlag) { // force os cache to the storage device, avoid force twice by judging diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index 1fc84166e05..a8245627763 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -24,6 +24,7 @@ import org.apache.iotdb.commons.file.SystemFileFactory; import org.apache.iotdb.db.conf.IoTDBConfig; import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; +import org.apache.iotdb.db.service.metrics.DataNodeExceptionMetrics; import org.apache.iotdb.db.service.metrics.WritingMetrics; import org.apache.iotdb.db.storageengine.dataregion.memtable.TsFileProcessor; import org.apache.iotdb.db.storageengine.dataregion.wal.io.CheckpointWriter; @@ -238,6 +239,7 @@ public class CheckpointManager implements AutoCloseable { .STORAGE_LOG_FAIL_TO_ROLL_WAL_NODE_S_CHECKPOINT_WRITER_CHANGE_SYSTEM_791DDAB7, identifier, e); + DataNodeExceptionMetrics.getInstance().recordFileSystemException(e); CommonDescriptor.getInstance().getConfig().handleUnrecoverableError(); } } finally { diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java index b41d8fb53ac..5e6eab5701b 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/service/metric/enums/Metric.java @@ -94,6 +94,7 @@ public enum Metric { PAM_ALLOCATED_FAILURE_COUNT("primitive_array_manager_allocated_failure_count"), PAM_RELEASED_FAILURE_COUNT("primitive_array_manager_released_failure_count"), TABLE_DISK_USAGE_CACHE("table_disk_usage_cache"), + FILE_SYSTEM_EXCEPTION_COUNT("file_system_exception_count"), // compaction related DATA_WRITTEN("data_written"),
