This is an automated email from the ASF dual-hosted git repository.
Hexiaoqiao pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4f9b20bf0a6 HDFS-17938. Add the metrics NumBlocks in datanode.
(#8566). Contributed by fuchaohong.
4f9b20bf0a6 is described below
commit 4f9b20bf0a637ac6b0c4900d7236a728a2782a1d
Author: fuchaohong <[email protected]>
AuthorDate: Fri Jul 3 20:41:39 2026 +0800
HDFS-17938. Add the metrics NumBlocks in datanode. (#8566). Contributed by
fuchaohong.
Signed-off-by: Ayush Saxena <[email protected]>
Signed-off-by: He Xiaoqiao <[email protected]>
---
.../datanode/fsdataset/impl/FsDatasetImpl.java | 5 ++++
.../datanode/fsdataset/impl/FsVolumeList.java | 12 +++++++++
.../datanode/metrics/DataNodeMetricHelper.java | 4 ++-
.../server/datanode/metrics/FSDatasetMBean.java | 5 ++++
.../hdfs/server/datanode/SimulatedFSDataset.java | 5 ++++
.../datanode/extdataset/ExternalDatasetImpl.java | 5 ++++
.../datanode/fsdataset/impl/TestFsDatasetImpl.java | 29 ++++++++++++++++++++++
7 files changed, 64 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 934c5faee2f..2db9789d277 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -3905,5 +3905,10 @@ public void setLastDirScannerFinishTime(long time) {
public long getPendingAsyncDeletions() {
return asyncDiskService.countPendingDeletions();
}
+
+ @Override
+ public long getNumBlocks() throws IOException {
+ return volumes.getNumBlocks();
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index a3d54865de0..e458861ee79 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -557,4 +557,16 @@ void shutdown() {
}
}
}
+
+ long getNumBlocks() throws IOException {
+ long numBlocks = 0L;
+ for (FsVolumeImpl v : volumes) {
+ try(FsVolumeReference ref = v.obtainReference()) {
+ numBlocks += v.getNumBlocks();
+ } catch (ClosedChannelException e) {
+ // ignore.
+ }
+ }
+ return numBlocks;
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java
index 24b92df1e4f..e381bbd3fbb 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/DataNodeMetricHelper.java
@@ -76,7 +76,9 @@ public static void getMetrics(MetricsCollector collector,
"Finish time of the last directory scan"),
beanClass.getLastDirScannerFinishTime())
.addGauge(Interns.info("PendingAsyncDeletions",
"The count of pending and running asynchronous disk operations"),
- beanClass.getPendingAsyncDeletions());
+ beanClass.getPendingAsyncDeletions())
+ .addGauge(Interns.info("NumBlocks", "Datanode number of blocks"),
+ beanClass.getNumBlocks());
}
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
index e559f84bf35..8da5e774348 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
@@ -132,4 +132,9 @@ public interface FSDatasetMBean extends MetricsSource {
* Returns the count of pending and running asynchronous disk operations.
*/
long getPendingAsyncDeletions();
+
+ /**
+ * Returns the number of blocks that the datanode
+ */
+ long getNumBlocks() throws IOException;
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 1ddc4e9602a..6f5ec1b0c52 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -1653,5 +1653,10 @@ public List<FsVolumeImpl> getVolumeList() {
public void setLastDirScannerFinishTime(long time) {
throw new UnsupportedOperationException();
}
+
+ @Override
+ public long getNumBlocks() throws IOException {
+ return 0L;
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index 24069fccdfa..ffb8de4a075 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -493,4 +493,9 @@ public long getLastDirScannerFinishTime() {
public long getPendingAsyncDeletions() {
return 0;
}
+
+ @Override
+ public long getNumBlocks() throws IOException {
+ return 0;
+ }
}
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index c68a1ced3a6..7c89406b28a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -2165,4 +2165,33 @@ public void testAppend() {
}
}
}
+
+ @Test
+ @Timeout(value = 30)
+ public void testGetNumBlocks() throws Exception {
+ Configuration config = new HdfsConfiguration();
+ config.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 1024);
+ try (MiniDFSCluster cluster = new MiniDFSCluster.Builder(config)
+ .numDataNodes(1)
+ .storagesPerDatanode(1)
+ .build()) {
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+ DataNode dataNode = cluster.getDataNodes().get(0);
+ FsDatasetImpl fsDataSetImpl = (FsDatasetImpl) dataNode.getFSDataset();
+
+ long initialBlocks = fsDataSetImpl.getNumBlocks();
+
+ int numFiles = 5;
+ int fileSize = 2048;
+ for (int i = 0; i < numFiles; i++) {
+ Path filePath = new Path("/testFile" + i);
+ DFSTestUtil.createFile(fs, filePath, fileSize, (short) 1, 0);
+ }
+
+ long expectedBlocks = initialBlocks + numFiles * (fileSize / 1024);
+ assertEquals(expectedBlocks, fsDataSetImpl.getNumBlocks(),
+ "Number of blocks should match expected count");
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
