This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 20976a571f6 branch-3.0: [feature](metrics) add catalog/db/table num
metrics #47891 (#47945)
20976a571f6 is described below
commit 20976a571f6b13c87231be211d4155efec84df43
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Mon Feb 24 11:29:58 2025 +0800
branch-3.0: [feature](metrics) add catalog/db/table num metrics #47891
(#47945)
Cherry-picked from #47891
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../java/org/apache/doris/catalog/Database.java | 4 ++
.../org/apache/doris/datasource/CatalogMgr.java | 4 ++
.../apache/doris/datasource/InternalCatalog.java | 4 ++
.../java/org/apache/doris/metric/MetricRepo.java | 34 +++++++++++++++++
.../java/org/apache/doris/metric/MetricsTest.java | 43 ++++++++++++++++++++++
5 files changed, 89 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 4f205be0381..4a6bc249b4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -1016,4 +1016,8 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>,
public String toString() {
return toJson();
}
+
+ public int getTableNum() {
+ return idToTable.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 19594f339b1..8ed93b78fd0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -840,4 +840,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
public Set<CatalogIf> getCopyOfCatalog() {
return new HashSet<>(idToCatalog.values());
}
+
+ public int getCatalogNum() {
+ return idToCatalog.size();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index c4a8983df6f..b6dc35b4562 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -267,6 +267,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
return Lists.newArrayList(idToDb.keySet());
}
+ public int getDbNum() {
+ return idToDb.size();
+ }
+
@Nullable
@Override
public Database getDbNullable(String dbName) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
index 14dd8834454..80800bc0738 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/metric/MetricRepo.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.alter.Alter;
import org.apache.doris.alter.AlterJobV2.JobType;
+import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
import org.apache.doris.cloud.system.CloudSystemInfoService;
import org.apache.doris.common.Config;
@@ -149,6 +150,11 @@ public final class MetricRepo {
public static Histogram HISTO_COMMIT_AND_PUBLISH_LATENCY;
+ // Catlaog/Database/Table num
+ public static GaugeMetric<Integer> GAUGE_CATALOG_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_DATABASE_NUM;
+ public static GaugeMetric<Integer> GAUGE_INTERNAL_TABLE_NUM;
+
private static Map<Pair<EtlJobType, JobState>, Long> loadJobNum =
Maps.newHashMap();
private static ScheduledThreadPoolExecutor metricTimer =
ThreadPoolManager.newDaemonScheduledThreadPool(1,
@@ -560,6 +566,34 @@ public final class MetricRepo {
HISTO_COMMIT_AND_PUBLISH_LATENCY = METRIC_REGISTER.histogram(
MetricRegistry.name("txn_commit_and_publish", "latency",
"ms"));
+ GAUGE_CATALOG_NUM = new GaugeMetric<Integer>("catalog_num",
+ MetricUnit.NOUNIT, "total catalog num") {
+ @Override
+ public Integer getValue() {
+ return Env.getCurrentEnv().getCatalogMgr().getCatalogNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_CATALOG_NUM);
+
+ GAUGE_INTERNAL_DATABASE_NUM = new
GaugeMetric<Integer>("internal_database_num",
+ MetricUnit.NOUNIT, "total internal database num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getDbNum();
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_DATABASE_NUM);
+
+ GAUGE_INTERNAL_TABLE_NUM = new
GaugeMetric<Integer>("internal_table_num",
+ MetricUnit.NOUNIT, "total internal table num") {
+ @Override
+ public Integer getValue() {
+ return
Env.getCurrentEnv().getCatalogMgr().getInternalCatalog().getAllDbs().stream()
+ .map(d -> (Database)
d).map(Database::getTableNum).reduce(0, Integer::sum);
+ }
+ };
+ DORIS_METRIC_REGISTER.addMetrics(GAUGE_INTERNAL_TABLE_NUM);
+
// init system metrics
initSystemMetrics();
CloudMetrics.init();
diff --git a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
index 5b8c4bdd728..3ce3b475153 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/metric/MetricsTest.java
@@ -19,6 +19,7 @@ package org.apache.doris.metric;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.JsonUtil;
+import org.apache.doris.metric.Metric.MetricUnit;
import org.apache.doris.monitor.jvm.JvmService;
import org.apache.doris.monitor.jvm.JvmStats;
@@ -122,4 +123,46 @@ public class MetricsTest {
Assert.assertTrue(size.get() <
JsonUtil.parseArray(finalMetricJson).size());
}
+
+ @Test
+ public void testCatalogAndDatabaseMetrics() {
+ List<Metric> catalogMetrics =
MetricRepo.getMetricsByName("catalog_num");
+ Assert.assertEquals(1, catalogMetrics.size());
+ GaugeMetric<Integer> catalogMetric = (GaugeMetric<Integer>)
catalogMetrics.get(0);
+ Assert.assertEquals("catalog_num", catalogMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, catalogMetric.getUnit());
+ Assert.assertEquals("total catalog num",
catalogMetric.getDescription());
+
+ List<Metric> dbMetrics =
MetricRepo.getMetricsByName("internal_database_num");
+ Assert.assertEquals(1, dbMetrics.size());
+ GaugeMetric<Integer> dbMetric = (GaugeMetric<Integer>)
dbMetrics.get(0);
+ Assert.assertEquals("internal_database_num", dbMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, dbMetric.getUnit());
+ Assert.assertEquals("total internal database num",
dbMetric.getDescription());
+
+ List<Metric> tableMetrics =
MetricRepo.getMetricsByName("internal_table_num");
+ Assert.assertEquals(1, tableMetrics.size());
+ GaugeMetric<Integer> tableMetric = (GaugeMetric<Integer>)
tableMetrics.get(0);
+ Assert.assertEquals("internal_table_num", tableMetric.getName());
+ Assert.assertEquals(MetricUnit.NOUNIT, tableMetric.getUnit());
+ Assert.assertEquals("total internal table num",
tableMetric.getDescription());
+
+ // Test metrics in Prometheus format
+ MetricVisitor visitor = new PrometheusMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(visitor);
+ String metricResult = visitor.finish();
+
+ Assert.assertTrue(metricResult.contains("# TYPE doris_fe_catalog_num
gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_database_num gauge"));
+ Assert.assertTrue(metricResult.contains("# TYPE
doris_fe_internal_table_num gauge"));
+
+ // Test metrics in JSON format
+ JsonMetricVisitor jsonVisitor = new JsonMetricVisitor();
+ MetricRepo.DORIS_METRIC_REGISTER.accept(jsonVisitor);
+ String jsonResult = jsonVisitor.finish();
+
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_catalog_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_database_num\""));
+
Assert.assertTrue(jsonResult.contains("\"metric\":\"doris_fe_internal_table_num\""));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
