This is an automated email from the ASF dual-hosted git repository. morningman pushed a commit to branch branch-1.2-lts in repository https://gitbox.apache.org/repos/asf/doris.git
commit df5ff77587ef471e3a685405d21f88ab16f3a427 Author: Xinyi Zou <[email protected]> AuthorDate: Mon Jan 30 09:55:32 2023 +0800 [improvement](metrics) Metrics add all rowset nums and segment nums (#16208) --- be/src/common/daemon.cpp | 6 ++++++ be/src/olap/tablet.h | 11 +++++++++++ be/src/olap/tablet_manager.cpp | 22 ++++++++++++++++++++++ be/src/olap/tablet_manager.h | 3 +++ be/src/util/doris_metrics.cpp | 6 ++++++ be/src/util/doris_metrics.h | 3 +++ 6 files changed, 51 insertions(+) diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp index 149b2f1ce4..2f6afb99b7 100644 --- a/be/src/common/daemon.cpp +++ b/be/src/common/daemon.cpp @@ -49,6 +49,7 @@ #include "exprs/utility_functions.h" #include "geo/geo_functions.h" #include "olap/options.h" +#include "olap/storage_engine.h" #include "runtime/bufferpool/buffer_pool.h" #include "runtime/exec_env.h" #include "runtime/fragment_mgr.h" @@ -336,6 +337,11 @@ void Daemon::calculate_metrics_thread() { DorisMetrics::instance()->system_metrics()->get_network_traffic( &lst_net_send_bytes, &lst_net_receive_bytes); } + + DorisMetrics::instance()->all_rowset_nums->set_value( + StorageEngine::instance()->tablet_manager()->get_rowset_nums()); + DorisMetrics::instance()->all_segment_nums->set_value( + StorageEngine::instance()->tablet_manager()->get_segment_nums()); } } while (!_stop_background_threads_latch.wait_for(std::chrono::seconds(15))); } diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h index dcbb0097fc..4b444f13ec 100644 --- a/be/src/olap/tablet.h +++ b/be/src/olap/tablet.h @@ -97,6 +97,7 @@ public: size_t num_rows(); int version_count() const; + uint64_t segment_count() const; Version max_version() const; Version max_version_unlocked() const; CumulativeCompactionPolicy* cumulative_compaction_policy(); @@ -563,6 +564,7 @@ inline size_t Tablet::num_rows() { } inline int Tablet::version_count() const { + std::shared_lock rdlock(_meta_lock); return _tablet_meta->version_count(); } @@ -570,6 +572,15 @@ inline Version Tablet::max_version() const { return _tablet_meta->max_version(); } +inline uint64_t Tablet::segment_count() const { + std::shared_lock rdlock(_meta_lock); + uint64_t segment_nums = 0; + for (auto& rs_meta : _tablet_meta->all_rs_metas()) { + segment_nums += rs_meta->num_segments(); + } + return segment_nums; +} + inline Version Tablet::max_version_unlocked() const { return _tablet_meta->max_version(); } diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp index 1a5183b080..23c3e7c72c 100644 --- a/be/src/olap/tablet_manager.cpp +++ b/be/src/olap/tablet_manager.cpp @@ -598,6 +598,28 @@ std::vector<TabletSharedPtr> TabletManager::get_all_tablet() { return res; } +uint64_t TabletManager::get_rowset_nums() { + uint64_t rowset_nums = 0; + for (const auto& tablets_shard : _tablets_shards) { + std::shared_lock rdlock(tablets_shard.lock); + for (const auto& tablet_map : tablets_shard.tablet_map) { + rowset_nums += tablet_map.second->version_count(); + } + } + return rowset_nums; +} + +uint64_t TabletManager::get_segment_nums() { + uint64_t segment_nums = 0; + for (const auto& tablets_shard : _tablets_shards) { + std::shared_lock rdlock(tablets_shard.lock); + for (const auto& tablet_map : tablets_shard.tablet_map) { + segment_nums += tablet_map.second->segment_count(); + } + } + return segment_nums; +} + bool TabletManager::get_tablet_id_and_schema_hash_from_path(const string& path, TTabletId* tablet_id, TSchemaHash* schema_hash) { diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h index 75c29bdac7..bbb298cae2 100644 --- a/be/src/olap/tablet_manager.h +++ b/be/src/olap/tablet_manager.h @@ -80,6 +80,9 @@ public: std::vector<TabletSharedPtr> get_all_tablet(); + uint64_t get_rowset_nums(); + uint64_t get_segment_nums(); + // Extract tablet_id and schema_hash from given path. // // The normal path pattern is like "/data/{shard_id}/{tablet_id}/{schema_hash}/xxx.data". diff --git a/be/src/util/doris_metrics.cpp b/be/src/util/doris_metrics.cpp index 6f61ef7c36..586a66fbb9 100644 --- a/be/src/util/doris_metrics.cpp +++ b/be/src/util/doris_metrics.cpp @@ -136,6 +136,9 @@ DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(process_fd_num_limit_hard, MetricUnit::NOUNIT DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_cumulative_max_compaction_score, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(tablet_base_max_compaction_score, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_rowset_nums, MetricUnit::NOUNIT); +DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(all_segment_nums, MetricUnit::NOUNIT); + DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_used_permits, MetricUnit::NOUNIT); DEFINE_GAUGE_METRIC_PROTOTYPE_2ARG(compaction_waitting_permits, MetricUnit::NOUNIT); @@ -259,6 +262,9 @@ DorisMetrics::DorisMetrics() : _metric_registry(_s_registry_name) { INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_cumulative_max_compaction_score); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, tablet_base_max_compaction_score); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_rowset_nums); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, all_segment_nums); + INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_used_permits); INT_GAUGE_METRIC_REGISTER(_server_metric_entity, compaction_waitting_permits); diff --git a/be/src/util/doris_metrics.h b/be/src/util/doris_metrics.h index d117456d01..300c35c9ec 100644 --- a/be/src/util/doris_metrics.h +++ b/be/src/util/doris_metrics.h @@ -137,6 +137,9 @@ public: IntGauge* tablet_cumulative_max_compaction_score; IntGauge* tablet_base_max_compaction_score; + IntGauge* all_rowset_nums; + IntGauge* all_segment_nums; + // permits have been used for all compaction tasks IntGauge* compaction_used_permits; // permits required by the compaction task which is waiting for permits --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
