This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8f79f175f86 [chore](cloud) Add show hotspot tablet API on BE (#35527)
8f79f175f86 is described below
commit 8f79f175f867ee70f295ec4c59e1fe02b2becbc4
Author: Gavin Chou <[email protected]>
AuthorDate: Sat Jun 1 22:57:58 2024 +0800
[chore](cloud) Add show hotspot tablet API on BE (#35527)
Usage:
* Get all tablets {metrics} with descending order `curl
beip:http_port/api/hotspot/tablet?metrics={metrics}`
* Get top n tablets {metrics} with descending order `curl
beip:http_port/api/hotspot/tablet?metrics={metrics}&topn={n}`
"metrics" must be specified, which is one of
* read_block
* write
* compaction
* num_rowsets
* num_cumu_rowsets
* num_base_rowsets
Co-authored-by: plat1ko <[email protected]>
TODO: Add support for local mode.
---
be/src/cloud/cloud_rowset_builder.cpp | 1 +
be/src/http/action/show_hotspot_action.cpp | 170 +++++++++++++++++++++++++++++
be/src/http/action/show_hotspot_action.h | 38 +++++++
be/src/olap/base_tablet.h | 3 +
be/src/olap/compaction.cpp | 2 +
be/src/service/http_service.cpp | 3 +
be/src/vec/exec/scan/new_olap_scanner.cpp | 2 +
7 files changed, 219 insertions(+)
diff --git a/be/src/cloud/cloud_rowset_builder.cpp
b/be/src/cloud/cloud_rowset_builder.cpp
index 3585878cd72..b8c5b718abf 100644
--- a/be/src/cloud/cloud_rowset_builder.cpp
+++ b/be/src/cloud/cloud_rowset_builder.cpp
@@ -108,6 +108,7 @@ void CloudRowsetBuilder::update_tablet_stats() {
tablet->fetch_add_approximate_data_size(_rowset->data_disk_size());
tablet->fetch_add_approximate_cumu_num_rowsets(1);
tablet->fetch_add_approximate_cumu_num_deltas(_rowset->num_segments());
+ tablet->write_count.fetch_add(1, std::memory_order_relaxed);
}
CloudTablet* CloudRowsetBuilder::cloud_tablet() {
diff --git a/be/src/http/action/show_hotspot_action.cpp
b/be/src/http/action/show_hotspot_action.cpp
new file mode 100644
index 00000000000..1164e61135e
--- /dev/null
+++ b/be/src/http/action/show_hotspot_action.cpp
@@ -0,0 +1,170 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "show_hotspot_action.h"
+
+#include <queue>
+#include <string>
+
+#include "cloud/cloud_tablet_mgr.h"
+#include "http/http_channel.h"
+#include "http/http_request.h"
+
+namespace doris {
+namespace {
+
+enum class Metrics {
+ READ_BLOCK = 0,
+ WRITE = 1,
+ COMPACTION = 2,
+ NUM_ROWSETS = 3,
+ NUM_BASE_ROWSETS = 4,
+ NUM_CUMU_ROWSETS = 5,
+ UNKNOWN = 100000,
+};
+
+Status check_param(HttpRequest* req, int& top_n, Metrics& metrics) {
+ const std::string TOPN_PARAM = "topn";
+
+ auto& topn_str = req->param(TOPN_PARAM);
+ if (!topn_str.empty()) {
+ try {
+ top_n = std::stoi(topn_str);
+ } catch (const std::exception& e) {
+ return Status::InternalError("convert topn failed, {}", e.what());
+ }
+ }
+
+ const std::string METRICS_PARAM = "metrics";
+ auto& metrics_str = req->param(METRICS_PARAM);
+ if (metrics_str.empty()) {
+ return Status::InternalError("metrics must be specified");
+ }
+
+ if (metrics_str == "read_block") {
+ metrics = Metrics::READ_BLOCK;
+ } else if (metrics_str == "write") {
+ metrics = Metrics::WRITE;
+ } else if (metrics_str == "compaction") {
+ metrics = Metrics::COMPACTION;
+ } else if (metrics_str == "num_rowsets") {
+ metrics = Metrics::NUM_ROWSETS;
+ } else if (metrics_str == "num_cumu_rowsets") {
+ metrics = Metrics::NUM_CUMU_ROWSETS;
+ } else if (metrics_str == "num_base_rowsets") {
+ metrics = Metrics::NUM_BASE_ROWSETS;
+ } else {
+ return Status::InternalError("unknown metrics: {}", metrics_str);
+ }
+
+ return Status::OK();
+}
+
+struct TabletCounter {
+ int64_t tablet_id {0};
+ int64_t count {0};
+};
+
+struct Comparator {
+ constexpr bool operator()(const TabletCounter& lhs, const TabletCounter&
rhs) const {
+ return lhs.count > rhs.count;
+ }
+};
+
+using MinHeap = std::priority_queue<TabletCounter, std::vector<TabletCounter>,
Comparator>;
+
+} // namespace
+
+void ShowHotspotAction::handle(HttpRequest* req) {
+ int topn = 0;
+ Metrics metrics {Metrics::UNKNOWN};
+ auto st = check_param(req, topn, metrics);
+ if (!st.ok()) [[unlikely]] {
+ HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, st.to_string());
+ return;
+ }
+
+ std::function<int64_t(CloudTablet&)> count_fn;
+ switch (metrics) {
+ case Metrics::READ_BLOCK:
+ count_fn = [](auto&& t) { return
t.read_block_count.load(std::memory_order_relaxed); };
+ break;
+ case Metrics::WRITE:
+ count_fn = [](auto&& t) { return
t.write_count.load(std::memory_order_relaxed); };
+ break;
+ case Metrics::COMPACTION:
+ count_fn = [](auto&& t) { return
t.compaction_count.load(std::memory_order_relaxed); };
+ break;
+ case Metrics::NUM_ROWSETS:
+ count_fn = [](auto&& t) { return
t.fetch_add_approximate_num_rowsets(0); };
+ break;
+ case Metrics::NUM_BASE_ROWSETS:
+ count_fn = [](auto&& t) {
+ return t.fetch_add_approximate_num_rowsets(0) -
+ t.fetch_add_approximate_cumu_num_rowsets(0);
+ };
+ break;
+ case Metrics::NUM_CUMU_ROWSETS:
+ count_fn = [](auto&& t) { return
t.fetch_add_approximate_cumu_num_rowsets(0); };
+ break;
+ default:
+ break;
+ }
+
+ if (!count_fn) {
+ HttpChannel::send_reply(req, HttpStatus::BAD_REQUEST, "metrics not
specified");
+ return;
+ }
+
+ auto tablets = _storage_engine.tablet_mgr().get_weak_tablets();
+ std::vector<TabletCounter> buffer;
+ buffer.reserve(tablets.size());
+ for (auto&& t : tablets) {
+ if (auto tablet = t.lock(); tablet) {
+ buffer.push_back({tablet->tablet_id(), count_fn(*tablet)});
+ }
+ }
+
+ if (topn <= 0) {
+ topn = tablets.size();
+ }
+
+ MinHeap min_heap;
+ for (auto&& counter : buffer) {
+ min_heap.push(counter);
+ if (min_heap.size() > topn) {
+ min_heap.pop();
+ }
+ }
+
+ buffer.resize(0);
+ while (!min_heap.empty()) {
+ buffer.push_back(min_heap.top());
+ min_heap.pop();
+ }
+
+ std::string res;
+ res.reserve(buffer.size() * 20);
+ // Descending order
+ std::for_each(buffer.rbegin(), buffer.rend(), [&res](auto&& counter) {
+ res += fmt::format("{} {}\n", counter.tablet_id, counter.count);
+ });
+
+ HttpChannel::send_reply(req, HttpStatus::OK, res);
+}
+
+} // namespace doris
diff --git a/be/src/http/action/show_hotspot_action.h
b/be/src/http/action/show_hotspot_action.h
new file mode 100644
index 00000000000..a18fb945cce
--- /dev/null
+++ b/be/src/http/action/show_hotspot_action.h
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "cloud/cloud_storage_engine.h"
+#include "http/http_handler.h"
+
+namespace doris {
+class CloudStorageEngine;
+
+class ShowHotspotAction final : public HttpHandler {
+public:
+ ShowHotspotAction(CloudStorageEngine& eng) : _storage_engine(eng) {}
+
+ ~ShowHotspotAction() override = default;
+
+ void handle(HttpRequest* req) override;
+
+private:
+ CloudStorageEngine& _storage_engine;
+};
+
+} // namespace doris
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 7043df6a25a..58b72f2ccba 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -299,6 +299,9 @@ public:
IntCounter* flush_bytes = nullptr;
IntCounter* flush_finish_count = nullptr;
std::atomic<int64_t> published_count = 0;
+ std::atomic<int64_t> read_block_count = 0;
+ std::atomic<int64_t> write_count = 0;
+ std::atomic<int64_t> compaction_count = 0;
};
} /* namespace doris */
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index 061d667df8b..847d50e36cb 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -22,6 +22,7 @@
#include <glog/logging.h>
#include <algorithm>
+#include <atomic>
#include <cstdlib>
#include <list>
#include <map>
@@ -386,6 +387,7 @@ Status CompactionMixin::execute_compact() {
data_dir->disks_compaction_num_increment(1);
Status st = execute_compact_impl(permits);
+ _tablet->compaction_count.fetch_add(1, std::memory_order_relaxed);
data_dir->disks_compaction_score_increment(-permits);
data_dir->disks_compaction_num_increment(-1);
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index ee5b4174387..1cbca7bacf7 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -53,6 +53,7 @@
#include "http/action/report_action.h"
#include "http/action/reset_rpc_channel_action.h"
#include "http/action/restore_tablet_action.h"
+#include "http/action/show_hotspot_action.h"
#include "http/action/snapshot_action.h"
#include "http/action/stream_load.h"
#include "http/action/stream_load_2pc.h"
@@ -380,6 +381,8 @@ void
HttpService::register_cloud_handler(CloudStorageEngine& engine) {
ClearFileCacheAction* clear_file_cache_action = _pool.add(new
ClearFileCacheAction());
_ev_http_server->register_handler(HttpMethod::POST,
"/api/clear_file_cache",
clear_file_cache_action);
+ auto* show_hotspot_action = _pool.add(new ShowHotspotAction(engine));
+ _ev_http_server->register_handler(HttpMethod::GET, "/api/hotspot/tablet",
show_hotspot_action);
}
// NOLINTEND(readability-function-size)
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index 4507afc2cb2..15709642fa2 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -25,6 +25,7 @@
#include <algorithm>
#include <array>
+#include <atomic>
#include <iterator>
#include <ostream>
#include <set>
@@ -527,6 +528,7 @@ Status NewOlapScanner::_get_block_impl(RuntimeState* state,
Block* block, bool*
_profile_updated = _tablet_reader->update_profile(_profile);
}
if (block->rows() > 0) {
+ _tablet_reader_params.tablet->read_block_count.fetch_add(1,
std::memory_order_relaxed);
*eof = false;
}
_update_realtime_counters();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
