This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new c989ac94672 branch-2.1: [chore](binlog) add ingesting/downloading
binlog latency metrics #48599 (#49002)
c989ac94672 is described below
commit c989ac94672b32f5893203d10632646a18a29d10
Author: walter <[email protected]>
AuthorDate: Fri Mar 14 11:22:23 2025 +0800
branch-2.1: [chore](binlog) add ingesting/downloading binlog latency
metrics #48599 (#49002)
cherry pick from #48599
---
be/src/http/action/download_binlog_action.cpp | 14 ++++++++++++++
be/src/service/backend_service.cpp | 6 ++++--
be/src/util/stopwatch.hpp | 14 ++++++++++++++
3 files changed, 32 insertions(+), 2 deletions(-)
diff --git a/be/src/http/action/download_binlog_action.cpp
b/be/src/http/action/download_binlog_action.cpp
index 912cc2f2ca3..07f552d70ac 100644
--- a/be/src/http/action/download_binlog_action.cpp
+++ b/be/src/http/action/download_binlog_action.cpp
@@ -34,6 +34,7 @@
#include "olap/storage_engine.h"
#include "olap/tablet_manager.h"
#include "runtime/exec_env.h"
+#include "util/stopwatch.hpp"
namespace doris {
@@ -47,6 +48,12 @@ const std::string kSegmentIndexParameter = "segment_index";
const std::string kSegmentIndexIdParameter = "segment_index_id";
const std::string kAcquireMD5Parameter = "acquire_md5";
+bvar::LatencyRecorder g_get_binlog_info_latency("doris_download_binlog",
"get_binlog_info");
+bvar::LatencyRecorder g_get_segment_file_latency("doris_download_binlog",
"get_segment_file");
+bvar::LatencyRecorder g_get_segment_index_file_latency("doris_download_binlog",
+
"get_segment_index_file");
+bvar::LatencyRecorder g_get_rowset_meta_latency("doris_download_binlog",
"get_rowset_meta");
+
// get http param, if no value throw exception
const auto& get_http_param(HttpRequest* req, const std::string& param_name) {
const auto& param = req->param(param_name);
@@ -143,6 +150,7 @@ void handle_get_segment_index_file(HttpRequest* req,
const auto& rowset_id = get_http_param(req, kRowsetIdParameter);
const auto& segment_index = get_http_param(req,
kSegmentIndexParameter);
const auto& segment_index_id = req->param(kSegmentIndexIdParameter);
+ auto segment_file_path = tablet->get_segment_filepath(rowset_id,
segment_index);
segment_index_file_path =
tablet->get_segment_index_filepath(rowset_id, segment_index,
segment_index_id);
is_acquire_md5 = !req->param(kAcquireMD5Parameter).empty();
@@ -218,14 +226,20 @@ void DownloadBinlogAction::handle(HttpRequest* req) {
const std::string& method = req->param(kMethodParameter);
// Step 3: dispatch
+ MonotonicStopWatch watch;
+ watch.start();
if (method == "get_binlog_info") {
handle_get_binlog_info(req);
+ g_get_binlog_info_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_file") {
handle_get_segment_file(req, _rate_limit_group.get());
+ g_get_segment_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_segment_index_file") {
handle_get_segment_index_file(req, _rate_limit_group.get());
+ g_get_segment_index_file_latency << watch.elapsed_time_microseconds();
} else if (method == "get_rowset_meta") {
handle_get_rowset_meta(req);
+ g_get_rowset_meta_latency << watch.elapsed_time_microseconds();
} else {
auto error_msg = fmt::format("invalid method: {}", method);
LOG(WARNING) << error_msg;
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index a6a5ba898ac..7403b979f8b 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -82,9 +82,10 @@ class TTransportException;
} // namespace apache
namespace doris {
-
namespace {
+bvar::LatencyRecorder g_ingest_binlog_latency("doris_backend_service",
"ingest_binlog");
+
struct IngestBinlogArg {
int64_t txn_id;
int64_t partition_id;
@@ -115,7 +116,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
std::vector<std::string> download_success_files;
Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus, &watch,
&total_download_bytes,
&total_download_files]() {
- auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() /
1000000);
+ g_ingest_binlog_latency << watch.elapsed_time_microseconds();
+ auto elapsed_time_ms =
static_cast<int64_t>(watch.elapsed_time_milliseconds());
double copy_rate = 0.0;
if (elapsed_time_ms > 0) {
copy_rate = total_download_bytes / ((double)elapsed_time_ms) /
1000;
diff --git a/be/src/util/stopwatch.hpp b/be/src/util/stopwatch.hpp
index 1c9857313be..9354b4694ad 100644
--- a/be/src/util/stopwatch.hpp
+++ b/be/src/util/stopwatch.hpp
@@ -75,6 +75,20 @@ public:
(end.tv_nsec - _start.tv_nsec);
}
+ // Return time in microseconds
+ uint64_t elapsed_time_microseconds() const { return elapsed_time() / 1000;
}
+
+ // Return time in milliseconds
+ uint64_t elapsed_time_milliseconds() const { return elapsed_time() / 1000
/ 1000; }
+
+ // Returns time in nanosecond.
+ int64_t elapsed_time_seconds(timespec end) const {
+ if (!_running) {
+ return _total_time / 1000L / 1000L / 1000L;
+ }
+ return end.tv_sec - _start.tv_sec;
+ }
+
private:
timespec _start;
uint64_t _total_time; // in nanosec
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
