This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e681c055a49 [chore](binlog) Log download binlog cost (#47360) (#47575)
e681c055a49 is described below
commit e681c055a493c5aeae424f42bdf65541371e1fc0
Author: walter <[email protected]>
AuthorDate: Sat Feb 8 16:22:23 2025 +0800
[chore](binlog) Log download binlog cost (#47360) (#47575)
cherry-pick #47360
---
be/src/service/backend_service.cpp | 21 +++++++++++++++++++--
1 file changed, 19 insertions(+), 2 deletions(-)
diff --git a/be/src/service/backend_service.cpp
b/be/src/service/backend_service.cpp
index 96785c9fd1a..a2a04db4c53 100644
--- a/be/src/service/backend_service.cpp
+++ b/be/src/service/backend_service.cpp
@@ -107,10 +107,23 @@ void _ingest_binlog(IngestBinlogArg* arg) {
auto& request = arg->request;
+ MonotonicStopWatch watch;
+ watch.start();
+ int64_t total_download_bytes = 0;
+ int64_t total_download_files = 0;
TStatus tstatus;
std::vector<std::string> download_success_files;
- Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus]() {
- LOG(INFO) << "ingest binlog. result: " <<
apache::thrift::ThriftDebugString(tstatus);
+ Defer defer {[=, &tstatus, ingest_binlog_tstatus = arg->tstatus, &watch,
&total_download_bytes,
+ &total_download_files]() {
+ auto elapsed_time_ms = static_cast<int64_t>(watch.elapsed_time() /
1000000);
+ double copy_rate = 0.0;
+ if (elapsed_time_ms > 0) {
+ copy_rate = total_download_bytes / ((double)elapsed_time_ms) /
1000;
+ }
+ LOG(INFO) << "ingest binlog elapsed " << elapsed_time_ms << " ms,
download "
+ << total_download_files << " files, total " <<
total_download_bytes
+ << " bytes, avg rate " << copy_rate
+ << " MB/s. result: " <<
apache::thrift::ThriftDebugString(tstatus);
if (tstatus.status_code != TStatusCode::OK) {
// abort txn
StorageEngine::instance()->txn_manager()->abort_txn(partition_id,
txn_id,
@@ -263,6 +276,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
status.to_thrift(&tstatus);
return;
}
+ total_download_bytes = total_size;
+ total_download_files = num_segments;
// Step 5.3: get all segment files
for (int64_t segment_index = 0; segment_index < num_segments;
++segment_index) {
@@ -435,6 +450,8 @@ void _ingest_binlog(IngestBinlogArg* arg) {
status.to_thrift(&tstatus);
return;
}
+ total_download_bytes += total_index_size;
+ total_download_files += segment_index_file_urls.size();
// Step 6.3: get all segment index files
DCHECK(segment_index_file_sizes.size() == segment_index_file_names.size());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
