This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 3b45611a39e branch-3.0: [opt](ms) print segment key bounds size when
prepare/commit/update rowset meta #44715 (#44888)
3b45611a39e is described below
commit 3b45611a39e26a929f6e4d80ec19ef8be13373d9
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 3 21:58:21 2024 +0800
branch-3.0: [opt](ms) print segment key bounds size when
prepare/commit/update rowset meta #44715 (#44888)
Cherry-picked from #44715
Co-authored-by: bobhan1 <[email protected]>
---
cloud/src/meta-service/meta_service.cpp | 32 +++++++++++++++++++++++++---
cloud/src/meta-service/meta_service_helper.h | 1 +
2 files changed, 30 insertions(+), 3 deletions(-)
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index 91a9dc0b554..c6b1c592b20 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1033,14 +1033,18 @@ void
MetaServiceImpl::prepare_rowset(::google::protobuf::RpcController* controll
prepare_rowset.SerializeToString(&val);
DCHECK_GT(prepare_rowset.expiration(), 0);
txn->put(prepare_rs_key, val);
+ std::size_t segment_key_bounds_bytes =
get_segments_key_bounds_bytes(rowset_meta);
LOG(INFO) << "put prepare_rs_key " << hex(prepare_rs_key) << " value_size
" << val.size()
- << " txn_id " << request->txn_id();
+ << " txn_id " << request->txn_id() << " segment_key_bounds_bytes
"
+ << segment_key_bounds_bytes;
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
LOG(WARNING) << "failed to prepare rowset, err=value too large"
<< ", txn_id=" << request->txn_id() << ", tablet_id="
<< tablet_id
<< ", rowset_id=" << rowset_id
+ << ", rowset_meta_bytes=" <<
rowset_meta.ByteSizeLong()
+ << ", segment_key_bounds_bytes=" <<
segment_key_bounds_bytes
<< ", rowset_meta=" << rowset_meta.ShortDebugString();
}
code = cast_as<ErrCategory::COMMIT>(err);
@@ -1167,15 +1171,18 @@ void
MetaServiceImpl::commit_rowset(::google::protobuf::RpcController* controlle
DCHECK_GT(rowset_meta.txn_expiration(), 0);
auto tmp_rs_val = rowset_meta.SerializeAsString();
txn->put(tmp_rs_key, tmp_rs_val);
+ std::size_t segment_key_bounds_bytes =
get_segments_key_bounds_bytes(rowset_meta);
LOG(INFO) << "put tmp_rs_key " << hex(tmp_rs_key) << " delete
recycle_rs_key "
<< hex(recycle_rs_key) << " value_size " << tmp_rs_val.size() <<
" txn_id "
- << request->txn_id();
+ << request->txn_id() << " segment_key_bounds_bytes " <<
segment_key_bounds_bytes;
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
LOG(WARNING) << "failed to commit rowset, err=value too large"
<< ", txn_id=" << request->txn_id() << ", tablet_id="
<< tablet_id
<< ", rowset_id=" << rowset_id
+ << ", rowset_meta_bytes=" <<
rowset_meta.ByteSizeLong()
+ << ", segment_key_bounds_bytes=" <<
segment_key_bounds_bytes
<< ", rowset_meta=" << rowset_meta.ShortDebugString();
}
code = cast_as<ErrCategory::COMMIT>(err);
@@ -1267,10 +1274,21 @@ void
MetaServiceImpl::update_tmp_rowset(::google::protobuf::RpcController* contr
}
txn->put(update_key, update_val);
+ std::size_t segment_key_bounds_bytes =
get_segments_key_bounds_bytes(rowset_meta);
LOG(INFO) << "xxx put "
- << "update_rowset_key " << hex(update_key) << " value_size " <<
update_val.size();
+ << "update_rowset_key " << hex(update_key) << " value_size " <<
update_val.size()
+ << " segment_key_bounds_bytes " << segment_key_bounds_bytes;
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
+ if (err == TxnErrorCode::TXN_VALUE_TOO_LARGE) {
+ const auto& rowset_id = rowset_meta.rowset_id_v2();
+ LOG(WARNING) << "failed to update tmp rowset, err=value too large"
+ << ", txn_id=" << request->txn_id() << ", tablet_id="
<< tablet_id
+ << ", rowset_id=" << rowset_id
+ << ", rowset_meta_bytes=" <<
rowset_meta.ByteSizeLong()
+ << ", segment_key_bounds_bytes=" <<
segment_key_bounds_bytes
+ << ", rowset_meta=" << rowset_meta.ShortDebugString();
+ }
code = cast_as<ErrCategory::COMMIT>(err);
ss << "failed to update rowset meta, err=" << err;
msg = ss.str();
@@ -2372,4 +2390,12 @@ MetaServiceResponseStatus
MetaServiceImpl::fix_tablet_stats(std::string cloud_un
return st;
}
+std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB&
rowset_meta) {
+ size_t ret {0};
+ for (const auto& key_bounds : rowset_meta.segments_key_bounds()) {
+ ret += key_bounds.ByteSizeLong();
+ }
+ return ret;
+}
+
} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service_helper.h
b/cloud/src/meta-service/meta_service_helper.h
index 4ef98ea2fb7..5355ac0eb61 100644
--- a/cloud/src/meta-service/meta_service_helper.h
+++ b/cloud/src/meta-service/meta_service_helper.h
@@ -248,4 +248,5 @@ void get_tablet_idx(MetaServiceCode& code, std::string&
msg, Transaction* txn,
bool is_dropped_tablet(Transaction* txn, const std::string& instance_id,
int64_t index_id,
int64_t partition_id);
+std::size_t get_segments_key_bounds_bytes(const doris::RowsetMetaCloudPB&
rowset_meta);
} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
