This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5668770f03c [Fix](MS) Fix get tablet stats value api (#35880)
5668770f03c is described below
commit 5668770f03c3f50590a6c6d57bf7dde620cf6c7e
Author: plat1ko <[email protected]>
AuthorDate: Wed Jun 5 08:45:28 2024 +0800
[Fix](MS) Fix get tablet stats value api (#35880)
Fix get tablet stats value api
---
cloud/src/meta-service/http_encode_key.cpp | 59 +++++++++++--
.../src/meta-service/meta_service_tablet_stats.cpp | 98 +++++++++++-----------
cloud/src/meta-service/meta_service_tablet_stats.h | 5 ++
cloud/test/http_encode_key_test.cpp | 36 +++++++-
4 files changed, 137 insertions(+), 61 deletions(-)
diff --git a/cloud/src/meta-service/http_encode_key.cpp
b/cloud/src/meta-service/http_encode_key.cpp
index f70f2accb50..e6433e57f9e 100644
--- a/cloud/src/meta-service/http_encode_key.cpp
+++ b/cloud/src/meta-service/http_encode_key.cpp
@@ -19,6 +19,7 @@
#include <fmt/format.h>
#include <gen_cpp/cloud.pb.h>
+#include <bit>
#include <iomanip>
#include <sstream>
#include <string>
@@ -30,10 +31,12 @@
#include "common/sync_point.h"
#include "common/util.h"
+#include "meta-service/codec.h"
#include "meta-service/doris_txn.h"
#include "meta-service/keys.h"
#include "meta-service/meta_service_http.h"
#include "meta-service/meta_service_schema.h"
+#include "meta-service/meta_service_tablet_stats.h"
#include "meta-service/txn_kv.h"
#include "meta-service/txn_kv_error.h"
@@ -130,6 +133,32 @@ static std::string parse_tablet_schema(const ValueBuf&
buf) {
return proto_to_json(pb);
}
+static std::string parse_tablet_stats(const ValueBuf& buf) {
+ if (buf.iters.empty()) {
+ return "";
+ }
+
+ TabletStatsPB stats;
+ auto&& it = buf.iters[0];
+ if (!it->has_next()) {
+ return "";
+ }
+
+ auto [k, v] = it->next();
+ stats.ParseFromArray(v.data(), v.size());
+
+ // Parse split tablet stats
+ TabletStats detached_stats;
+ int ret = get_detached_tablet_stats(*it, detached_stats);
+ if (ret != 0) {
+ return "";
+ }
+
+ merge_tablet_stats(stats, detached_stats);
+
+ return proto_to_json(stats);
+}
+
// See keys.h to get all types of key, e.g: MetaRowsetKeyInfo
// key_type -> {{param1, param2 ...}, encoding_func, value_parsing_func}
// clang-format off
@@ -150,7 +179,7 @@ static std::unordered_map<std::string_view,
{"RecyclePartKey", {{"instance_id", "part_id"},
[](param_type& p) { return
recycle_partition_key(KeyInfoSetter<RecyclePartKeyInfo>{p}.get()); }
, parse<RecyclePartitionPB>}} ,
{"RecycleRowsetKey", {{"instance_id", "tablet_id", "rowset_id"},
[](param_type& p) { return
recycle_rowset_key(KeyInfoSetter<RecycleRowsetKeyInfo>{p}.get()); }
, parse<RecycleRowsetPB>}} ,
{"RecycleTxnKey", {{"instance_id", "db_id", "txn_id"},
[](param_type& p) { return
recycle_txn_key(KeyInfoSetter<RecycleTxnKeyInfo>{p}.get()); }
, parse<RecycleTxnPB>}} ,
- {"StatsTabletKey", {{"instance_id", "table_id", "index_id",
"part_id", "tablet_id"}, [](param_type& p) { return
stats_tablet_key(KeyInfoSetter<StatsTabletKeyInfo>{p}.get()); }
, parse<TabletStatsPB>}} ,
+ {"StatsTabletKey", {{"instance_id", "table_id", "index_id",
"part_id", "tablet_id"}, [](param_type& p) { return
stats_tablet_key(KeyInfoSetter<StatsTabletKeyInfo>{p}.get()); }
, parse_tablet_stats}} ,
{"JobTabletKey", {{"instance_id", "table_id", "index_id",
"part_id", "tablet_id"}, [](param_type& p) { return
job_tablet_key(KeyInfoSetter<JobTabletKeyInfo>{p}.get()); }
, parse<TabletJobInfoPB>}} ,
{"CopyJobKey", {{"instance_id", "stage_id", "table_id",
"copy_id", "group_id"}, [](param_type& p) { return
copy_job_key(KeyInfoSetter<CopyJobKeyInfo>{p}.get()); }
, parse<CopyJobPB>}} ,
{"CopyFileKey", {{"instance_id", "stage_id", "table_id",
"obj_key", "obj_etag"}, [](param_type& p) { return
copy_file_key(KeyInfoSetter<CopyFileKeyInfo>{p}.get()); }
, parse<CopyFilePB>}} ,
@@ -209,8 +238,27 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const
brpc::URI& uri) {
return http_json_reply(MetaServiceCode::KV_TXN_CREATE_ERR,
fmt::format("failed to create txn, err={}",
err));
}
+
+ std::string_view key_type = http_query(uri, "key_type");
+ auto it = param_set.find(key_type);
+ if (it == param_set.end()) {
+ return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
+ fmt::format("key_type not supported: {}",
+ (key_type.empty() ? "(empty)" :
key_type)));
+ }
+
ValueBuf value;
- err = cloud::get(txn.get(), key, &value, true);
+ if (key_type == "StatsTabletKey") {
+ // FIXME(plat1ko): hard code
+ std::string end_key {key};
+ encode_bytes("\xff", &end_key);
+ std::unique_ptr<RangeGetIterator> it;
+ err = txn->get(key, end_key, &it, true);
+ value.iters.push_back(std::move(it));
+ } else {
+ err = cloud::get(txn.get(), key, &value, true);
+ }
+
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
// FIXME: Key not found err
return http_json_reply(MetaServiceCode::KV_TXN_GET_ERR,
@@ -220,13 +268,6 @@ HttpResponse process_http_get_value(TxnKv* txn_kv, const
brpc::URI& uri) {
return http_json_reply(MetaServiceCode::KV_TXN_GET_ERR,
fmt::format("failed to get kv, key={}",
hex(key)));
}
- std::string_view key_type = http_query(uri, "key_type");
- auto it = param_set.find(key_type);
- if (it == param_set.end()) {
- return http_json_reply(MetaServiceCode::INVALID_ARGUMENT,
- fmt::format("key_type not supported: {}",
- (key_type.empty() ? "(empty)" :
key_type)));
- }
auto readable_value = std::get<2>(it->second)(value);
if (readable_value.empty()) [[unlikely]] {
return http_json_reply(MetaServiceCode::PROTOBUF_PARSE_ERR,
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.cpp
b/cloud/src/meta-service/meta_service_tablet_stats.cpp
index d5ba690d068..868f89e3559 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.cpp
+++ b/cloud/src/meta-service/meta_service_tablet_stats.cpp
@@ -55,61 +55,57 @@ void internal_get_tablet_stats(MetaServiceCode& code,
std::string& msg, Transact
return;
}
// Parse split tablet stats
- do {
- if (!it->has_next()) {
- break;
+ int ret = get_detached_tablet_stats(*it, detached_stats);
+ if (ret != 0) {
+ code = MetaServiceCode::PROTOBUF_PARSE_ERR;
+ msg = fmt::format("marformed splitted tablet stats kv, key={}",
hex(k));
+ return;
+ }
+}
+
+int get_detached_tablet_stats(RangeGetIterator& iter, TabletStats&
detached_stats) {
+ while (iter.has_next()) {
+ auto [k, v] = iter.next();
+ int64_t val;
+ if (v.size() != sizeof(val)) [[unlikely]] {
+ LOG(WARNING) << "malformed tablet stats value. key=" << hex(k);
+ return -1;
}
- while (it->has_next()) {
- auto [k, v] = it->next();
- if (!it->has_next() && it->more()) {
- begin_key = k;
- }
- // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size"
- auto k1 = k;
- k1.remove_prefix(1);
- std::vector<std::tuple<std::variant<int64_t, std::string>, int,
int>> out;
- if (decode_key(&k1, &out) != 0) [[unlikely]] {
- code = MetaServiceCode::UNDEFINED_ERR;
- msg = fmt::format("failed to decode tablet stats key, key={}",
hex(k));
- return;
- }
- if (out.size() != 8) [[unlikely]] {
- code = MetaServiceCode::UNDEFINED_ERR;
- msg = fmt::format("failed to decode tablet stats key, key={}",
hex(k));
- return;
- }
- auto suffix = std::get_if<std::string>(&std::get<0>(out.back()));
- if (!suffix) [[unlikely]] {
- code = MetaServiceCode::UNDEFINED_ERR;
- msg = fmt::format("failed to decode tablet stats key, key={}",
hex(k));
- return;
- }
- int64_t val = *reinterpret_cast<const int64_t*>(v.data());
- if (*suffix == STATS_KEY_SUFFIX_DATA_SIZE) {
- detached_stats.data_size = val;
- } else if (*suffix == STATS_KEY_SUFFIX_NUM_ROWS) {
- detached_stats.num_rows = val;
- } else if (*suffix == STATS_KEY_SUFFIX_NUM_ROWSETS) {
- detached_stats.num_rowsets = val;
- } else if (*suffix == STATS_KEY_SUFFIX_NUM_SEGS) {
- detached_stats.num_segs = val;
- } else {
- VLOG_DEBUG << "unknown suffix=" << *suffix;
- }
+
+ // 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size"
+ k.remove_prefix(1);
+ constexpr size_t key_parts = 8;
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>>
out;
+ if (decode_key(&k, &out) != 0 || out.size() != key_parts) [[unlikely]]
{
+ LOG(WARNING) << "malformed tablet stats key. key=" << hex(k);
+ return -1;
}
- if (it->more()) {
- begin_key.push_back('\x00'); // Update to next smallest key for
iteration
- err = txn->get(begin_key, end_key, &it, snapshot);
- if (err != TxnErrorCode::TXN_OK) {
- code = cast_as<ErrCategory::READ>(err);
- msg = fmt::format("failed to get tablet stats, err={}
tablet_id={}", err,
- idx.tablet_id());
- return;
- }
+
+ auto* suffix = std::get_if<std::string>(&std::get<0>(out.back()));
+ if (!suffix) [[unlikely]] {
+ LOG(WARNING) << "malformed tablet stats key. key=" << hex(k);
+ return -1;
+ }
+
+ std::memcpy(&val, v.data(), sizeof(val));
+ if constexpr (std::endian::native == std::endian::big) {
+ val = bswap_64(val);
+ }
+
+ if (*suffix == STATS_KEY_SUFFIX_DATA_SIZE) {
+ detached_stats.data_size = val;
+ } else if (*suffix == STATS_KEY_SUFFIX_NUM_ROWS) {
+ detached_stats.num_rows = val;
+ } else if (*suffix == STATS_KEY_SUFFIX_NUM_ROWSETS) {
+ detached_stats.num_rowsets = val;
+ } else if (*suffix == STATS_KEY_SUFFIX_NUM_SEGS) {
+ detached_stats.num_segs = val;
} else {
- break;
+ LOG(WARNING) << "unknown suffix=" << *suffix << " key=" << hex(k);
}
- } while (true);
+ }
+
+ return 0;
}
void merge_tablet_stats(TabletStatsPB& stats, const TabletStats&
detached_stats) {
diff --git a/cloud/src/meta-service/meta_service_tablet_stats.h
b/cloud/src/meta-service/meta_service_tablet_stats.h
index f341887e3f1..3e8b4f82a0b 100644
--- a/cloud/src/meta-service/meta_service_tablet_stats.h
+++ b/cloud/src/meta-service/meta_service_tablet_stats.h
@@ -21,6 +21,7 @@
namespace doris::cloud {
class Transaction;
+class RangeGetIterator;
// Detached tablet stats
struct TabletStats {
@@ -46,4 +47,8 @@ void internal_get_tablet_stats(MetaServiceCode& code,
std::string& msg, Transact
const std::string& instance_id, const
TabletIndexPB& idx,
TabletStatsPB& stats, bool snapshot = false);
+// Get detached tablet stats via `iter`, `iter.next` SHOULD be the first
splitted tablet stats KV.
+// Return 0 if success, otherwise error.
+[[nodiscard]] int get_detached_tablet_stats(RangeGetIterator& iter,
TabletStats& detached_stats);
+
} // namespace doris::cloud
diff --git a/cloud/test/http_encode_key_test.cpp
b/cloud/test/http_encode_key_test.cpp
index 25182d54489..d6debec2208 100644
--- a/cloud/test/http_encode_key_test.cpp
+++ b/cloud/test/http_encode_key_test.cpp
@@ -16,15 +16,22 @@
// under the License.
#include <brpc/uri.h>
+#include <gen_cpp/cloud.pb.h>
#include <gtest/gtest.h>
+#include "common/logging.h"
#include "common/sync_point.h"
+#include "meta-service/keys.h"
#include "meta-service/mem_txn_kv.h"
#include "meta-service/meta_service_http.h"
using namespace doris::cloud;
int main(int argc, char** argv) {
+ if (!doris::cloud::init_glog("http_encode_key_test")) {
+ std::cerr << "failed to init glog" << std::endl;
+ return -1;
+ }
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}
@@ -314,7 +321,7 @@ txn_id=126419752960)",
pb.set_num_rowsets(10);
return pb.SerializeAsString();
},
-
R"({"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"num_rowsets":"10"})",
+
R"({"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"0","num_rows":"0","num_rowsets":"10","num_segments":"0"})",
},
Input {
"JobTabletKey",
@@ -545,4 +552,31 @@ TEST(HttpGetValueTest,
process_http_get_value_test_cover_all_template) {
// std::cout << http_res.body << std::endl;
EXPECT_EQ(http_res.body, input.value);
}
+
+ // Test splitted tablet stats KV
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+
+ TabletStatsPB stats;
+ stats.set_cumulative_compaction_cnt(10);
+ stats.set_num_rowsets(100);
+ auto key = stats_tablet_key({"gavin-instance", 10001, 10002, 10003,
10004});
+ txn->put(key, stats.SerializeAsString());
+ std::string num_rowsets_key;
+ stats_tablet_num_rowsets_key({"gavin-instance", 10001, 10002, 10003,
10004}, &num_rowsets_key);
+ txn->atomic_add(num_rowsets_key, 300);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+
+ Input input {
+ .key_type = "StatsTabletKey",
+ .param =
"instance_id=gavin-instance&table_id=10001&index_id=10002&part_id=10003&"
+ "tablet_id="
+ "10004",
+ .value =
+
R"({"data_size":"0","num_rows":"0","num_rowsets":"400","num_segments":"0","cumulative_compaction_cnt":"10"})",
+ };
+ auto url = gen_url(input, true);
+ ASSERT_EQ(uri.SetHttpURL(url), 0); // clear and set query string
+ auto http_res = process_http_get_value(txn_kv.get(), uri);
+ EXPECT_EQ(http_res.status_code, 200);
+ EXPECT_EQ(http_res.body, input.value);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
