This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 56d90714608 branch-4.2: [fix](cloud) Harden metadata key encoding
validation (#68457) (#69233)
56d90714608 is described below
commit 56d9071460828863b137a38aa6fe1124a6d97126
Author: Gavin Chou <[email protected]>
AuthorDate: Mon Oct 19 08:59:26 2026 +0800
branch-4.2: [fix](cloud) Harden metadata key encoding validation (#68457)
(#69233)
## Proposed changes
Backport #68457 to branch-4.2.
- Harden metadata key decoding against malformed tags, extra fields, and
truncated snapshot reference keys.
- Add independent golden coverage and verify HTTP key registry coverage.
- Keep the backport scoped to key types available on branch-4.2; the
Table Stream and TSO fence pieces from master are omitted because those
keyspaces are not present on this branch.
## Validation
- sh run-cloud-ut.sh --run --fdb fdb_cluster0:[email protected]:4500
--filter keys_test:*.*
- sh run-cloud-ut.sh --run --fdb fdb_cluster0:[email protected]:4500
--filter http_encode_key_test:*.*
- sh run-cloud-ut.sh --run --fdb fdb_cluster0:[email protected]:4500
--filter codec_test:*.*
---
cloud/src/meta-service/http_encode_key.cpp | 18 +++
cloud/src/meta-service/meta_service_http.h | 6 +
cloud/src/meta-store/codec.cpp | 2 +-
cloud/src/meta-store/keys.cpp | 34 ++++-
cloud/src/meta-store/keys.h | 44 ++++--
cloud/test/http_encode_key_test.cpp | 76 ++++++++--
cloud/test/keys_test.cpp | 230 ++++++++++++++++++++++-------
7 files changed, 327 insertions(+), 83 deletions(-)
diff --git a/cloud/src/meta-service/http_encode_key.cpp
b/cloud/src/meta-service/http_encode_key.cpp
index cae621cbfa9..19baf88cbf3 100644
--- a/cloud/src/meta-service/http_encode_key.cpp
+++ b/cloud/src/meta-service/http_encode_key.cpp
@@ -308,6 +308,24 @@ static std::unordered_map<std::string_view,
};
// clang-format on
+std::vector<std::string_view> get_supported_http_key_types() {
+ std::vector<std::string_view> key_types;
+ key_types.reserve(param_set.size());
+ for (const auto& entry : param_set) {
+ key_types.push_back(entry.first);
+ }
+ return key_types;
+}
+
+std::vector<std::string_view> get_supported_http_versioned_key_types() {
+ std::vector<std::string_view> key_types;
+ key_types.reserve(versioned_param_set.size());
+ for (const auto& entry : versioned_param_set) {
+ key_types.push_back(entry.first);
+ }
+ return key_types;
+}
+
static MetaServiceResponseStatus encode_key(const brpc::URI& uri, std::string*
key,
std::string* key_type = nullptr) {
MetaServiceResponseStatus status;
diff --git a/cloud/src/meta-service/meta_service_http.h
b/cloud/src/meta-service/meta_service_http.h
index 3c989483923..cb6b3a87e98 100644
--- a/cloud/src/meta-service/meta_service_http.h
+++ b/cloud/src/meta-service/meta_service_http.h
@@ -26,6 +26,7 @@
#include <string>
#include <string_view>
#include <unordered_map>
+#include <vector>
#include "common/util.h"
@@ -48,6 +49,11 @@ HttpResponse process_http_set_value(TxnKv* txn_kv,
brpc::Controller* ctrl);
HttpResponse process_http_encode_key(const brpc::URI& uri);
+// The order is unspecified. These lists let callers verify that tooling
covers every registered
+// key type without duplicating the registry itself.
+std::vector<std::string_view> get_supported_http_key_types();
+std::vector<std::string_view> get_supported_http_versioned_key_types();
+
/// Return the query value or an empty string if not exists.
inline static std::string_view http_query(const brpc::URI& uri, const char*
name) {
return uri.GetQuery(name) ? *uri.GetQuery(name) : std::string_view();
diff --git a/cloud/src/meta-store/codec.cpp b/cloud/src/meta-store/codec.cpp
index 2fbb3671a74..ba07fdceeb9 100644
--- a/cloud/src/meta-store/codec.cpp
+++ b/cloud/src/meta-store/codec.cpp
@@ -51,7 +51,7 @@ void encode_bytes(std::string_view bytes, std::string* b) {
* @return 0 for success
*/
int decode_bytes(std::string_view* in, std::string* out) {
- if (in->at(0) != EncodingTag::BYTES_TAG) return -1;
+ if (in->empty() || in->at(0) != EncodingTag::BYTES_TAG) return -1;
using byte = unsigned char;
in->remove_prefix(1); // Remove bytes marker
while (true) {
diff --git a/cloud/src/meta-store/keys.cpp b/cloud/src/meta-store/keys.cpp
index cffe5154bff..7dc321b6d33 100644
--- a/cloud/src/meta-store/keys.cpp
+++ b/cloud/src/meta-store/keys.cpp
@@ -360,7 +360,7 @@ void packed_file_key(const PackedFileKeyInfo& in,
std::string* out) {
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in,
std::string* out) {
encode_prefix(in, out); // 0x01 "meta"
${instance_id}
encode_bytes(META_KEY_INFIX_DELETE_BITMAP_PENDING, out); //
"delete_bitmap_pending"
- encode_int64(std::get<1>(in), out); // table_id
+ encode_int64(std::get<1>(in), out); // tablet_id
}
void meta_schema_pb_dictionary_key(const MetaSchemaPBDictionaryInfo& in,
std::string* out) {
@@ -609,7 +609,7 @@ std::string version_key_prefix(std::string_view
instance_id) {
std::string index_key_prefix(std::string_view instance_id) {
std::string out;
out.push_back(CLOUD_VERSIONED_KEY_SPACE03);
- encode_bytes(INDEX_INDEX_KEY_INFIX, &out); // "version"
+ encode_bytes(INDEX_INDEX_KEY_INFIX, &out); // "index"
encode_bytes(instance_id, &out); // instance_id
return out;
}
@@ -953,7 +953,7 @@ bool decode_table_version_key(std::string_view* in,
int64_t* table_id, Versionst
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
auto res = decode_key(in, &out, timestamp);
- if (res != 0 || out.size() != 5) {
+ if (res != 0 || out.size() != 5 || std::get<1>(out.back()) !=
EncodingTag::VERSIONSTAMP_TAG) {
return false;
}
@@ -962,6 +962,7 @@ bool decode_table_version_key(std::string_view* in,
int64_t* table_id, Versionst
std::get<std::string>(std::get<0>(out[2])) !=
TABLE_VERSION_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*table_id = std::get<int64_t>(std::get<0>(out[3]));
} catch (const std::bad_variant_access& e) {
return false;
@@ -985,9 +986,11 @@ bool decode_partition_inverted_index_key(std::string_view*
in, int64_t* db_id, i
}
try {
- if (std::get<std::string>(std::get<0>(out[2])) !=
PARTITION_INVERTED_INDEX_KEY_INFIX) {
+ if (std::get<std::string>(std::get<0>(out[0])) != INDEX_KEY_PREFIX ||
+ std::get<std::string>(std::get<0>(out[2])) !=
PARTITION_INVERTED_INDEX_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*db_id = std::get<int64_t>(std::get<0>(out[3]));
*table_id = std::get<int64_t>(std::get<0>(out[4]));
*partition_id = std::get<int64_t>(std::get<0>(out[5]));
@@ -1008,7 +1011,7 @@ bool decode_meta_partition_key(std::string_view* in,
int64_t* partition_id,
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
auto res = decode_key(in, &out, timestamp);
- if (res != 0 || out.size() != 5) {
+ if (res != 0 || out.size() != 5 || std::get<1>(out.back()) !=
EncodingTag::VERSIONSTAMP_TAG) {
return false;
}
@@ -1017,6 +1020,7 @@ bool decode_meta_partition_key(std::string_view* in,
int64_t* partition_id,
std::get<std::string>(std::get<0>(out[2])) !=
META_PARTITION_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*partition_id = std::get<int64_t>(std::get<0>(out[3]));
} catch (const std::bad_variant_access& e) {
return false;
@@ -1034,7 +1038,7 @@ bool decode_meta_index_key(std::string_view* in, int64_t*
index_id, Versionstamp
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
auto res = decode_key(in, &out, timestamp);
- if (res != 0 || out.size() != 5) {
+ if (res != 0 || out.size() != 5 || std::get<1>(out.back()) !=
EncodingTag::VERSIONSTAMP_TAG) {
return false;
}
@@ -1043,6 +1047,7 @@ bool decode_meta_index_key(std::string_view* in, int64_t*
index_id, Versionstamp
std::get<std::string>(std::get<0>(out[2])) !=
META_INDEX_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*index_id = std::get<int64_t>(std::get<0>(out[3]));
} catch (const std::bad_variant_access& e) {
return false;
@@ -1069,6 +1074,7 @@ bool decode_meta_schema_key(std::string_view* in,
int64_t* index_id, int64_t* sc
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_SCHEMA) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*index_id = std::get<int64_t>(std::get<0>(out[3]));
*schema_version = std::get<int64_t>(std::get<0>(out[4]));
} catch (const std::bad_variant_access& e) {
@@ -1087,7 +1093,7 @@ bool decode_meta_tablet_key(std::string_view* in,
int64_t* tablet_id, Versionsta
std::vector<std::tuple<std::variant<int64_t, std::string>, int, int>> out;
auto res = decode_key(in, &out, timestamp);
- if (res != 0 || out.size() < 5) {
+ if (res != 0 || out.size() != 5 || std::get<1>(out.back()) !=
EncodingTag::VERSIONSTAMP_TAG) {
return false;
}
@@ -1096,6 +1102,7 @@ bool decode_meta_tablet_key(std::string_view* in,
int64_t* tablet_id, Versionsta
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_TABLET) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*tablet_id = std::get<int64_t>(std::get<0>(out[3]));
} catch (const std::bad_variant_access& e) {
return false;
@@ -1126,6 +1133,7 @@ bool decode_tablet_inverted_index_key(std::string_view*
in, int64_t* db_id, int6
std::get<std::string>(std::get<0>(out[2])) !=
TABLET_INVERTED_INDEX_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*db_id = std::get<int64_t>(std::get<0>(out[3]));
*table_id = std::get<int64_t>(std::get<0>(out[4]));
*index_id = std::get<int64_t>(std::get<0>(out[5]));
@@ -1182,7 +1190,9 @@ bool decode_snapshot_ref_key(std::string_view* in,
std::string* instance_id,
}
// Decode ref_instance_id
- if (ref_instance_id && decode_bytes(in, ref_instance_id) != 0) {
+ std::string ignored_ref_instance_id;
+ if (decode_bytes(in, ref_instance_id != nullptr ? ref_instance_id :
&ignored_ref_instance_id) !=
+ 0) {
return false;
}
@@ -1208,6 +1218,7 @@ bool decode_data_rowset_ref_count_key(std::string_view*
in, int64_t* tablet_id,
std::get<std::string>(std::get<0>(out[2])) !=
META_ROWSET_REF_COUNT_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*tablet_id = std::get<int64_t>(std::get<0>(out[3]));
*rowset_id = std::get<std::string>(std::get<0>(out[4]));
} catch (const std::bad_variant_access& e) {
@@ -1238,6 +1249,7 @@ bool decode_stats_tablet_key(std::string_view* in,
int64_t* table_id, int64_t* i
std::get<std::string>(std::get<0>(out[2])) !=
STATS_KEY_INFIX_TABLET) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*table_id = std::get<int64_t>(std::get<0>(out[3]));
*index_id = std::get<int64_t>(std::get<0>(out[4]));
*partition_id = std::get<int64_t>(std::get<0>(out[5]));
@@ -1268,6 +1280,7 @@ bool decode_table_version_key(std::string_view* in,
int64_t* db_id, int64_t* tbl
std::get<std::string>(std::get<0>(out[2])) !=
TABLE_VERSION_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*db_id = std::get<int64_t>(std::get<0>(out[3]));
*tbl_id = std::get<int64_t>(std::get<0>(out[4]));
} catch (const std::bad_variant_access& e) {
@@ -1296,6 +1309,7 @@ bool decode_tablet_schema_key(std::string_view* in,
int64_t* index_id, int64_t*
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_SCHEMA) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*index_id = std::get<int64_t>(std::get<0>(out[3]));
*schema_version = std::get<int64_t>(std::get<0>(out[4]));
} catch (const std::bad_variant_access& e) {
@@ -1325,6 +1339,7 @@ bool decode_partition_version_key(std::string_view* in,
int64_t* db_id, int64_t*
std::get<std::string>(std::get<0>(out[2])) !=
PARTITION_VERSION_KEY_INFIX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*db_id = std::get<int64_t>(std::get<0>(out[3]));
*tbl_id = std::get<int64_t>(std::get<0>(out[4]));
*partition_id = std::get<int64_t>(std::get<0>(out[5]));
@@ -1355,6 +1370,7 @@ bool decode_meta_tablet_key(std::string_view* in,
int64_t* table_id, int64_t* in
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_TABLET) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*table_id = std::get<int64_t>(std::get<0>(out[3]));
*index_id = std::get<int64_t>(std::get<0>(out[4]));
*partition_id = std::get<int64_t>(std::get<0>(out[5]));
@@ -1385,6 +1401,7 @@ bool decode_meta_rowset_key(std::string_view* in,
int64_t* tablet_id, int64_t* v
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_ROWSET) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*tablet_id = std::get<int64_t>(std::get<0>(out[3]));
*version = std::get<int64_t>(std::get<0>(out[4]));
} catch (const std::bad_variant_access& e) {
@@ -1413,6 +1430,7 @@ bool decode_meta_tablet_idx_key(std::string_view* in,
int64_t* tablet_id) {
std::get<std::string>(std::get<0>(out[2])) !=
META_KEY_INFIX_TABLET_IDX) {
return false;
}
+ (void)std::get<std::string>(std::get<0>(out[1]));
*tablet_id = std::get<int64_t>(std::get<0>(out[3]));
} catch (const std::bad_variant_access& e) {
return false;
diff --git a/cloud/src/meta-store/keys.h b/cloud/src/meta-store/keys.h
index be05418979a..f3f1c356379 100644
--- a/cloud/src/meta-store/keys.h
+++ b/cloud/src/meta-store/keys.h
@@ -20,6 +20,8 @@
#include <cstdint>
#include <string>
#include <tuple>
+#include <type_traits>
+#include <utility>
#include <variant>
#include <vector>
@@ -32,7 +34,7 @@
//
// 0x01 "txn" ${instance_id} "txn_label" ${db_id} ${label}
-> TxnLabelPB ${version_timestamp}
// 0x01 "txn" ${instance_id} "txn_info" ${db_id} ${txn_id}
-> TxnInfoPB
-// 0x01 "txn" ${instance_id} "txn_db_tbl" ${txn_id}
-> TxnIndexPB
+// 0x01 "txn" ${instance_id} "txn_index" ${txn_id}
-> TxnIndexPB
// 0x01 "txn" ${instance_id} "txn_running" ${db_id} ${txn_id}
-> TxnRunningPB
//
// 0x01 "version" ${instance_id} "partition" ${db_id} ${tbl_id}
${partition_id} -> VersionPB
@@ -44,11 +46,11 @@
// 0x01 "meta" ${instance_id} "tablet_index" ${tablet_id}
-> TabletIndexPB
// 0x01 "meta" ${instance_id} "schema" ${index_id} ${schema_version}
-> TabletSchemaCloudPB
// 0x01 "meta" ${instance_id} "delete_bitmap_lock" ${table_id} ${partition_id}
-> DeleteBitmapUpdateLockPB
-// 0x01 "meta" ${instance_id} "delete_bitmap_pending" ${table_id}
-> PendingDeleteBitmapPB
+// 0x01 "meta" ${instance_id} "delete_bitmap_pending" ${tablet_id}
-> PendingDeleteBitmapPB
// 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id}
${version} ${segment_id} -> roaringbitmap
// 0x01 "meta" ${instance_id} "tablet_schema_pb_dict" ${index_id}
-> SchemaCloudDictionary
-// 0x01 "meta" ${instance_id} "mow_tablet_job" ${table_id} ${initiator_id}
-> MowTabletJobPB
-// 0x01 "meta" ${instance_id} "packed_file" ${packed_file_path}
-> PackedFileInfoPB
+// 0x01 "meta" ${instance_id} "mow_tablet_job" ${table_id} ${initiator_id}
-> MowTabletJobPB
+// 0x01 "meta" ${instance_id} "packed_file" ${packed_file_path}
-> PackedFileInfoPB
//
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} -> TabletStatsPB
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size" -> int64
@@ -69,12 +71,13 @@
// 0x01 "job" ${instance_id} "check"
-> JobRecyclePB
// 0x01 "job" ${instance_id} "snapshot_data_migrator"
-> JobRecyclePB
// 0x01 "job" ${instance_id} "snapshot_chain_compactor"
-> JobRecyclePB
+// 0x01 "job" ${instance_id} "routine_load_progress" ${db_id} ${job_id}
-> RoutineLoadProgressPB
// 0x01 "job" ${instance_id} "streaming_job" ${db_id} ${job_id}
-> StreamingJobPB
//
// 0x01 "copy" ${instance_id} "job" ${stage_id} ${table_id} ${copy_id}
${group_id} -> CopyJobPB
// 0x01 "copy" ${instance_id} "loading_file" ${stage_id} ${table_id}
${obj_name} ${etag} -> CopyFilePB
//
-// 0x01 "storage_vault" ${instance_id} "vault" ${resource_id}
-> StorageVaultPB
+// 0x01 "storage_vault" ${instance_id} "vault" ${resource_id/vault_id}
-> StorageVaultPB
//
// 0x01 "job" ${instance_id} "restore_tablet" ${tablet_id}
-> RestoreJobCloudPB
// 0x01 "job" ${instance_id} "restore_rowset" ${tablet_id} ${version}
-> RowsetMetaCloudPB
@@ -131,17 +134,40 @@ static constexpr std::string_view
STATS_KEY_SUFFIX_SEGMENT_SIZE = "segment_size"
// clang-format off
/**
- * Wraps std::tuple for differnet types even if the underlying type is the
same.
+ * Wraps std::tuple to create different types even if the underlying type is
the same.
*
- * @param N for elemination of same underlying types of type alias when we use
- * `using` to declare a new type.
+ * @param N distinguishes type aliases that have the same underlying tuple
type.
*
* @param Base for base tuple, the underlying type
*/
template<size_t N, typename Base>
+struct BasicKeyInfo;
+
+template<typename T>
+struct IsBasicKeyInfo : std::false_type {};
+
+template<size_t N, typename Base>
+struct IsBasicKeyInfo<BasicKeyInfo<N, Base>> : std::true_type {};
+
+template<typename T>
+inline constexpr bool is_basic_key_info_v =
+ IsBasicKeyInfo<std::remove_cv_t<std::remove_reference_t<T>>>::value;
+
+template<size_t N, typename Base>
struct BasicKeyInfo : Base {
- template<typename... Args>
+ template<typename... Args,
+ std::enable_if_t<(!is_basic_key_info_v<Args> && ...), int> = 0>
BasicKeyInfo(Args&&... args) : Base(std::forward<Args>(args)...) {}
+
+ // KeyInfo wrappers with different semantics may share the same tuple
Base. Require
+ // cross-wrapper conversions to be explicit, and keep the forwarding
constructor from
+ // silently providing another conversion path.
+ template<size_t M>
+ explicit BasicKeyInfo(const BasicKeyInfo<M, Base>& other) : Base(other) {}
+
+ template<size_t M>
+ explicit BasicKeyInfo(BasicKeyInfo<M, Base>&& other) :
Base(std::move(other)) {}
+
constexpr static size_t n = N;
using base_type = Base;
};
diff --git a/cloud/test/http_encode_key_test.cpp
b/cloud/test/http_encode_key_test.cpp
index 1d29076313d..fff10c3a2c3 100644
--- a/cloud/test/http_encode_key_test.cpp
+++ b/cloud/test/http_encode_key_test.cpp
@@ -19,6 +19,8 @@
#include <gen_cpp/cloud.pb.h>
#include <gtest/gtest.h>
+#include <set>
+
#include "common/defer.h"
#include "common/logging.h"
#include "cpp/sync_point.h"
@@ -331,6 +333,8 @@ txn_id=126419752960)",
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f726f77730001",
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f726f77736574730001",
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9106e756d5f736567730001",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d910696e6465785f73697a650001",
+
"01107374617473000110676176696e2d696e7374616e63650001107461626c6574000112000000000000276612000000000000271a1200000000000027101200000000000f63d9107365676d656e745f73697a650001",
},
[]() -> std::vector<std::string> {
TabletStatsPB pb;
@@ -345,11 +349,11 @@ txn_id=126419752960)",
pb.set_num_segments(12);
pb.set_index_size(1);
pb.set_segment_size(1);
- return {pb.SerializeAsString(),
{"\x01\x00\x00\x00\x00\x00\x00\x00",8}, {"\x02\x00\x00\x00\x00\x00\x00\x00",8},
{"\x03\x00\x00\x00\x00\x00\x00\x00",8}, {"\x04\x00\x00\x00\x00\x00\x00\x00",8}};
+ return {pb.SerializeAsString(),
{"\x01\x00\x00\x00\x00\x00\x00\x00",8}, {"\x02\x00\x00\x00\x00\x00\x00\x00",8},
{"\x03\x00\x00\x00\x00\x00\x00\x00",8}, {"\x04\x00\x00\x00\x00\x00\x00\x00",8},
{"\x05\x00\x00\x00\x00\x00\x00\x00",8}, {"\x06\x00\x00\x00\x00\x00\x00\x00",8}};
},
R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
-detached_stats:
{"data_size":"1","num_rows":"2","num_rowsets":"3","num_segments":"4","index_size":"0","segment_size":"0"}
-merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"3","num_rows":"12","num_rowsets":"14","num_segments":"16","index_size":"1","segment_size":"1"}
+detached_stats:
{"data_size":"1","num_rows":"2","num_rowsets":"3","num_segments":"4","index_size":"5","segment_size":"6"}
+merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008601"},"data_size":"3","num_rows":"12","num_rowsets":"14","num_segments":"16","index_size":"6","segment_size":"7"}
)",
},
Input { // aggregated_stats + half detached_stats (num_segs == 0, there is
num_rowsets detached stats)
@@ -391,7 +395,7 @@ merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"100
idx->set_table_id(10086);
idx->set_index_id(100010);
idx->set_partition_id(10000);
- idx->set_tablet_id(1008602);
+ idx->set_tablet_id(1008603);
pb.set_data_size(2);
pb.set_num_rows(10);
pb.set_num_rowsets(11);
@@ -400,9 +404,9 @@ merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"100
pb.set_segment_size(1);
return {pb.SerializeAsString()};
},
- R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
+ R"(aggregated_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008603"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
detached_stats:
{"data_size":"0","num_rows":"0","num_rowsets":"0","num_segments":"0","index_size":"0","segment_size":"0"}
-merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008602"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
+merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"10000","tablet_id":"1008603"},"data_size":"2","num_rows":"10","num_rowsets":"11","num_segments":"12","index_size":"1","segment_size":"1"}
)",
},
Input {
@@ -541,6 +545,30 @@ merged_stats:
{"idx":{"table_id":"10086","index_id":"100010","partition_id":"100
},
R"({"partition_to_offset":{"1000":"1234"}})",
},
+ Input {
+ "StorageVaultKey",
+ "instance_id=gavin-instance&vault_id=vault-id",
+
{"011073746f726167655f7661756c74000110676176696e2d696e7374616e63650001107661756c740001107661756c742d69640001"},
+ []() -> std::vector<std::string> {
+ StorageVaultPB pb;
+ pb.set_id("vault-id");
+ pb.set_name("vault-name");
+ return {pb.SerializeAsString()};
+ },
+ R"({"id":"vault-id","name":"vault-name"})",
+ },
+ Input {
+ "MetaSchemaPBDictionaryKey",
+ "instance_id=gavin-instance&index_id=10086",
+
{"01106d657461000110676176696e2d696e7374616e63650001107461626c65745f736368656d615f70625f646963740001120000000000002766"},
+ []() -> std::vector<std::string> {
+ doris::SchemaCloudDictionary pb;
+ pb.set_current_column_dict_id(1);
+ pb.set_current_index_dict_id(2);
+ return {pb.SerializeAsString()};
+ },
+ R"({"current_column_dict_id":"1","current_index_dict_id":"2"})",
+ },
Input {
"MetaServiceRegistryKey",
"",
@@ -596,7 +624,8 @@ TEST(HttpEncodeKeyTest,
process_http_encode_key_test_cover_all_template) {
EXPECT_EQ(uri.SetHttpURL(url.str()), 0); // clear and set query string
auto http_res = process_http_encode_key(uri);
EXPECT_EQ(http_res.status_code, 200);
- EXPECT_NE(http_res.body.find(input.key[0]), std::string::npos)
+ const std::string encoded_key_line = "\n" + input.key[0] + "\n";
+ EXPECT_NE(http_res.body.find(encoded_key_line), std::string::npos)
<< "real full text: " << http_res.body << "\nexpect contains:
" << input.key[0];
}
}
@@ -871,6 +900,27 @@ static auto versioned_test_inputs = std::array {
};
// clang-format on
+// Keep the HTTP key registries and their golden fixtures in one-to-one
correspondence.
+TEST(HttpEncodeKeyTest, RegisteredKeyTypesMatchGoldenFixtures) {
+ std::set<std::string_view> fixture_key_types;
+ for (const auto& input : test_inputs) {
+ fixture_key_types.insert(input.key_type);
+ }
+ const auto supported_key_types = get_supported_http_key_types();
+ const std::set<std::string_view>
registered_key_types(supported_key_types.begin(),
+
supported_key_types.end());
+ EXPECT_EQ(fixture_key_types, registered_key_types);
+
+ std::set<std::string_view> versioned_fixture_key_types;
+ for (const auto& input : versioned_test_inputs) {
+ versioned_fixture_key_types.insert(input.key_type);
+ }
+ const auto supported_versioned_key_types =
get_supported_http_versioned_key_types();
+ const std::set<std::string_view> registered_versioned_key_types(
+ supported_versioned_key_types.begin(),
supported_versioned_key_types.end());
+ EXPECT_EQ(versioned_fixture_key_types, registered_versioned_key_types);
+}
+
TEST(HttpEncodeKeyTest, process_http_encode_versioned_key_test) {
static auto format_fdb_key = [](const std::string& s) {
std::stringstream r;
@@ -891,7 +941,8 @@ TEST(HttpEncodeKeyTest,
process_http_encode_versioned_key_test) {
EXPECT_EQ(uri.SetHttpURL(url.str()), 0); // clear and set query string
auto http_res = process_http_encode_key(uri);
EXPECT_EQ(http_res.status_code, 200) << "Failed for key_type: " <<
input.key_type;
- EXPECT_NE(http_res.body.find(input.key[0]), std::string::npos)
+ const std::string encoded_key_line = "\n" + input.key[0] + "\n";
+ EXPECT_NE(http_res.body.find(encoded_key_line), std::string::npos)
<< "real full text: " << http_res.body << "\nexpect contains:
" << input.key[0]
<< "\n"
<< format_fdb_key(input.key[0]) << "\nkey_type: " <<
input.key_type
@@ -912,10 +963,11 @@ TEST(HttpEncodeKeyTest,
process_http_encode_versioned_key_with_versionstamp) {
auto http_res = process_http_encode_key(uri);
EXPECT_EQ(http_res.status_code, 200);
- // The encoded key should contain both the base key and the versionstamp
-
EXPECT_NE(http_res.body.find("031076657273696f6e000110676176696e2d696e7374616e63650001107061727"
- "46974696f6e0001120000000000002766"),
- std::string::npos);
+ const std::string encoded_key_line =
+
"\n031076657273696f6e000110676176696e2d696e7374616e6365000110706172746974696f6e"
+ "00011200000000000027661300000000000000010001ff\n";
+ EXPECT_NE(http_res.body.find(encoded_key_line), std::string::npos)
+ << "real full text: " << http_res.body << "\nexpect contains: " <<
encoded_key_line;
}
TEST(HttpGetValueTest, process_http_get_value_versioned_key_test) {
diff --git a/cloud/test/keys_test.cpp b/cloud/test/keys_test.cpp
index c6bd381ce14..a2baa2ccef3 100644
--- a/cloud/test/keys_test.cpp
+++ b/cloud/test/keys_test.cpp
@@ -24,6 +24,9 @@
#include <cstdint>
#include <cstring>
#include <iostream>
+#include <limits>
+#include <type_traits>
+#include <utility>
#include <vector>
#include "common/util.h"
@@ -58,31 +61,6 @@ void encode_bytes(std::string_view bytes, std::string* b);
int decode_bytes(std::string_view* in, std::string* out);
} // namespace doris::cloud
-// clang-format off
-// Possible key encoding schemas:
-//
-// 0x01 "instance" ${instance_id} -> InstanceInfoPB
-//
-// 0x01 "txn" ${instance_id} "txn_label" ${db_id} ${label} -> TxnLabelPB
${version_timestamp}
-// 0x01 "txn" ${instance_id} "txn_info" ${db_id} ${version_timestamp} ->
TxnInfoPB
-// 0x01 "txn" ${instance_id} "txn_index" ${version_timestamp} -> TxnIndexPB
-// 0x01 "txn" ${instance_id} "txn_running" ${db_id} ${version_timestamp} ->
TxnRunningPB // creaet at begin, delete at commit
-//
-// 0x01 "version" ${instance_id} "partition" ${db_id} ${tbl_id}
${partition_id} -> VersionPB
-//
-// 0x01 "meta" ${instance_id} "rowset" ${tablet_id} ${version} ${rowset_id} ->
RowsetMetaCloudPB
-// 0x01 "meta" ${instance_id} "rowset_tmp" ${txn_id} ${rowset_id} ->
RowsetMetaCloudPB
-// 0x01 "meta" ${instance_id} "tablet" ${table_id} ${tablet_id} ->
TabletMetaCloudPB
-// 0x01 "meta" ${instance_id} "tablet_table" ${tablet_id} -> ${table_id}
-// 0x01 "meta" ${instance_id} "tablet_tmp" ${table_id} ${tablet_id} ->
TabletMetaCloudPB
-//
-// 0x01 "trash" ${instacne_id} "table" -> TableTrashPB
-//
-// 0x01 "node_status" ${instance_id} "compute" ${backend_id} ->
ComputeNodeStatusPB
-//
-// 0x01 "job" ${instance_id} "streaming_job" ${db_id} ${job_id} ->
StreamingJobPB
-// clang-format on
-
TEST(KeysTest, InstanceKeyTest) {
using namespace doris::cloud;
@@ -106,6 +84,34 @@ TEST(KeysTest, InstanceKeyTest) {
EXPECT_EQ(instance_id, dec_instance_id);
}
+TEST(KeysTest, FixedEncodingGoldenTest) {
+ using namespace doris::cloud;
+
+ static_assert(!std::is_convertible_v<MetaRowsetKeyInfo,
MetaRowsetTmpKeyInfo>);
+ static_assert(std::is_constructible_v<MetaRowsetTmpKeyInfo,
MetaRowsetKeyInfo>);
+
+ EXPECT_EQ(hex(mow_tablet_job_key({"gavin-instance", 10086, 10010})),
+
"01106d657461000110676176696e2d696e7374616e63650001106d6f775f7461626c65745f"
+ "6a6f62000112000000000000276612000000000000271a");
+ EXPECT_EQ(hex(storage_vault_key({"gavin-instance", "vault-id"})),
+
"011073746f726167655f7661756c74000110676176696e2d696e7374616e6365000110766175"
+ "6c740001107661756c742d69640001");
+ EXPECT_EQ(hex(meta_schema_pb_dictionary_key({"gavin-instance", 10086})),
+
"01106d657461000110676176696e2d696e7374616e63650001107461626c65745f736368656d"
+ "615f70625f646963740001120000000000002766");
+
+ const std::string all_partitions_lock =
+ meta_delete_bitmap_update_lock_key({"gavin-instance", 10086, -1});
+ EXPECT_EQ(hex(all_partitions_lock),
+
"01106d657461000110676176696e2d696e7374616e636500011064656c6574655f6269746d61"
+ "705f6c6f636b0001120000000000002766110000000000000001");
+ EXPECT_LT(all_partitions_lock,
+ meta_delete_bitmap_update_lock_key({"gavin-instance", 10086,
0}));
+ EXPECT_LT(meta_delete_bitmap_update_lock_key({"gavin-instance", 10086, 0}),
+ meta_delete_bitmap_update_lock_key(
+ {"gavin-instance", 10086,
std::numeric_limits<int64_t>::max()}));
+}
+
TEST(KeysTest, MetaKeysTest) {
using namespace doris::cloud;
std::string instance_id = "instance_id_deadbeef";
@@ -150,18 +156,18 @@ TEST(KeysTest, MetaKeysTest) {
}
// tmp rowset meta key
- // 0x01 "meta" ${instance_id} "rowset_tmp" ${tablet_id} ${version}
+ // 0x01 "meta" ${instance_id} "rowset_tmp" ${txn_id} ${tablet_id}
{
- int64_t tablet_id = 10086;
- int64_t version = 100;
- MetaRowsetKeyInfo rowset_key {instance_id, tablet_id, version};
+ int64_t txn_id = 10086;
+ int64_t tablet_id = 100;
+ MetaRowsetTmpKeyInfo rowset_key {instance_id, txn_id, tablet_id};
std::string encoded_rowset_key0;
meta_rowset_tmp_key(rowset_key, &encoded_rowset_key0);
std::cout << hex(encoded_rowset_key0) << std::endl;
std::string dec_instance_id;
+ int64_t dec_txn_id = 0;
int64_t dec_tablet_id = 0;
- int64_t dec_version = 0;
std::string_view key_sv(encoded_rowset_key0);
std::string dec_meta_prefix;
@@ -170,17 +176,17 @@ TEST(KeysTest, MetaKeysTest) {
ASSERT_EQ(decode_bytes(&key_sv, &dec_meta_prefix), 0);
ASSERT_EQ(decode_bytes(&key_sv, &dec_instance_id), 0);
ASSERT_EQ(decode_bytes(&key_sv, &dec_rowset_prefix), 0);
- ASSERT_EQ(decode_int64(&key_sv, &dec_tablet_id), 0) << hex(key_sv);
- ASSERT_EQ(decode_int64(&key_sv, &dec_version), 0);
+ ASSERT_EQ(decode_int64(&key_sv, &dec_txn_id), 0) << hex(key_sv);
+ ASSERT_EQ(decode_int64(&key_sv, &dec_tablet_id), 0);
ASSERT_TRUE(key_sv.empty());
EXPECT_EQ("meta", dec_meta_prefix);
EXPECT_EQ("rowset_tmp", dec_rowset_prefix);
EXPECT_EQ(instance_id, dec_instance_id);
+ EXPECT_EQ(txn_id, dec_txn_id);
EXPECT_EQ(tablet_id, dec_tablet_id);
- EXPECT_EQ(version, dec_version);
- std::get<2>(rowset_key) = version + 1;
+ std::get<2>(rowset_key) = tablet_id + 1;
std::string encoded_rowset_key1;
meta_rowset_tmp_key(rowset_key, &encoded_rowset_key1);
std::cout << hex(encoded_rowset_key1) << std::endl;
@@ -227,7 +233,7 @@ TEST(KeysTest, MetaKeysTest) {
EXPECT_EQ(partition_id, dec_partition_id);
EXPECT_EQ(tablet_id, dec_tablet_id);
- std::get<2>(tablet_key) = tablet_id + 1;
+ std::get<4>(tablet_key) = tablet_id + 1;
std::string encoded_rowset_key1;
meta_tablet_key(tablet_key, &encoded_rowset_key1);
std::cout << hex(encoded_rowset_key1) << std::endl;
@@ -429,7 +435,7 @@ TEST(KeysTest, TxnKeysTest) {
ASSERT_GT(encoded_txn_index_key1, encoded_txn_index_key0);
}
- // 0x01 "txn" ${instance_id} "txn_info" ${db_id} ${version_timestamp} ->
TxnInfoPB
+ // 0x01 "txn" ${instance_id} "txn_info" ${db_id} ${txn_id} -> TxnInfoPB
{
int64_t db_id = 12345678;
int64_t txn_id = 10086;
@@ -474,7 +480,7 @@ TEST(KeysTest, TxnKeysTest) {
ASSERT_GT(encoded_txn_info_key2, encoded_txn_info_key0);
}
- // 0x01 "txn" ${instance_id} "txn_index" ${version_timestamp} -> TxnIndexPB
+ // 0x01 "txn" ${instance_id} "txn_index" ${txn_id} -> TxnIndexPB
{
int64_t txn_id = 12343212453;
TxnIndexKeyInfo txn_index_key_ {instance_id, txn_id};
@@ -508,7 +514,7 @@ TEST(KeysTest, TxnKeysTest) {
ASSERT_GT(encoded_txn_index_key1, encoded_txn_index_key0);
}
- // 0x01 "txn" ${instance_id} "txn_running" ${db_id} ${version_timestamp}
-> ${table_id_list}
+ // 0x01 "txn" ${instance_id} "txn_running" ${db_id} ${txn_id} ->
${table_id_list}
{
int64_t db_id = 98712345;
int64_t txn_id = 12343212453;
@@ -1064,7 +1070,7 @@ TEST(KeysTest, CopyKeysTest) {
EXPECT_EQ(group_id, dec_group_id);
}
- // 0x01 "copy" ${instance_id} "loading_files" ${stage_id} ${table_id}
${obj_name} ${etag} -> CopyFilePB
+ // 0x01 "copy" ${instance_id} "loading_file" ${stage_id} ${table_id}
${obj_name} ${etag} -> CopyFilePB
{
std::string stage_id = "9482049283";
int64_t table_id = 3745823784;
@@ -1104,20 +1110,6 @@ TEST(KeysTest, CopyKeysTest) {
}
}
-TEST(KeysTest, DecodeKeysTest) {
- using namespace doris::cloud;
- // clang-format off
- std::string key =
"011074786e000110696e7374616e63655f69645f646561646265656600011074786e5f696e646578000112000000000000271310696e736572745f336664356164313264303035346139622d386337373664333231386336616462370001";
- // clang-format on
- auto pretty_key = prettify_key(key);
- ASSERT_TRUE(!pretty_key.empty()) << key;
- std::cout << "\n" << pretty_key << std::endl;
-
- pretty_key = prettify_key(key, true);
- ASSERT_TRUE(!pretty_key.empty()) << key;
- std::cout << "\n" << pretty_key << std::endl;
-}
-
TEST(KeysTest, MetaSchemaPBDictionaryTest) {
using namespace doris::cloud;
std::string instance_id = "instance_id_meta_dict";
@@ -2408,6 +2400,23 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
SnapshotReferenceKeyInfo key_info {instance_id, timestamp,
ref_instance_id};
std::string encoded_key = snapshot_reference_key(key_info);
+ std::string prefix = snapshot_reference_key_prefix(instance_id);
+ EXPECT_TRUE(encoded_key.starts_with(prefix));
+ {
+ std::string_view prefix_view = prefix;
+ remove_versioned_space_prefix(&prefix_view);
+ std::string decoded_prefix;
+ std::string decoded_instance_id;
+ std::string decoded_infix;
+ ASSERT_EQ(decode_bytes(&prefix_view, &decoded_prefix), 0);
+ ASSERT_EQ(decode_bytes(&prefix_view, &decoded_instance_id), 0);
+ ASSERT_EQ(decode_bytes(&prefix_view, &decoded_infix), 0);
+ EXPECT_TRUE(prefix_view.empty());
+ EXPECT_EQ(decoded_prefix, "snapshot");
+ EXPECT_EQ(decoded_instance_id, instance_id);
+ EXPECT_EQ(decoded_infix, "reference");
+ }
+
// Test decode_snapshot_ref_key - decode all fields
{
std::string dec_instance_id;
@@ -2418,6 +2427,7 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
bool ret = decode_snapshot_ref_key(&key_view, &dec_instance_id,
&dec_timestamp,
&dec_ref_instance_id);
ASSERT_TRUE(ret);
+ EXPECT_TRUE(key_view.empty());
EXPECT_EQ(dec_instance_id, instance_id);
EXPECT_EQ(dec_timestamp, timestamp);
EXPECT_EQ(dec_ref_instance_id, ref_instance_id);
@@ -2429,6 +2439,7 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
std::string_view key_view = encoded_key;
bool ret = decode_snapshot_ref_key(&key_view, nullptr, nullptr,
&dec_ref_instance_id);
ASSERT_TRUE(ret);
+ EXPECT_TRUE(key_view.empty());
EXPECT_EQ(dec_ref_instance_id, ref_instance_id);
}
@@ -2439,6 +2450,7 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
std::string_view key_view = encoded_key;
bool ret = decode_snapshot_ref_key(&key_view, &dec_instance_id,
&dec_timestamp, nullptr);
ASSERT_TRUE(ret);
+ EXPECT_TRUE(key_view.empty());
EXPECT_EQ(dec_instance_id, instance_id);
EXPECT_EQ(dec_timestamp, timestamp);
}
@@ -2460,6 +2472,31 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
ASSERT_FALSE(ret);
}
+ // Every strict prefix of a valid key must be rejected without throwing.
+ for (size_t size = 0; size < encoded_key.size(); ++size) {
+ SCOPED_TRACE(size);
+ std::string_view key_view(encoded_key.data(), size);
+ EXPECT_FALSE(decode_snapshot_ref_key(&key_view, nullptr, nullptr,
nullptr));
+ }
+
+ // A successfully decoded key may retain a caller-owned suffix.
+ {
+ std::string key_with_suffix = encoded_key;
+ key_with_suffix.push_back('\0');
+ std::string_view key_view = key_with_suffix;
+ EXPECT_TRUE(decode_snapshot_ref_key(&key_view, nullptr, nullptr,
nullptr));
+ EXPECT_EQ(key_view, std::string_view("\0", 1));
+ }
+
+ // The timestamp field must carry a versionstamp tag, not merely another
decodable field.
+ {
+ std::string invalid_key = prefix;
+ encode_int64(1, &invalid_key);
+ encode_bytes(ref_instance_id, &invalid_key);
+ std::string_view key_view = invalid_key;
+ EXPECT_FALSE(decode_snapshot_ref_key(&key_view, nullptr, nullptr,
nullptr));
+ }
+
// Test with multiple ref_instance_ids to ensure uniqueness
{
std::string ref_id1 = "ref_instance_1";
@@ -2481,6 +2518,9 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
ASSERT_TRUE(decode_snapshot_ref_key(&key_view1, nullptr, nullptr,
&dec_ref1));
ASSERT_TRUE(decode_snapshot_ref_key(&key_view2, nullptr, nullptr,
&dec_ref2));
ASSERT_TRUE(decode_snapshot_ref_key(&key_view3, nullptr, nullptr,
&dec_ref3));
+ EXPECT_TRUE(key_view1.empty());
+ EXPECT_TRUE(key_view2.empty());
+ EXPECT_TRUE(key_view3.empty());
EXPECT_EQ(dec_ref1, ref_id1);
EXPECT_EQ(dec_ref2, ref_id2);
@@ -2488,6 +2528,90 @@ TEST(KeysTest, DecodeSnapshotRefKeyTest) {
}
}
+TEST(KeysTest, VersionedDecodersRequireExactSchema) {
+ using namespace doris::cloud;
+
+ const Versionstamp timestamp(123, 4);
+ auto append_timestamp = [&](std::string key) {
+ encode_versionstamp(timestamp, &key);
+ encode_versionstamp_end(&key);
+ return key;
+ };
+ auto append_int = [](std::string key) {
+ encode_int64(123, &key);
+ return key;
+ };
+
+ {
+ std::string valid_key =
+ append_timestamp(versioned::table_version_key({"instance-id",
100}));
+ std::string_view key_view = valid_key;
+ int64_t table_id = 0;
+ Versionstamp decoded_timestamp;
+ ASSERT_TRUE(versioned::decode_table_version_key(&key_view, &table_id,
&decoded_timestamp));
+ EXPECT_TRUE(key_view.empty());
+ EXPECT_EQ(table_id, 100);
+ EXPECT_EQ(decoded_timestamp, timestamp);
+
+ std::string invalid_key =
append_int(versioned::table_version_key({"instance-id", 100}));
+ key_view = invalid_key;
+ EXPECT_FALSE(versioned::decode_table_version_key(&key_view, &table_id,
&decoded_timestamp));
+
+ invalid_key.assign(1, static_cast<char>(CLOUD_VERSIONED_KEY_SPACE03));
+ encode_bytes("version", &invalid_key);
+ encode_int64(1, &invalid_key); // instance_id must be encoded as bytes.
+ encode_bytes("table", &invalid_key);
+ encode_int64(100, &invalid_key);
+ invalid_key = append_timestamp(std::move(invalid_key));
+ key_view = invalid_key;
+ EXPECT_FALSE(versioned::decode_table_version_key(&key_view, &table_id,
&decoded_timestamp));
+ }
+
+ {
+ std::string invalid_key =
append_int(versioned::meta_partition_key({"instance-id", 101}));
+ std::string_view key_view = invalid_key;
+ int64_t partition_id = 0;
+ Versionstamp decoded_timestamp;
+ EXPECT_FALSE(
+ versioned::decode_meta_partition_key(&key_view, &partition_id,
&decoded_timestamp));
+ }
+
+ {
+ std::string invalid_key =
append_int(versioned::meta_index_key({"instance-id", 102}));
+ std::string_view key_view = invalid_key;
+ int64_t index_id = 0;
+ Versionstamp decoded_timestamp;
+ EXPECT_FALSE(versioned::decode_meta_index_key(&key_view, &index_id,
&decoded_timestamp));
+ }
+
+ {
+ std::string invalid_key =
append_int(versioned::meta_tablet_key({"instance-id", 103}));
+ std::string_view key_view = invalid_key;
+ int64_t tablet_id = 0;
+ Versionstamp decoded_timestamp;
+ EXPECT_FALSE(versioned::decode_meta_tablet_key(&key_view, &tablet_id,
&decoded_timestamp));
+
+ invalid_key = versioned::meta_tablet_key({"instance-id", 103});
+ encode_int64(104, &invalid_key);
+ invalid_key = append_timestamp(std::move(invalid_key));
+ key_view = invalid_key;
+ EXPECT_FALSE(versioned::decode_meta_tablet_key(&key_view, &tablet_id,
&decoded_timestamp));
+ }
+
+ {
+ std::string invalid_key =
+ versioned::partition_inverted_index_key({"instance-id", 100,
101, 102});
+ ASSERT_GT(invalid_key.size(), 2);
+ invalid_key[2] = 'x'; // Corrupt the encoded "index" family prefix.
+ std::string_view key_view = invalid_key;
+ int64_t db_id = 0;
+ int64_t table_id = 0;
+ int64_t partition_id = 0;
+ EXPECT_FALSE(versioned::decode_partition_inverted_index_key(&key_view,
&db_id, &table_id,
+
&partition_id));
+ }
+}
+
TEST(KeysTest, DecodeInstanceKey) {
using namespace doris::cloud;
std::string instance_id = "instance_id_deadbeef";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
