This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 204797a537d [feature](merge-cloud) Add cloud rowset meta & tablet meta
definition (#29911)
204797a537d is described below
commit 204797a537d3bbea9fcbeb83a749b45df4b4c6d8
Author: walter <[email protected]>
AuthorDate: Fri Jan 12 20:41:28 2024 +0800
[feature](merge-cloud) Add cloud rowset meta & tablet meta definition
(#29911)
---
be/src/cloud/cloud_meta_mgr.cpp | 4 +-
be/src/cloud/pb_convert.cpp | 578 +++++++++++++++++++++++++++++++++++++---
be/src/cloud/pb_convert.h | 23 ++
gensrc/proto/cloud.proto | 14 +-
gensrc/proto/olap_file.proto | 133 ++++++++-
5 files changed, 700 insertions(+), 52 deletions(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index f2fcf5132f6..e6915eae914 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -29,6 +29,7 @@
#include <vector>
#include "cloud/config.h"
+#include "cloud/pb_convert.h"
#include "common/logging.h"
#include "common/status.h"
#include "common/sync_point.h"
@@ -207,7 +208,8 @@ Status CloudMetaMgr::get_tablet_meta(int64_t tablet_id,
TabletMetaSharedPtr* tab
return Status::InternalError("failed to get tablet meta: {}",
resp.status().msg());
}
*tablet_meta = std::make_shared<TabletMeta>();
- (*tablet_meta)->init_from_pb(resp.tablet_meta());
+ (*tablet_meta)
+
->init_from_pb(cloud_tablet_meta_to_doris(std::move(*resp.mutable_tablet_meta())));
VLOG_DEBUG << "get tablet meta, tablet_id: " <<
(*tablet_meta)->tablet_id();
return Status::OK();
}
diff --git a/be/src/cloud/pb_convert.cpp b/be/src/cloud/pb_convert.cpp
index 6cb388df103..664a9f53c50 100644
--- a/be/src/cloud/pb_convert.cpp
+++ b/be/src/cloud/pb_convert.cpp
@@ -17,62 +17,276 @@
#include "cloud/pb_convert.h"
+#include <gen_cpp/olap_file.pb.h>
+
+#include <utility>
+
namespace doris::cloud {
-TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) {
+RowsetMetaCloudPB doris_rowset_meta_to_cloud(const RowsetMetaPB& in) {
+ RowsetMetaCloudPB out;
+ doris_rowset_meta_to_cloud(&out, in);
+ return out;
+}
+
+RowsetMetaCloudPB doris_rowset_meta_to_cloud(RowsetMetaPB&& in) {
+ RowsetMetaCloudPB out;
+ doris_rowset_meta_to_cloud(&out, std::move(in));
+ return out;
+}
+
+void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB&
in) {
+ // ATTN: please keep the set order aligned with the definition of proto
`TabletSchemaCloudPB`.
+ out->set_rowset_id(in.rowset_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_txn_id(in.txn_id());
+ out->set_tablet_schema_hash(in.tablet_schema_hash());
+ out->set_rowset_type(in.rowset_type());
+ out->set_rowset_state(in.rowset_state());
+ out->set_start_version(in.start_version());
+ out->set_end_version(in.end_version());
+ out->set_version_hash(in.version_hash());
+ out->set_num_rows(in.num_rows());
+ out->set_total_disk_size(in.total_disk_size());
+ out->set_data_disk_size(in.data_disk_size());
+ out->set_index_disk_size(in.index_disk_size());
+ out->mutable_zone_maps()->CopyFrom(in.zone_maps());
+ if (in.has_delete_predicate()) {
+ out->mutable_delete_predicate()->CopyFrom(in.delete_predicate());
+ }
+ out->set_empty(in.empty());
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ out->set_delete_flag(in.delete_flag());
+ out->set_creation_time(in.creation_time());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_num_segments(in.num_segments());
+ out->set_rowset_id_v2(in.rowset_id_v2());
+ out->set_resource_id(in.resource_id());
+ out->set_newest_write_timestamp(in.newest_write_timestamp());
+ out->mutable_segments_key_bounds()->CopyFrom(in.segments_key_bounds());
+ if (in.has_tablet_schema()) {
+ doris_tablet_schema_to_cloud(out->mutable_tablet_schema(),
in.tablet_schema());
+ }
+ out->set_txn_expiration(in.txn_expiration());
+ out->set_segments_overlap_pb(in.segments_overlap_pb());
+ out->mutable_segments_file_size()->CopyFrom(in.segments_file_size());
+ out->set_index_id(in.index_id());
+ if (in.has_schema_version()) {
+ // See cloud/src/meta-service/meta_service_schema.cpp for details.
+ out->set_schema_version(in.schema_version());
+ }
+ out->set_enable_segments_file_size(in.enable_segments_file_size());
+}
+
+void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in) {
// ATTN: please keep the set order aligned with the definition of proto
`TabletSchemaCloudPB`.
+ out->set_rowset_id(in.rowset_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_txn_id(in.txn_id());
+ out->set_tablet_schema_hash(in.tablet_schema_hash());
+ out->set_rowset_type(in.rowset_type());
+ out->set_rowset_state(in.rowset_state());
+ out->set_start_version(in.start_version());
+ out->set_end_version(in.end_version());
+ out->set_version_hash(in.version_hash());
+ out->set_num_rows(in.num_rows());
+ out->set_total_disk_size(in.total_disk_size());
+ out->set_data_disk_size(in.data_disk_size());
+ out->set_index_disk_size(in.index_disk_size());
+ out->mutable_zone_maps()->Swap(in.mutable_zone_maps());
+ if (in.has_delete_predicate()) {
+ out->mutable_delete_predicate()->Swap(in.mutable_delete_predicate());
+ }
+ out->set_empty(in.empty());
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ out->set_delete_flag(in.delete_flag());
+ out->set_creation_time(in.creation_time());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_num_segments(in.num_segments());
+ out->set_rowset_id_v2(in.rowset_id_v2());
+ out->set_resource_id(in.resource_id());
+ out->set_newest_write_timestamp(in.newest_write_timestamp());
+ out->mutable_segments_key_bounds()->Swap(in.mutable_segments_key_bounds());
+ if (in.has_tablet_schema()) {
+ doris_tablet_schema_to_cloud(out->mutable_tablet_schema(),
+ std::move(*in.mutable_tablet_schema()));
+ }
+ out->set_txn_expiration(in.txn_expiration());
+ out->set_segments_overlap_pb(in.segments_overlap_pb());
+ out->mutable_segments_file_size()->Swap(in.mutable_segments_file_size());
+ out->set_index_id(in.index_id());
+ if (in.has_schema_version()) {
+ // See cloud/src/meta-service/meta_service_schema.cpp for details.
+ out->set_schema_version(in.schema_version());
+ }
+ out->set_enable_segments_file_size(in.enable_segments_file_size());
+}
+
+RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB& in) {
+ RowsetMetaPB out;
+ cloud_rowset_meta_to_doris(&out, in);
+ return out;
+}
+
+RowsetMetaPB cloud_rowset_meta_to_doris(RowsetMetaCloudPB&& in) {
+ RowsetMetaPB out;
+ cloud_rowset_meta_to_doris(&out, std::move(in));
+ return out;
+}
+
+void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB&
in) {
+ // ATTN: please keep the set order aligned with the definition of proto
`TabletSchemaCloudPB`.
+ out->set_rowset_id(in.rowset_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_txn_id(in.txn_id());
+ out->set_tablet_schema_hash(in.tablet_schema_hash());
+ out->set_rowset_type(in.rowset_type());
+ out->set_rowset_state(in.rowset_state());
+ out->set_start_version(in.start_version());
+ out->set_end_version(in.end_version());
+ out->set_version_hash(in.version_hash());
+ out->set_num_rows(in.num_rows());
+ out->set_total_disk_size(in.total_disk_size());
+ out->set_data_disk_size(in.data_disk_size());
+ out->set_index_disk_size(in.index_disk_size());
+ out->mutable_zone_maps()->CopyFrom(in.zone_maps());
+ if (in.has_delete_predicate()) {
+ out->mutable_delete_predicate()->CopyFrom(in.delete_predicate());
+ }
+ out->set_empty(in.empty());
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ out->set_delete_flag(in.delete_flag());
+ out->set_creation_time(in.creation_time());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_num_segments(in.num_segments());
+ out->set_rowset_id_v2(in.rowset_id_v2());
+ out->set_resource_id(in.resource_id());
+ out->set_newest_write_timestamp(in.newest_write_timestamp());
+ out->mutable_segments_key_bounds()->CopyFrom(in.segments_key_bounds());
+ if (in.has_tablet_schema()) {
+ cloud_tablet_schema_to_doris(out->mutable_tablet_schema(),
in.tablet_schema());
+ }
+ out->set_txn_expiration(in.txn_expiration());
+ out->set_segments_overlap_pb(in.segments_overlap_pb());
+ out->mutable_segments_file_size()->CopyFrom(in.segments_file_size());
+ out->set_index_id(in.index_id());
+ if (in.has_schema_version()) {
+ // See cloud/src/meta-service/meta_service_schema.cpp for details.
+ out->set_schema_version(in.schema_version());
+ }
+ out->set_enable_segments_file_size(in.enable_segments_file_size());
+}
+
+void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in) {
+ // ATTN: please keep the set order aligned with the definition of proto
`TabletSchemaCloudPB`.
+ out->set_rowset_id(in.rowset_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_txn_id(in.txn_id());
+ out->set_tablet_schema_hash(in.tablet_schema_hash());
+ out->set_rowset_type(in.rowset_type());
+ out->set_rowset_state(in.rowset_state());
+ out->set_start_version(in.start_version());
+ out->set_end_version(in.end_version());
+ out->set_version_hash(in.version_hash());
+ out->set_num_rows(in.num_rows());
+ out->set_total_disk_size(in.total_disk_size());
+ out->set_data_disk_size(in.data_disk_size());
+ out->set_index_disk_size(in.index_disk_size());
+ out->mutable_zone_maps()->Swap(in.mutable_zone_maps());
+ if (in.has_delete_predicate()) {
+ out->mutable_delete_predicate()->Swap(in.mutable_delete_predicate());
+ }
+ out->set_empty(in.empty());
+ out->mutable_load_id()->CopyFrom(in.load_id());
+ out->set_delete_flag(in.delete_flag());
+ out->set_creation_time(in.creation_time());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_num_segments(in.num_segments());
+ out->set_rowset_id_v2(in.rowset_id_v2());
+ out->set_resource_id(in.resource_id());
+ out->set_newest_write_timestamp(in.newest_write_timestamp());
+ out->mutable_segments_key_bounds()->Swap(in.mutable_segments_key_bounds());
+ if (in.has_tablet_schema()) {
+ cloud_tablet_schema_to_doris(out->mutable_tablet_schema(),
+ std::move(*in.mutable_tablet_schema()));
+ }
+ out->set_txn_expiration(in.txn_expiration());
+ out->set_segments_overlap_pb(in.segments_overlap_pb());
+ out->mutable_segments_file_size()->Swap(in.mutable_segments_file_size());
+ out->set_index_id(in.index_id());
+ if (in.has_schema_version()) {
+ // See cloud/src/meta-service/meta_service_schema.cpp for details.
+ out->set_schema_version(in.schema_version());
+ }
+ out->set_enable_segments_file_size(in.enable_segments_file_size());
+}
+
+TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB& in) {
TabletSchemaCloudPB out;
- out.set_keys_type(in.keys_type());
- out.mutable_column()->CopyFrom(in.column());
- out.set_num_short_key_columns(in.num_short_key_columns());
- out.set_num_rows_per_row_block(in.num_rows_per_row_block());
- out.set_compress_kind(in.compress_kind());
- out.set_bf_fpp(in.bf_fpp());
- out.set_next_column_unique_id(in.next_column_unique_id());
- out.set_delete_sign_idx(in.delete_sign_idx());
- out.set_sequence_col_idx(in.sequence_col_idx());
- out.set_sort_type(in.sort_type());
- out.set_sort_col_num(in.sort_col_num());
- out.set_compression_type(in.compression_type());
- out.set_schema_version(in.schema_version());
- out.set_disable_auto_compaction(in.disable_auto_compaction());
- out.mutable_index()->CopyFrom(in.index());
- out.set_version_col_idx(in.version_col_idx());
- out.set_store_row_column(in.store_row_column());
-
out.set_enable_single_replica_compaction(in.enable_single_replica_compaction());
- out.set_skip_write_index_on_load(in.skip_write_index_on_load());
- out.mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
- out.set_is_dynamic_schema(in.is_dynamic_schema());
+ doris_tablet_schema_to_cloud(&out, in);
return out;
}
TabletSchemaCloudPB doris_tablet_schema_to_cloud(TabletSchemaPB&& in) {
// ATTN: please keep the set order aligned with the definition of proto.
TabletSchemaCloudPB out;
- out.set_keys_type(in.keys_type());
- out.mutable_column()->Swap(in.mutable_column());
- out.set_num_short_key_columns(in.num_short_key_columns());
- out.set_num_rows_per_row_block(in.num_rows_per_row_block());
- out.set_compress_kind(in.compress_kind());
- out.set_bf_fpp(in.bf_fpp());
- out.set_next_column_unique_id(in.next_column_unique_id());
- out.set_delete_sign_idx(in.delete_sign_idx());
- out.set_sequence_col_idx(in.sequence_col_idx());
- out.set_sort_type(in.sort_type());
- out.set_sort_col_num(in.sort_col_num());
- out.set_compression_type(in.compression_type());
- out.set_schema_version(in.schema_version());
- out.set_disable_auto_compaction(in.disable_auto_compaction());
- out.mutable_index()->Swap(in.mutable_index());
- out.set_version_col_idx(in.version_col_idx());
- out.set_store_row_column(in.store_row_column());
-
out.set_enable_single_replica_compaction(in.enable_single_replica_compaction());
- out.set_skip_write_index_on_load(in.skip_write_index_on_load());
- out.mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
- out.set_is_dynamic_schema(in.is_dynamic_schema());
+ doris_tablet_schema_to_cloud(&out, std::move(in));
return out;
}
+void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const
TabletSchemaPB& in) {
+ // ATTN: please keep the set order aligned with the definition of proto.
+ out->set_keys_type(in.keys_type());
+ out->mutable_column()->CopyFrom(in.column());
+ out->set_num_short_key_columns(in.num_short_key_columns());
+ out->set_num_rows_per_row_block(in.num_rows_per_row_block());
+ out->set_compress_kind(in.compress_kind());
+ out->set_bf_fpp(in.bf_fpp());
+ out->set_next_column_unique_id(in.next_column_unique_id());
+ out->set_delete_sign_idx(in.delete_sign_idx());
+ out->set_sequence_col_idx(in.sequence_col_idx());
+ out->set_sort_type(in.sort_type());
+ out->set_sort_col_num(in.sort_col_num());
+ out->set_compression_type(in.compression_type());
+ out->set_schema_version(in.schema_version());
+ out->set_disable_auto_compaction(in.disable_auto_compaction());
+ out->mutable_index()->CopyFrom(in.index());
+ out->set_version_col_idx(in.version_col_idx());
+ out->set_store_row_column(in.store_row_column());
+
out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
+ out->set_skip_write_index_on_load(in.skip_write_index_on_load());
+ out->mutable_cluster_key_idxes()->CopyFrom(in.cluster_key_idxes());
+ out->set_is_dynamic_schema(in.is_dynamic_schema());
+}
+
+void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&&
in) {
+ out->set_keys_type(in.keys_type());
+ out->mutable_column()->Swap(in.mutable_column());
+ out->set_num_short_key_columns(in.num_short_key_columns());
+ out->set_num_rows_per_row_block(in.num_rows_per_row_block());
+ out->set_compress_kind(in.compress_kind());
+ out->set_bf_fpp(in.bf_fpp());
+ out->set_next_column_unique_id(in.next_column_unique_id());
+ out->set_delete_sign_idx(in.delete_sign_idx());
+ out->set_sequence_col_idx(in.sequence_col_idx());
+ out->set_sort_type(in.sort_type());
+ out->set_sort_col_num(in.sort_col_num());
+ out->set_compression_type(in.compression_type());
+ out->set_schema_version(in.schema_version());
+ out->set_disable_auto_compaction(in.disable_auto_compaction());
+ out->mutable_index()->Swap(in.mutable_index());
+ out->set_version_col_idx(in.version_col_idx());
+ out->set_store_row_column(in.store_row_column());
+
out->set_enable_single_replica_compaction(in.enable_single_replica_compaction());
+ out->set_skip_write_index_on_load(in.skip_write_index_on_load());
+ out->mutable_cluster_key_idxes()->Swap(in.mutable_cluster_key_idxes());
+ out->set_is_dynamic_schema(in.is_dynamic_schema());
+}
+
TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB& in) {
TabletSchemaPB out;
cloud_tablet_schema_to_doris(&out, in);
@@ -135,4 +349,282 @@ void cloud_tablet_schema_to_doris(TabletSchemaPB* out,
TabletSchemaCloudPB&& in)
out->set_is_dynamic_schema(in.is_dynamic_schema());
}
+TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB& in) {
+ TabletMetaCloudPB out;
+ doris_tablet_meta_to_cloud(&out, in);
+ return out;
+}
+
+TabletMetaCloudPB doris_tablet_meta_to_cloud(TabletMetaPB&& in) {
+ TabletMetaCloudPB out;
+ doris_tablet_meta_to_cloud(&out, std::move(in));
+ return out;
+}
+
+void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB&
in) {
+ // ATTN: please keep the set order aligned with the definition of proto.
+ out->set_table_id(in.table_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_schema_hash(in.schema_hash());
+ out->set_shard_id(in.shard_id());
+ out->set_creation_time(in.creation_time());
+ out->set_cumulative_layer_point(in.cumulative_layer_point());
+ out->set_tablet_state(in.tablet_state());
+ if (in.has_schema()) {
+ doris_tablet_schema_to_cloud(out->mutable_schema(), in.schema());
+ }
+ if (in.rs_metas_size()) {
+ out->mutable_rs_metas()->Reserve(in.rs_metas_size());
+ for (const auto& rs_meta : in.rs_metas()) {
+ doris_rowset_meta_to_cloud(out->add_rs_metas(), rs_meta);
+ }
+ }
+ // ATTN: inc_rs_metas are deprecated, ignored here.
+ if (in.has_alter_task()) {
+ out->mutable_alter_task()->CopyFrom(in.alter_task());
+ }
+ out->set_in_restore_mode(in.in_restore_mode());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_end_rowset_id(in.end_rowset_id());
+ out->set_preferred_rowset_type(in.preferred_rowset_type());
+ out->set_tablet_type(in.tablet_type());
+ if (in.stale_rs_metas_size()) {
+ out->mutable_stale_rs_metas()->Reserve(in.stale_rs_metas_size());
+ for (const auto& rs_meta : in.stale_rs_metas()) {
+ doris_rowset_meta_to_cloud(out->add_stale_rs_metas(), rs_meta);
+ }
+ }
+ out->set_replica_id(in.replica_id());
+ if (in.has_delete_bitmap()) {
+ out->mutable_delete_bitmap()->CopyFrom(in.delete_bitmap());
+ }
+
out->set_enable_unique_key_merge_on_write(in.enable_unique_key_merge_on_write());
+ out->set_storage_policy_id(in.storage_policy_id());
+ out->mutable_cooldown_meta_id()->CopyFrom(in.cooldown_meta_id());
+ if (in.has_binlog_config()) {
+ out->mutable_binlog_config()->CopyFrom(in.binlog_config());
+ }
+ out->set_compaction_policy(in.compaction_policy());
+
out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes());
+ out->set_time_series_compaction_file_count_threshold(
+ in.time_series_compaction_file_count_threshold());
+ out->set_time_series_compaction_time_threshold_seconds(
+ in.time_series_compaction_time_threshold_seconds());
+ out->set_time_series_compaction_empty_rowsets_threshold(
+ in.time_series_compaction_empty_rowsets_threshold());
+ out->set_index_id(in.index_id());
+ out->set_is_in_memory(in.is_in_memory());
+ out->set_is_persistent(in.is_persistent());
+ out->set_table_name(in.table_name());
+ out->set_ttl_seconds(in.ttl_seconds());
+ if (in.has_schema_version()) {
+ out->set_schema_version(in.schema_version());
+ }
+}
+
+void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in) {
+ // ATTN: please keep the set order aligned with the definition of proto.
+ out->set_table_id(in.table_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_schema_hash(in.schema_hash());
+ out->set_shard_id(in.shard_id());
+ out->set_creation_time(in.creation_time());
+ out->set_cumulative_layer_point(in.cumulative_layer_point());
+ out->set_tablet_state(in.tablet_state());
+ if (in.has_schema()) {
+ doris_tablet_schema_to_cloud(out->mutable_schema(),
std::move(*in.mutable_schema()));
+ }
+ if (in.rs_metas_size()) {
+ size_t rs_metas_size = in.rs_metas_size();
+ out->mutable_rs_metas()->Reserve(rs_metas_size);
+ for (size_t i = 0; i < rs_metas_size; ++i) {
+ doris_rowset_meta_to_cloud(out->add_rs_metas(),
std::move(*in.mutable_rs_metas(i)));
+ }
+ }
+ // ATTN: inc_rs_metas are deprecated, ignored here.
+ if (in.has_alter_task()) {
+ out->mutable_alter_task()->Swap(in.mutable_alter_task());
+ }
+ out->set_in_restore_mode(in.in_restore_mode());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_end_rowset_id(in.end_rowset_id());
+ out->set_preferred_rowset_type(in.preferred_rowset_type());
+ out->set_tablet_type(in.tablet_type());
+ if (in.stale_rs_metas_size()) {
+ size_t rs_metas_size = in.stale_rs_metas_size();
+ out->mutable_stale_rs_metas()->Reserve(rs_metas_size);
+ for (size_t i = 0; i < rs_metas_size; i++) {
+ doris_rowset_meta_to_cloud(out->add_stale_rs_metas(),
+
std::move(*in.mutable_stale_rs_metas(i)));
+ }
+ }
+ out->set_replica_id(in.replica_id());
+ if (in.has_delete_bitmap()) {
+ out->mutable_delete_bitmap()->Swap(in.mutable_delete_bitmap());
+ }
+
out->set_enable_unique_key_merge_on_write(in.enable_unique_key_merge_on_write());
+ out->set_storage_policy_id(in.storage_policy_id());
+ out->mutable_cooldown_meta_id()->CopyFrom(in.cooldown_meta_id());
+ if (in.has_binlog_config()) {
+ out->mutable_binlog_config()->Swap(in.mutable_binlog_config());
+ }
+ out->set_compaction_policy(in.compaction_policy());
+
out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes());
+ out->set_time_series_compaction_file_count_threshold(
+ in.time_series_compaction_file_count_threshold());
+ out->set_time_series_compaction_time_threshold_seconds(
+ in.time_series_compaction_time_threshold_seconds());
+ out->set_time_series_compaction_empty_rowsets_threshold(
+ in.time_series_compaction_empty_rowsets_threshold());
+ out->set_index_id(in.index_id());
+ out->set_is_in_memory(in.is_in_memory());
+ out->set_is_persistent(in.is_persistent());
+ out->set_table_name(in.table_name());
+ out->set_ttl_seconds(in.ttl_seconds());
+ if (in.has_schema_version()) {
+ out->set_schema_version(in.schema_version());
+ }
+}
+
+TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB& in) {
+ TabletMetaPB out;
+ cloud_tablet_meta_to_doris(&out, in);
+ return out;
+}
+
+TabletMetaPB cloud_tablet_meta_to_doris(TabletMetaCloudPB&& in) {
+ TabletMetaPB out;
+ cloud_tablet_meta_to_doris(&out, std::move(in));
+ return out;
+}
+
+void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB&
in) {
+ // ATTN: please keep the set order aligned with the definition of proto.
+ out->set_table_id(in.table_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_schema_hash(in.schema_hash());
+ out->set_shard_id(in.shard_id());
+ out->set_creation_time(in.creation_time());
+ out->set_cumulative_layer_point(in.cumulative_layer_point());
+ out->set_tablet_state(in.tablet_state());
+ if (in.has_schema()) {
+ cloud_tablet_schema_to_doris(out->mutable_schema(), in.schema());
+ }
+ if (in.rs_metas_size()) {
+ out->mutable_rs_metas()->Reserve(in.rs_metas_size());
+ for (const auto& rs_meta : in.rs_metas()) {
+ cloud_rowset_meta_to_doris(out->add_rs_metas(), rs_meta);
+ }
+ }
+ // ATTN: inc_rs_metas are deprecated, ignored here.
+ if (in.has_alter_task()) {
+ out->mutable_alter_task()->CopyFrom(in.alter_task());
+ }
+ out->set_in_restore_mode(in.in_restore_mode());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_end_rowset_id(in.end_rowset_id());
+ out->set_preferred_rowset_type(in.preferred_rowset_type());
+ out->set_tablet_type(in.tablet_type());
+ if (in.stale_rs_metas_size()) {
+ out->mutable_stale_rs_metas()->Reserve(in.stale_rs_metas_size());
+ for (const auto& rs_meta : in.stale_rs_metas()) {
+ cloud_rowset_meta_to_doris(out->add_stale_rs_metas(), rs_meta);
+ }
+ }
+ out->set_replica_id(in.replica_id());
+ if (in.has_delete_bitmap()) {
+ out->mutable_delete_bitmap()->CopyFrom(in.delete_bitmap());
+ }
+
out->set_enable_unique_key_merge_on_write(in.enable_unique_key_merge_on_write());
+ out->set_storage_policy_id(in.storage_policy_id());
+ out->mutable_cooldown_meta_id()->CopyFrom(in.cooldown_meta_id());
+ if (in.has_binlog_config()) {
+ out->mutable_binlog_config()->CopyFrom(in.binlog_config());
+ }
+ out->set_compaction_policy(in.compaction_policy());
+
out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes());
+ out->set_time_series_compaction_file_count_threshold(
+ in.time_series_compaction_file_count_threshold());
+ out->set_time_series_compaction_time_threshold_seconds(
+ in.time_series_compaction_time_threshold_seconds());
+ out->set_time_series_compaction_empty_rowsets_threshold(
+ in.time_series_compaction_empty_rowsets_threshold());
+ out->set_index_id(in.index_id());
+ out->set_is_in_memory(in.is_in_memory());
+ out->set_is_persistent(in.is_persistent());
+ out->set_table_name(in.table_name());
+ out->set_ttl_seconds(in.ttl_seconds());
+ if (in.has_schema_version()) {
+ out->set_schema_version(in.schema_version());
+ }
+}
+
+void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in) {
+ // ATTN: please keep the set order aligned with the definition of proto.
+ out->set_table_id(in.table_id());
+ out->set_partition_id(in.partition_id());
+ out->set_tablet_id(in.tablet_id());
+ out->set_schema_hash(in.schema_hash());
+ out->set_shard_id(in.shard_id());
+ out->set_creation_time(in.creation_time());
+ out->set_cumulative_layer_point(in.cumulative_layer_point());
+ out->set_tablet_state(in.tablet_state());
+ if (in.has_schema()) {
+ cloud_tablet_schema_to_doris(out->mutable_schema(),
std::move(*in.mutable_schema()));
+ }
+ if (in.rs_metas_size()) {
+ size_t rs_metas_size = in.rs_metas_size();
+ out->mutable_rs_metas()->Reserve(rs_metas_size);
+ for (size_t i = 0; i < rs_metas_size; ++i) {
+ cloud_rowset_meta_to_doris(out->add_rs_metas(),
std::move(*in.mutable_rs_metas(i)));
+ }
+ }
+ // ATTN: inc_rs_metas are deprecated, ignored here.
+ if (in.has_alter_task()) {
+ out->mutable_alter_task()->Swap(in.mutable_alter_task());
+ }
+ out->set_in_restore_mode(in.in_restore_mode());
+ out->mutable_tablet_uid()->CopyFrom(in.tablet_uid());
+ out->set_end_rowset_id(in.end_rowset_id());
+ out->set_preferred_rowset_type(in.preferred_rowset_type());
+ out->set_tablet_type(in.tablet_type());
+ if (in.stale_rs_metas_size()) {
+ size_t rs_metas_size = in.stale_rs_metas_size();
+ out->mutable_stale_rs_metas()->Reserve(rs_metas_size);
+ for (size_t i = 0; i < rs_metas_size; i++) {
+ cloud_rowset_meta_to_doris(out->add_stale_rs_metas(),
+
std::move(*in.mutable_stale_rs_metas(i)));
+ }
+ }
+ out->set_replica_id(in.replica_id());
+ if (in.has_delete_bitmap()) {
+ out->mutable_delete_bitmap()->Swap(in.mutable_delete_bitmap());
+ }
+
out->set_enable_unique_key_merge_on_write(in.enable_unique_key_merge_on_write());
+ out->set_storage_policy_id(in.storage_policy_id());
+ out->mutable_cooldown_meta_id()->CopyFrom(in.cooldown_meta_id());
+ if (in.has_binlog_config()) {
+ out->mutable_binlog_config()->Swap(in.mutable_binlog_config());
+ }
+ out->set_compaction_policy(in.compaction_policy());
+
out->set_time_series_compaction_goal_size_mbytes(in.time_series_compaction_goal_size_mbytes());
+ out->set_time_series_compaction_file_count_threshold(
+ in.time_series_compaction_file_count_threshold());
+ out->set_time_series_compaction_time_threshold_seconds(
+ in.time_series_compaction_time_threshold_seconds());
+ out->set_time_series_compaction_empty_rowsets_threshold(
+ in.time_series_compaction_empty_rowsets_threshold());
+ out->set_index_id(in.index_id());
+ out->set_is_in_memory(in.is_in_memory());
+ out->set_is_persistent(in.is_persistent());
+ out->set_table_name(in.table_name());
+ out->set_ttl_seconds(in.ttl_seconds());
+ if (in.has_schema_version()) {
+ out->set_schema_version(in.schema_version());
+ }
+}
+
} // namespace doris::cloud
diff --git a/be/src/cloud/pb_convert.h b/be/src/cloud/pb_convert.h
index db8b77d7b09..0cfa033f293 100644
--- a/be/src/cloud/pb_convert.h
+++ b/be/src/cloud/pb_convert.h
@@ -19,11 +19,34 @@
namespace doris::cloud {
+// RowsetMetaPB <=> RowsetMetaCloudPB
+RowsetMetaCloudPB doris_rowset_meta_to_cloud(const RowsetMetaPB&);
+RowsetMetaCloudPB doris_rowset_meta_to_cloud(RowsetMetaPB&&);
+void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, const RowsetMetaPB&
in);
+void doris_rowset_meta_to_cloud(RowsetMetaCloudPB* out, RowsetMetaPB&& in);
+RowsetMetaPB cloud_rowset_meta_to_doris(const RowsetMetaCloudPB&);
+RowsetMetaPB cloud_rowset_meta_to_doris(RowsetMetaCloudPB&&);
+void cloud_rowset_meta_to_doris(RowsetMetaPB* out, const RowsetMetaCloudPB&
in);
+void cloud_rowset_meta_to_doris(RowsetMetaPB* out, RowsetMetaCloudPB&& in);
+
+// TabletSchemaPB <=> TabletSchemaCloudPB
TabletSchemaCloudPB doris_tablet_schema_to_cloud(const TabletSchemaPB&);
TabletSchemaCloudPB doris_tablet_schema_to_cloud(TabletSchemaPB&&);
+void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, const
TabletSchemaPB& in);
+void doris_tablet_schema_to_cloud(TabletSchemaCloudPB* out, TabletSchemaPB&&
in);
TabletSchemaPB cloud_tablet_schema_to_doris(const TabletSchemaCloudPB&);
TabletSchemaPB cloud_tablet_schema_to_doris(TabletSchemaCloudPB&&);
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, const
TabletSchemaCloudPB& in);
void cloud_tablet_schema_to_doris(TabletSchemaPB* out, TabletSchemaCloudPB&&
in);
+// TabletMetaPB <=> TabletMetaCloudPB
+TabletMetaCloudPB doris_tablet_meta_to_cloud(const TabletMetaPB&);
+TabletMetaCloudPB doris_tablet_meta_to_cloud(TabletMetaPB&&);
+void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, const TabletMetaPB&
in);
+void doris_tablet_meta_to_cloud(TabletMetaCloudPB* out, TabletMetaPB&& in);
+TabletMetaPB cloud_tablet_meta_to_doris(const TabletMetaCloudPB&);
+TabletMetaPB cloud_tablet_meta_to_doris(TabletMetaCloudPB&&);
+void cloud_tablet_meta_to_doris(TabletMetaPB* out, const TabletMetaCloudPB&
in);
+void cloud_tablet_meta_to_doris(TabletMetaPB* out, TabletMetaCloudPB&& in);
+
} // namespace doris::cloud
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index b3794e58128..ebf684a1ce4 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -396,7 +396,7 @@ message RecycleRowsetPB {
optional int64 creation_time = 3;
optional int64 expiration = 4; // expiration timestamp
optional Type type = 5;
- optional doris.RowsetMetaPB rowset_meta = 6;
+ optional doris.RowsetMetaCloudPB rowset_meta = 6;
}
message RecycleStagePB {
@@ -724,7 +724,7 @@ message GetObjStoreInfoResponse {
message CreateTabletsRequest {
optional string cloud_unique_id = 1; // For auth
- repeated doris.TabletMetaPB tablet_metas = 2;
+ repeated doris.TabletMetaCloudPB tablet_metas = 2;
}
message CreateTabletsResponse {
@@ -743,7 +743,7 @@ message UpdateTabletResponse {
message UpdateTabletSchemaRequest {
optional string cloud_unique_id = 1; // For auth
optional int64 tablet_id = 2;
- optional doris.TabletSchemaPB tablet_schema = 3;
+ optional doris.TabletSchemaCloudPB tablet_schema = 3;
}
message UpdateTabletSchemaResponse {
@@ -764,18 +764,18 @@ message GetTabletRequest {
message GetTabletResponse {
optional MetaServiceResponseStatus status = 1;
- optional doris.TabletMetaPB tablet_meta = 2;
+ optional doris.TabletMetaCloudPB tablet_meta = 2;
}
message CreateRowsetRequest {
optional string cloud_unique_id = 1; // For auth
- optional doris.RowsetMetaPB rowset_meta = 2;
+ optional doris.RowsetMetaCloudPB rowset_meta = 2;
optional bool temporary = 3;
}
message CreateRowsetResponse {
optional MetaServiceResponseStatus status = 1;
- optional doris.RowsetMetaPB existed_rowset_meta = 2;
+ optional doris.RowsetMetaCloudPB existed_rowset_meta = 2;
}
message GetRowsetRequest {
@@ -791,7 +791,7 @@ message GetRowsetRequest {
message GetRowsetResponse {
optional MetaServiceResponseStatus status = 1;
- repeated doris.RowsetMetaPB rowset_meta = 2;
+ repeated doris.RowsetMetaCloudPB rowset_meta = 2;
optional TabletStatsPB stats = 3;
}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 6a407858a33..8e2ba5c0498 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -60,6 +60,8 @@ message KeyBoundsPB {
required bytes max_key = 2;
}
+// ATTN: When adding or deleting fields, please update `message
RowsetMetaCloudPB`
+// simultaneously and modify the conversion function in the
be/src/cloud/pb_convert.{h,cpp}.
message RowsetMetaPB {
required int64 rowset_id = 1; // Deprecated. Use rowset_id_v2 instead.
optional int64 partition_id = 2;
@@ -111,6 +113,75 @@ message RowsetMetaPB {
repeated KeyBoundsPB segments_key_bounds = 27;
// tablet meta pb, for compaction
optional TabletSchemaPB tablet_schema = 28;
+ // alpha_rowset_extra_meta_pb is deleted
+ reserved 50;
+ // to indicate whether the data between the segments overlap
+ optional SegmentsOverlapPB segments_overlap_pb = 51 [default =
OVERLAP_UNKNOWN];
+
+ // For cloud
+ // for data recycling
+ optional int64 txn_expiration = 1000;
+ // the field is a vector, rename it
+ repeated int64 segments_file_size = 1001;
+ // index_id, schema_version -> schema
+ optional int64 index_id = 1002;
+ optional int32 schema_version = 1003;
+ // If enable_segments_file_size is false,
+ // the segments_file_size maybe is empty or error
+ optional bool enable_segments_file_size = 1004;
+}
+
+message RowsetMetaCloudPB {
+ required int64 rowset_id = 1; // Deprecated. Use rowset_id_v2 instead.
+ optional int64 partition_id = 2;
+ optional int64 tablet_id = 3;
+ // only for pending rowset
+ optional int64 txn_id = 4;
+ // tablet id and schema hash could find tablet
+ optional int32 tablet_schema_hash = 5;
+ optional RowsetTypePB rowset_type = 6;
+ optional RowsetStatePB rowset_state = 7;
+ // only for visible rowset. For alpha rowset, it's the same as
PDelta.start_version
+ optional int64 start_version = 8;
+ // only for visible rowset. For alpha rowset, it's the same as
PDelta.end_version
+ optional int64 end_version = 9;
+ // only for visible rowset. For alpha rowset, it's the same as
PDelta.version_hash
+ optional int64 version_hash = 10;
+ // total number of rows. For alpha rowset, it's calculated from segment
group
+ optional int64 num_rows = 11;
+ // calculated by index + data
+ optional int64 total_disk_size = 12;
+ // calculated sum(segmentgroup.data_size)
+ optional int64 data_disk_size = 13;
+ // calculated sum(segmentgroup.index_size)
+ optional int64 index_disk_size = 14;
+ // rowset level column min/max/null statistics, not not used currently
+ repeated ZoneMap zone_maps = 15;
+ optional DeletePredicatePB delete_predicate = 16;
+ // calculated from segment group
+ optional bool empty = 17;
+ // only for pending rowset
+ optional PUniqueId load_id = 18;
+ // not set, get from olap index header msg?
+ optional bool delete_flag = 19;
+ // For alpha rowset, it's the same as PDelta.creation_time
+ optional int64 creation_time = 20;
+ optional PUniqueId tablet_uid = 21;
+ // total number of segments
+ optional int64 num_segments = 22;
+ // rowset id definition, it will replace required rowset id
+ optional string rowset_id_v2 = 23;
+ // resource id
+ optional string resource_id = 24;
+ // used to be oldest write time: earliest write time
+ reserved 25;
+ // latest write time
+ optional int64 newest_write_timestamp = 26 [default = -1];
+ // the encoded segment min/max key of segments in this rowset,
+ // only used in unique key data model with primary_key_index support.
+ repeated KeyBoundsPB segments_key_bounds = 27;
+ // tablet meta pb, for compaction
+ optional TabletSchemaCloudPB tablet_schema = 28;
// for data recycling in CLOUD_MODE
optional int64 txn_expiration = 29;
// alpha_rowset_extra_meta_pb is deleted
@@ -118,7 +189,7 @@ message RowsetMetaPB {
// to indicate whether the data between the segments overlap
optional SegmentsOverlapPB segments_overlap_pb = 51 [default =
OVERLAP_UNKNOWN];
- // For cloud
+ // cloud
// the field is a vector, rename it
repeated int64 segments_file_size = 100;
// index_id, schema_version -> schema
@@ -339,6 +410,8 @@ message BinlogConfigPB {
optional int64 max_history_nums = 4;
}
+// ATTN: When adding or deleting fields, please update `message
TabletMetaCloudPB`
+// simultaneously and modify the conversion function in the
be/src/cloud/pb_convert.{h,cpp}.
message TabletMetaPB {
optional int64 table_id = 1; // ?
optional int64 partition_id = 2; // ?
@@ -379,6 +452,64 @@ message TabletMetaPB {
optional int64 time_series_compaction_file_count_threshold = 30 [default =
2000];
optional int64 time_series_compaction_time_threshold_seconds = 31 [default
= 3600];
optional int64 time_series_compaction_empty_rowsets_threshold = 32
[default = 5];
+
+ // For cloud
+ optional int64 index_id = 1000;
+ optional bool is_in_memory = 1001;
+ optional bool is_persistent = 1002;
+ optional string table_name = 1003;
+ optional int64 ttl_seconds = 1004;
+ optional int32 schema_version = 1005; // index_id, schema_version -> schema
+}
+
+message TabletMetaCloudPB {
+ optional int64 table_id = 1; // ?
+ optional int64 partition_id = 2; // ?
+ optional int64 tablet_id = 3; // OlapHeaderMessage.tablet_id
+ optional int32 schema_hash = 4; // OlapHeaderMessage.schema_hash
+ optional int32 shard_id = 5; // OlapHeaderMessage.shard
+ optional int64 creation_time = 6; // OlapHeaderMessage.creation_time
+ optional int64 cumulative_layer_point = 7; //
OlapHeaderMessage.cumulative_layer_point
+
+ optional TabletStatePB tablet_state = 8;
+ optional TabletSchemaCloudPB schema = 9;
+ repeated RowsetMetaCloudPB rs_metas = 10;
+ // @Deprecated
+ repeated RowsetMetaCloudPB inc_rs_metas = 11;
+ optional AlterTabletPB alter_task = 12;
+ // if true, this tablet will not do compaction,
+ // and does not create init version
+ optional bool in_restore_mode = 13 [default = false]; //
OlapHeaderMessage.in_restore_mode
+ // a uniqued id to identified tablet with same tablet_id and schema hash
+ optional PUniqueId tablet_uid = 14;
+ optional int64 end_rowset_id = 15;
+ optional RowsetTypePB preferred_rowset_type = 16;
+ optional TabletTypePB tablet_type = 17;
+ repeated RowsetMetaCloudPB stale_rs_metas = 18;
+ // optional StorageMediumPB storage_medium = 19 [default = HDD];
+ // optional string remote_storage_name = 20;
+ optional int64 replica_id = 21 [default = 0];
+ // 22 used to be string storage_policy = 22;
+ reserved 22;
+ optional DeleteBitmapPB delete_bitmap = 23;
+ // Use primary key index to speed up tabel unique key model
+ optional bool enable_unique_key_merge_on_write = 24 [default = false];
+ optional int64 index_id = 25;
+ optional bool is_in_memory = 26 [default=false];
+ optional bool is_persistent = 27 [default=false];
+ optional int64 storage_policy_id = 28;
+ optional PUniqueId cooldown_meta_id = 29;
+ optional BinlogConfigPB binlog_config = 30;
+ optional string compaction_policy = 31 [default = "size_based"];
+ optional int64 time_series_compaction_goal_size_mbytes = 32 [default =
1024];
+ optional int64 time_series_compaction_file_count_threshold = 33 [default =
2000];
+ optional int64 time_series_compaction_time_threshold_seconds = 34 [default
= 3600];
+ optional int64 time_series_compaction_empty_rowsets_threshold = 35
[default = 5];
+
+ // Use for selectdb-cloud
+ optional string table_name = 101;
+ optional int64 ttl_seconds = 102;
+ optional int32 schema_version = 103; // index_id, schema_version -> schema
}
message OLAPRawDeltaHeaderMessage {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
