This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new c5b8626b968 branch-4.0: [Opt](cloud) use unordered_map to reduce meta
lock held time in `delete_expired_stale_rowsets` #53467 (#56475)
c5b8626b968 is described below
commit c5b8626b9683d3413c464ebc59c602ee7a62a725
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Fri Sep 26 15:29:10 2025 +0800
branch-4.0: [Opt](cloud) use unordered_map to reduce meta lock held time in
`delete_expired_stale_rowsets` #53467 (#56475)
Cherry-picked from #53467
Co-authored-by: bobhan1 <[email protected]>
---
be/src/agent/task_worker_pool.cpp | 8 +-
be/src/cloud/cloud_meta_mgr.cpp | 2 +-
be/src/cloud/cloud_schema_change_job.cpp | 14 +-
be/src/cloud/cloud_snapshot_mgr.cpp | 2 +-
be/src/cloud/cloud_tablet.cpp | 11 +-
.../exec/schema_scanner/schema_tablets_scanner.cpp | 11 +-
be/src/olap/base_tablet.cpp | 27 +--
be/src/olap/cumulative_compaction_policy.cpp | 8 +-
be/src/olap/cumulative_compaction_policy.h | 5 +-
.../cumulative_compaction_time_series_policy.cpp | 8 +-
.../cumulative_compaction_time_series_policy.h | 3 +-
be/src/olap/data_dir.cpp | 4 +-
be/src/olap/olap_server.cpp | 2 +-
be/src/olap/rowset/rowset_meta.h | 3 +-
be/src/olap/tablet.cpp | 25 ++-
be/src/olap/tablet.h | 2 +-
be/src/olap/tablet_meta.cpp | 107 ++++------
be/src/olap/tablet_meta.h | 36 ++--
be/src/olap/task/engine_clone_task.cpp | 2 +-
be/src/olap/version_graph.cpp | 199 +-----------------
be/src/olap/version_graph.h | 225 ++++++++++++++++++++-
be/test/cloud/cloud_meta_mgr_test.cpp | 22 +-
.../compaction_delete_bitmap_calculator_test.cpp | 4 +-
be/test/olap/delete_handler_test.cpp | 2 +-
24 files changed, 369 insertions(+), 363 deletions(-)
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index e3ca1a4b473..4f623124e47 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -895,7 +895,7 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
tablet->tablet_meta()->mutable_tablet_schema()->set_is_in_memory(
tablet_meta_info.is_in_memory);
std::shared_lock rlock(tablet->get_header_lock());
- for (auto& rowset_meta :
tablet->tablet_meta()->all_mutable_rs_metas()) {
+ for (auto& [_, rowset_meta] :
tablet->tablet_meta()->all_mutable_rs_metas()) {
rowset_meta->tablet_schema()->set_is_in_memory(tablet_meta_info.is_in_memory);
}
tablet->tablet_schema_unlocked()->set_is_in_memory(tablet_meta_info.is_in_memory);
@@ -993,7 +993,7 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
std::shared_lock rlock(tablet->get_header_lock());
tablet->tablet_meta()->mutable_tablet_schema()->set_enable_single_replica_compaction(
tablet_meta_info.enable_single_replica_compaction);
- for (auto& rowset_meta :
tablet->tablet_meta()->all_mutable_rs_metas()) {
+ for (auto& [_, rowset_meta] :
tablet->tablet_meta()->all_mutable_rs_metas()) {
rowset_meta->tablet_schema()->set_enable_single_replica_compaction(
tablet_meta_info.enable_single_replica_compaction);
}
@@ -1005,7 +1005,7 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
std::shared_lock rlock(tablet->get_header_lock());
tablet->tablet_meta()->mutable_tablet_schema()->set_disable_auto_compaction(
tablet_meta_info.disable_auto_compaction);
- for (auto& rowset_meta :
tablet->tablet_meta()->all_mutable_rs_metas()) {
+ for (auto& [_, rowset_meta] :
tablet->tablet_meta()->all_mutable_rs_metas()) {
rowset_meta->tablet_schema()->set_disable_auto_compaction(
tablet_meta_info.disable_auto_compaction);
}
@@ -1018,7 +1018,7 @@ void update_tablet_meta_callback(StorageEngine& engine,
const TAgentTaskRequest&
std::shared_lock rlock(tablet->get_header_lock());
tablet->tablet_meta()->mutable_tablet_schema()->set_skip_write_index_on_load(
tablet_meta_info.skip_write_index_on_load);
- for (auto& rowset_meta :
tablet->tablet_meta()->all_mutable_rs_metas()) {
+ for (auto& [_, rowset_meta] :
tablet->tablet_meta()->all_mutable_rs_metas()) {
rowset_meta->tablet_schema()->set_skip_write_index_on_load(
tablet_meta_info.skip_write_index_on_load);
}
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index bedac44b67a..cc2440e1b2e 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -2035,7 +2035,7 @@ Status CloudMetaMgr::fill_version_holes(CloudTablet*
tablet, int64_t max_version
}
Versions existing_versions;
- for (const auto& rs : tablet->tablet_meta()->all_rs_metas()) {
+ for (const auto& [_, rs] : tablet->tablet_meta()->all_rs_metas()) {
existing_versions.emplace_back(rs->version());
}
diff --git a/be/src/cloud/cloud_schema_change_job.cpp
b/be/src/cloud/cloud_schema_change_job.cpp
index 6af55f7e507..7b78033b8e3 100644
--- a/be/src/cloud/cloud_schema_change_job.cpp
+++ b/be/src/cloud/cloud_schema_change_job.cpp
@@ -490,12 +490,14 @@ Status
CloudSchemaChangeJob::_process_delete_bitmap(int64_t alter_version,
tmp_meta->delete_bitmap().delete_bitmap.clear();
// Keep only version [0-1] rowset, other rowsets will be added in
_output_rowsets
auto& rs_metas = tmp_meta->all_mutable_rs_metas();
- rs_metas.erase(std::remove_if(rs_metas.begin(), rs_metas.end(),
- [](const RowsetMetaSharedPtr& rs_meta) {
- return !(rs_meta->version().first == 0 &&
- rs_meta->version().second == 1);
- }),
- rs_metas.end());
+ for (auto it = rs_metas.begin(); it != rs_metas.end();) {
+ const auto& rs_meta = it->second;
+ if (rs_meta->version().first == 0 && rs_meta->version().second == 1) {
+ ++it;
+ } else {
+ it = rs_metas.erase(it);
+ }
+ }
std::shared_ptr<CloudTablet> tmp_tablet =
std::make_shared<CloudTablet>(_cloud_storage_engine, tmp_meta);
diff --git a/be/src/cloud/cloud_snapshot_mgr.cpp
b/be/src/cloud/cloud_snapshot_mgr.cpp
index 5ceac0700a9..c943f6b55fb 100644
--- a/be/src/cloud/cloud_snapshot_mgr.cpp
+++ b/be/src/cloud/cloud_snapshot_mgr.cpp
@@ -83,7 +83,7 @@ Status CloudSnapshotMgr::make_snapshot(int64_t
target_tablet_id, StorageResource
if (tablet_meta.all_rs_metas().size() > 0) {
tablet_meta_pb.mutable_inc_rs_metas()->Reserve(
cast_set<int>(tablet_meta.all_rs_metas().size()));
- for (auto& rs : tablet_meta.all_rs_metas()) {
+ for (auto& [_, rs] : tablet_meta.all_rs_metas()) {
rs->to_rowset_pb(tablet_meta_pb.add_rs_metas());
}
}
diff --git a/be/src/cloud/cloud_tablet.cpp b/be/src/cloud/cloud_tablet.cpp
index 6e5db7257ce..f1f02ecc38d 100644
--- a/be/src/cloud/cloud_tablet.cpp
+++ b/be/src/cloud/cloud_tablet.cpp
@@ -297,10 +297,11 @@ Status
CloudTablet::capture_rs_readers_with_freshness_tolerance(
return ret;
};
// use std::views::concat after C++26
- bool should_fallback = std::ranges::any_of(_tablet_meta->all_rs_metas(),
-
should_be_visible_but_not_warmed_up) ||
-
std::ranges::any_of(_tablet_meta->all_stale_rs_metas(),
-
should_be_visible_but_not_warmed_up);
+ bool should_fallback =
+
std::ranges::any_of(std::views::values(_tablet_meta->all_rs_metas()),
+ should_be_visible_but_not_warmed_up) ||
+
std::ranges::any_of(std::views::values(_tablet_meta->all_stale_rs_metas()),
+ should_be_visible_but_not_warmed_up);
if (should_fallback) {
rlock.unlock();
g_capture_with_freshness_tolerance_fallback_count << 1;
@@ -983,7 +984,7 @@ int64_t CloudTablet::get_cloud_base_compaction_score()
const {
bool has_delete = false;
int64_t point = cumulative_layer_point();
std::shared_lock<std::shared_mutex> rlock(_meta_lock);
- for (const auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
if (rs_meta->start_version() >= point) {
continue;
}
diff --git a/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
index 3ba1339596a..957ce370b8b 100644
--- a/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
+++ b/be/src/exec/schema_scanner/schema_tablets_scanner.cpp
@@ -161,8 +161,8 @@ Status
SchemaTabletsScanner::_fill_block_impl(vectorized::Block* block) {
[&tablet]() {
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
return std::accumulate(rs_metas.begin(), rs_metas.end(), 0,
- [](int64_t val,
RowsetMetaSharedPtr& rs_meta) {
- return val +
rs_meta->num_segments();
+ [](int64_t val, const auto& it) {
+ return val +
it.second->num_segments();
});
}(),
block);
@@ -214,10 +214,9 @@ Status
SchemaTabletsScanner::_fill_block_impl(vectorized::Block* block) {
17,
[&tablet]() {
const auto& rs_metas =
tablet->tablet_meta()->all_rs_metas();
- return std::any_of(rs_metas.begin(), rs_metas.end(),
- [](const RowsetMetaSharedPtr& rs_meta) {
- return
rs_meta->is_segments_overlapping();
- });
+ return std::any_of(rs_metas.begin(), rs_metas.end(),
[](const auto& it) {
+ return it.second->is_segments_overlapping();
+ });
}(),
block);
}
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 28cb02b99b2..b919db8d92b 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -131,11 +131,14 @@ BaseTablet::BaseTablet(TabletMetaSharedPtr tablet_meta) :
_tablet_meta(std::move
// Before doris 1.2 version, rowset metas don't have tablet schema.
// And when upgrade to doris 1.2 version,
// all rowset metas will be set the tablet schmea from tablet meta.
- if (_tablet_meta->all_rs_metas().empty() ||
!_tablet_meta->all_rs_metas()[0]->tablet_schema()) {
+ if (_tablet_meta->all_rs_metas().empty() ||
+ !_tablet_meta->all_rs_metas().begin()->second->tablet_schema()) {
_max_version_schema = _tablet_meta->tablet_schema();
} else {
- _max_version_schema =
-
tablet_schema_with_merged_max_schema_version(_tablet_meta->all_rs_metas());
+ std::vector<RowsetMetaSharedPtr>
rowset_metas(_tablet_meta->all_rs_metas().size());
+ std::transform(_tablet_meta->all_rs_metas().begin(),
_tablet_meta->all_rs_metas().end(),
+ rowset_metas.begin(), [](const auto& it) { return
it.second; });
+ _max_version_schema =
tablet_schema_with_merged_max_schema_version(rowset_metas);
}
DCHECK(_max_version_schema);
g_total_tablet_num << 1;
@@ -149,8 +152,7 @@ BaseTablet::~BaseTablet() {
TabletSchemaSPtr BaseTablet::tablet_schema_with_merged_max_schema_version(
const std::vector<RowsetMetaSharedPtr>& rowset_metas) {
RowsetMetaSharedPtr max_schema_version_rs = *std::max_element(
- rowset_metas.begin(), rowset_metas.end(),
- [](const RowsetMetaSharedPtr& a, const RowsetMetaSharedPtr& b) {
+ rowset_metas.begin(), rowset_metas.end(), [](const auto& a, const
auto& b) -> bool {
return !a->tablet_schema()
? true
: (!b->tablet_schema()
@@ -182,10 +184,9 @@ void BaseTablet::update_max_version_schema(const
TabletSchemaSPtr& tablet_schema
uint32_t BaseTablet::get_real_compaction_score() const {
std::shared_lock l(_meta_lock);
const auto& rs_metas = _tablet_meta->all_rs_metas();
- return std::accumulate(rs_metas.begin(), rs_metas.end(), 0,
- [](uint32_t score, const RowsetMetaSharedPtr&
rs_meta) {
- return score + rs_meta->get_compaction_score();
- });
+ return std::accumulate(rs_metas.begin(), rs_metas.end(), 0, [](uint32_t
score, const auto& it) {
+ return score + it.second->get_compaction_score();
+ });
}
Status BaseTablet::capture_rs_readers_unlocked(const Versions& version_path,
@@ -291,8 +292,8 @@ Versions BaseTablet::get_missed_versions(int64_t
spec_version) const {
Versions existing_versions;
{
std::shared_lock rdlock(_meta_lock);
- for (const auto& rs : _tablet_meta->all_rs_metas()) {
- existing_versions.emplace_back(rs->version());
+ for (const auto& [ver, _] : _tablet_meta->all_rs_metas()) {
+ existing_versions.emplace_back(ver);
}
}
return calc_missed_versions(spec_version, std::move(existing_versions));
@@ -302,8 +303,8 @@ Versions BaseTablet::get_missed_versions_unlocked(int64_t
spec_version) const {
DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
Versions existing_versions;
- for (const auto& rs : _tablet_meta->all_rs_metas()) {
- existing_versions.emplace_back(rs->version());
+ for (const auto& [ver, _] : _tablet_meta->all_rs_metas()) {
+ existing_versions.emplace_back(ver);
}
return calc_missed_versions(spec_version, std::move(existing_versions));
}
diff --git a/be/src/olap/cumulative_compaction_policy.cpp
b/be/src/olap/cumulative_compaction_policy.cpp
index b7666ef1e20..6f74f86bf9d 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -43,8 +43,8 @@
SizeBasedCumulativeCompactionPolicy::SizeBasedCumulativeCompactionPolicy(
_compaction_min_size(compaction_min_size) {}
void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point(
- Tablet* tablet, const std::vector<RowsetMetaSharedPtr>& all_metas,
- int64_t current_cumulative_point, int64_t* ret_cumulative_point) {
+ Tablet* tablet, const RowsetMetaMapContainer& all_metas, int64_t
current_cumulative_point,
+ int64_t* ret_cumulative_point) {
*ret_cumulative_point = Tablet::K_INVALID_CUMULATIVE_POINT;
if (current_cumulative_point != Tablet::K_INVALID_CUMULATIVE_POINT) {
// only calculate the point once.
@@ -57,7 +57,7 @@ void
SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point(
}
std::list<RowsetMetaSharedPtr> existing_rss;
- for (auto& rs : all_metas) {
+ for (const auto& [_, rs] : all_metas) {
existing_rss.emplace_back(rs);
}
@@ -182,7 +182,7 @@ uint32_t
SizeBasedCumulativeCompactionPolicy::calc_cumulative_compaction_score(T
// NOTE: tablet._meta_lock is hold
auto& rs_metas = tablet->tablet_meta()->all_rs_metas();
// check the base rowset and collect the rowsets of cumulative part
- for (auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
if (rs_meta->start_version() < first_version) {
first_version = rs_meta->start_version();
first_meta = rs_meta;
diff --git a/be/src/olap/cumulative_compaction_policy.h
b/be/src/olap/cumulative_compaction_policy.h
index c7dea00562e..1da0689f4aa 100644
--- a/be/src/olap/cumulative_compaction_policy.h
+++ b/be/src/olap/cumulative_compaction_policy.h
@@ -92,7 +92,7 @@ public:
/// param current_cumulative_point, current cumulative position
/// return cumulative_point, the result of calculating cumulative point
position
virtual void calculate_cumulative_point(Tablet* tablet,
- const
std::vector<RowsetMetaSharedPtr>& all_rowsets,
+ const RowsetMetaMapContainer&
all_rowsets,
int64_t current_cumulative_point,
int64_t* cumulative_point) = 0;
@@ -128,8 +128,7 @@ public:
/// SizeBased cumulative compaction policy implements calculate cumulative
point function.
/// When the first time the tablet does compact, this calculation is
executed. Its main policy is to find first rowset
/// which does not satisfied the promotion conditions.
- void calculate_cumulative_point(Tablet* tablet,
- const std::vector<RowsetMetaSharedPtr>&
all_rowsets,
+ void calculate_cumulative_point(Tablet* tablet, const
RowsetMetaMapContainer& all_rowsets,
int64_t current_cumulative_point,
int64_t* cumulative_point) override;
diff --git a/be/src/olap/cumulative_compaction_time_series_policy.cpp
b/be/src/olap/cumulative_compaction_time_series_policy.cpp
index 8cb7f0f9eaf..ef4872837dc 100644
--- a/be/src/olap/cumulative_compaction_time_series_policy.cpp
+++ b/be/src/olap/cumulative_compaction_time_series_policy.cpp
@@ -58,7 +58,7 @@ uint32_t
TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
// NOTE: tablet._meta_lock is hold
auto& rs_metas = tablet->tablet_meta()->all_rs_metas();
// check the base rowset and collect the rowsets of cumulative part
- for (auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
int64_t start_version = rs_meta->start_version();
int64_t end_version = rs_meta->end_version();
if (start_version < first_version) {
@@ -169,8 +169,8 @@ uint32_t
TimeSeriesCumulativeCompactionPolicy::calc_cumulative_compaction_score(
}
void TimeSeriesCumulativeCompactionPolicy::calculate_cumulative_point(
- Tablet* tablet, const std::vector<RowsetMetaSharedPtr>& all_metas,
- int64_t current_cumulative_point, int64_t* ret_cumulative_point) {
+ Tablet* tablet, const RowsetMetaMapContainer& all_metas, int64_t
current_cumulative_point,
+ int64_t* ret_cumulative_point) {
*ret_cumulative_point = Tablet::K_INVALID_CUMULATIVE_POINT;
if (current_cumulative_point != Tablet::K_INVALID_CUMULATIVE_POINT) {
// only calculate the point once.
@@ -183,7 +183,7 @@ void
TimeSeriesCumulativeCompactionPolicy::calculate_cumulative_point(
}
std::list<RowsetMetaSharedPtr> existing_rss;
- for (auto& rs : all_metas) {
+ for (const auto& [_, rs] : all_metas) {
existing_rss.emplace_back(rs);
}
diff --git a/be/src/olap/cumulative_compaction_time_series_policy.h
b/be/src/olap/cumulative_compaction_time_series_policy.h
index a18cbd85ffb..c71faeb2782 100644
--- a/be/src/olap/cumulative_compaction_time_series_policy.h
+++ b/be/src/olap/cumulative_compaction_time_series_policy.h
@@ -42,8 +42,7 @@ public:
/// When the first time the tablet does compact, this calculation is
executed. Its main policy is to find first rowset
/// which does not satisfied the _compaction_goal_size * 0.8.
/// The result of compaction may be slightly smaller than the
_compaction_goal_size.
- void calculate_cumulative_point(Tablet* tablet,
- const std::vector<RowsetMetaSharedPtr>&
all_rowsets,
+ void calculate_cumulative_point(Tablet* tablet, const
RowsetMetaMapContainer& all_rowsets,
int64_t current_cumulative_point,
int64_t* cumulative_point) override;
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index c5e7edcf6f3..8e3c4a9d915 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -615,9 +615,9 @@ Status DataDir::load() {
if (!tablet) {
return true;
}
- const std::vector<RowsetMetaSharedPtr>& all_rowsets =
tablet->tablet_meta()->all_rs_metas();
+ const auto& all_rowsets = tablet->tablet_meta()->all_rs_metas();
RowsetIdUnorderedSet rowset_ids;
- for (auto& rowset_meta : all_rowsets) {
+ for (const auto& [_, rowset_meta] : all_rowsets) {
rowset_ids.insert(rowset_meta->rowset_id());
}
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 836652d183c..906cffc9a42 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -1411,7 +1411,7 @@ void StorageEngine::do_remove_unused_remote_files() {
UniqueId cooldown_meta_id;
{
std::shared_lock rlock(t->get_header_lock());
- for (auto&& rs_meta : t->tablet_meta()->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : t->tablet_meta()->all_rs_metas()) {
if (!rs_meta->is_local()) {
cooldowned_rowsets.insert(rs_meta->rowset_id());
}
diff --git a/be/src/olap/rowset/rowset_meta.h b/be/src/olap/rowset/rowset_meta.h
index 6d536d5c2a2..4cd346ce414 100644
--- a/be/src/olap/rowset/rowset_meta.h
+++ b/be/src/olap/rowset/rowset_meta.h
@@ -446,8 +446,9 @@ private:
std::atomic<int64_t> _stale_at_s {0};
};
-#include "common/compile_check_end.h"
+using RowsetMetaMapContainer = std::unordered_map<Version,
RowsetMetaSharedPtr, HashOfVersion>;
+#include "common/compile_check_end.h"
} // namespace doris
#endif // DORIS_BE_SRC_OLAP_ROWSET_ROWSET_META_H
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index ff71371f1bf..69791626ab8 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -273,7 +273,7 @@ Tablet::Tablet(StorageEngine& engine, TabletMetaSharedPtr
tablet_meta, DataDir*
bool Tablet::set_tablet_schema_into_rowset_meta() {
bool flag = false;
- for (auto&& rowset_meta : _tablet_meta->all_mutable_rs_metas()) {
+ for (auto& [_, rowset_meta] : _tablet_meta->all_mutable_rs_metas()) {
if (!rowset_meta->tablet_schema()) {
rowset_meta->set_tablet_schema(_tablet_meta->tablet_schema());
flag = true;
@@ -294,7 +294,7 @@ Status Tablet::_init_once_action() {
_tablet_meta->compaction_policy());
#endif
- for (const auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
Version version = rs_meta->version();
RowsetSharedPtr rowset;
res = create_rowset(rs_meta, &rowset);
@@ -309,7 +309,7 @@ Status Tablet::_init_once_action() {
// init stale rowset
int64_t now = ::time(nullptr);
- for (const auto& stale_rs_meta : _tablet_meta->all_stale_rs_metas()) {
+ for (const auto& [_, stale_rs_meta] : _tablet_meta->all_stale_rs_metas()) {
Version version = stale_rs_meta->version();
if (!stale_rs_meta->has_stale_at()) {
@@ -1108,7 +1108,7 @@ uint32_t Tablet::calc_cold_data_compaction_score() const {
int64_t max_delete_version = 0;
{
std::shared_lock rlock(_meta_lock);
- for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
if (!rs_meta->is_local()) {
cooldowned_rowsets.push_back(rs_meta);
if (rs_meta->has_delete_predicate() &&
@@ -1152,7 +1152,7 @@ uint32_t Tablet::_calc_base_compaction_score() const {
const int64_t point = cumulative_layer_point();
bool base_rowset_exist = false;
bool has_delete = false;
- for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
if (rs_meta->start_version() == 0) {
base_rowset_exist = true;
}
@@ -1186,8 +1186,8 @@ void
Tablet::_max_continuous_version_from_beginning_unlocked(Version* version, V
bool*
has_version_cross) const {
std::vector<Version> existing_versions;
*has_version_cross = false;
- for (auto& rs : _tablet_meta->all_rs_metas()) {
- existing_versions.emplace_back(rs->version());
+ for (const auto& [ver, _] : _tablet_meta->all_rs_metas()) {
+ existing_versions.emplace_back(ver);
}
// sort the existing versions in ascending order
@@ -1557,7 +1557,7 @@ bool Tablet::do_tablet_meta_checkpoint() {
save_meta();
// if save meta successfully, then should remove the rowset meta existing
in tablet
// meta from rowset meta store
- for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
// If we delete it from rowset manager's meta explicitly in previous
checkpoint, just skip.
if (rs_meta->is_remove_from_rowset_meta()) {
continue;
@@ -1573,7 +1573,7 @@ bool Tablet::do_tablet_meta_checkpoint() {
}
// check _stale_rs_version_map to remove meta from rowset meta store
- for (auto& rs_meta : _tablet_meta->all_stale_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_stale_rs_metas()) {
// If we delete it from rowset manager's meta explicitly in previous
checkpoint, just skip.
if (rs_meta->is_remove_from_rowset_meta()) {
continue;
@@ -2262,7 +2262,7 @@ Status Tablet::write_cooldown_meta() {
UniqueId cooldown_meta_id;
{
std::shared_lock meta_rlock(_meta_lock);
- for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
if (!rs_meta->is_local()) {
cooldowned_rs_metas.push_back(rs_meta);
}
@@ -2558,7 +2558,7 @@ void Tablet::record_unused_remote_rowset(const RowsetId&
rowset_id, const std::s
Status Tablet::remove_all_remote_rowsets() {
DCHECK(tablet_state() == TABLET_SHUTDOWN);
std::set<std::string> resource_ids;
- for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
if (!rs_meta->is_local()) {
resource_ids.insert(rs_meta->resource_id());
}
@@ -2831,8 +2831,7 @@ void Tablet::check_table_size_correctness() {
if (!config::enable_table_size_correctness_check) {
return;
}
- const std::vector<RowsetMetaSharedPtr>& all_rs_metas =
_tablet_meta->all_rs_metas();
- for (const auto& rs_meta : all_rs_metas) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
int64_t total_segment_size = get_segment_file_size(rs_meta);
int64_t total_inverted_index_size =
get_inverted_index_file_size(rs_meta);
if (rs_meta->data_disk_size() != total_segment_size ||
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index 28139a64f20..1b7e809eec3 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -735,7 +735,7 @@ inline Version Tablet::max_version() const {
inline uint64_t Tablet::segment_count() const {
std::shared_lock rdlock(_meta_lock);
uint64_t segment_nums = 0;
- for (const auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : _tablet_meta->all_rs_metas()) {
segment_nums += rs_meta->num_segments();
}
return segment_nums;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index d150351cd92..60585a45359 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -751,7 +751,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
for (auto& it : tablet_meta_pb.rs_metas()) {
RowsetMetaSharedPtr rs_meta(new RowsetMeta());
rs_meta->init_from_pb(it);
- _rs_metas.push_back(std::move(rs_meta));
+ _rs_metas.emplace(rs_meta->version(), rs_meta);
}
// For mow table, delete bitmap of stale rowsets has not been persisted.
@@ -761,7 +761,7 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
for (auto& it : tablet_meta_pb.stale_rs_metas()) {
RowsetMetaSharedPtr rs_meta(new RowsetMeta());
rs_meta->init_from_pb(it);
- _stale_rs_metas.push_back(std::move(rs_meta));
+ _stale_rs_metas.emplace(rs_meta->version(), rs_meta);
}
}
@@ -848,10 +848,10 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb)
{
// RowsetMetaPB is separated from TabletMetaPB
if (!config::is_cloud_mode()) {
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_rs_metas());
}
- for (auto rs : _stale_rs_metas) {
+ for (const auto& [_, rs] : _stale_rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas());
}
}
@@ -875,7 +875,7 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
if (_enable_unique_key_merge_on_write) {
std::set<RowsetId> stale_rs_ids;
- for (const auto& rowset : _stale_rs_metas) {
+ for (const auto& [_, rowset] : _stale_rs_metas) {
stale_rs_ids.insert(rowset->rowset_id());
}
DeleteBitmapPB* delete_bitmap_pb =
tablet_meta_pb->mutable_delete_bitmap();
@@ -916,7 +916,7 @@ void TabletMeta::to_json(string* json_string,
json2pb::Pb2JsonOptions& options)
Version TabletMeta::max_version() const {
Version max_version = {-1, 0};
- for (auto& rs_meta : _rs_metas) {
+ for (const auto& [_, rs_meta] : _rs_metas) {
if (rs_meta->end_version() > max_version.second) {
max_version = rs_meta->version();
}
@@ -926,7 +926,7 @@ Version TabletMeta::max_version() const {
size_t TabletMeta::version_count_cross_with_range(const Version& range) const {
size_t count = 0;
- for (const auto& rs_meta : _rs_metas) {
+ for (const auto& [_, rs_meta] : _rs_metas) {
if (!(range.first > rs_meta->version().second || range.second <
rs_meta->version().first)) {
count++;
}
@@ -936,7 +936,7 @@ size_t TabletMeta::version_count_cross_with_range(const
Version& range) const {
Status TabletMeta::add_rs_meta(const RowsetMetaSharedPtr& rs_meta) {
// check RowsetMeta is valid
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (rs->version() == rs_meta->version()) {
if (rs->rowset_id() != rs_meta->rowset_id()) {
return Status::Error<PUSH_VERSION_ALREADY_EXIST>(
@@ -948,34 +948,29 @@ Status TabletMeta::add_rs_meta(const RowsetMetaSharedPtr&
rs_meta) {
}
}
}
- _rs_metas.push_back(rs_meta);
+ _rs_metas.emplace(rs_meta->version(), rs_meta);
return Status::OK();
}
void TabletMeta::add_rowsets_unchecked(const std::vector<RowsetSharedPtr>&
to_add) {
for (const auto& rs : to_add) {
- _rs_metas.push_back(rs->rowset_meta());
+ _rs_metas.emplace(rs->rowset_meta()->version(), rs->rowset_meta());
}
}
void TabletMeta::delete_rs_meta_by_version(const Version& version,
std::vector<RowsetMetaSharedPtr>*
deleted_rs_metas) {
size_t rowset_cache_version_size = 0;
- auto it = _rs_metas.begin();
- while (it != _rs_metas.end()) {
- if ((*it)->version() == version) {
- if (deleted_rs_metas != nullptr) {
- deleted_rs_metas->push_back(*it);
- }
- _rs_metas.erase(it);
- if (_enable_unique_key_merge_on_write) {
- rowset_cache_version_size =
-
_delete_bitmap->remove_rowset_cache_version((*it)->rowset_id());
- }
- return;
- } else {
- ++it;
+ if (auto it = _rs_metas.find(version); it != _rs_metas.end()) {
+ if (deleted_rs_metas != nullptr) {
+ deleted_rs_metas->push_back(it->second);
}
+ auto rowset_id = it->second->rowset_id();
+ _rs_metas.erase(it);
+ if (_enable_unique_key_merge_on_write) {
+ rowset_cache_version_size =
_delete_bitmap->remove_rowset_cache_version(rowset_id);
+ }
+ return;
}
_check_mow_rowset_cache_version_size(rowset_cache_version_size);
}
@@ -986,28 +981,25 @@ void TabletMeta::modify_rs_metas(const
std::vector<RowsetMetaSharedPtr>& to_add,
size_t rowset_cache_version_size = 0;
// Remove to_delete rowsets from _rs_metas
for (auto rs_to_del : to_delete) {
- auto it = _rs_metas.begin();
- while (it != _rs_metas.end()) {
- if (rs_to_del->version() == (*it)->version()) {
- _rs_metas.erase(it);
- if (_enable_unique_key_merge_on_write) {
- rowset_cache_version_size =
-
_delete_bitmap->remove_rowset_cache_version((*it)->rowset_id());
- }
- // there should be only one rowset match the version
- break;
- } else {
- ++it;
+ if (auto it = _rs_metas.find(rs_to_del->version()); it !=
_rs_metas.end()) {
+ auto rowset_id = it->second->rowset_id();
+ _rs_metas.erase(it);
+ if (_enable_unique_key_merge_on_write) {
+ rowset_cache_version_size =
_delete_bitmap->remove_rowset_cache_version(rowset_id);
}
}
}
if (!same_version) {
// put to_delete rowsets in _stale_rs_metas.
- _stale_rs_metas.insert(_stale_rs_metas.end(), to_delete.begin(),
to_delete.end());
+ for (auto rs_to_del : to_delete) {
+ _stale_rs_metas.emplace(rs_to_del->version(), rs_to_del);
+ }
}
// put to_add rowsets in _rs_metas.
- _rs_metas.insert(_rs_metas.end(), to_add.begin(), to_add.end());
+ for (auto rs_to_add : to_add) {
+ _rs_metas.emplace(rs_to_add->version(), rs_to_add);
+ }
_check_mow_rowset_cache_version_size(rowset_cache_version_size);
}
@@ -1018,7 +1010,10 @@ void TabletMeta::modify_rs_metas(const
std::vector<RowsetMetaSharedPtr>& to_add,
void TabletMeta::revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas) {
{
std::lock_guard<std::shared_mutex> wrlock(_meta_lock);
- _rs_metas = std::move(rs_metas);
+ _rs_metas.clear();
+ for (auto& rs_meta : rs_metas) {
+ _rs_metas.emplace(rs_meta->version(), rs_meta);
+ }
_stale_rs_metas.clear();
}
if (_enable_unique_key_merge_on_write) {
@@ -1033,13 +1028,13 @@ void
TabletMeta::revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas) {
// TabletMeta's _meta_lock
void TabletMeta::revise_delete_bitmap_unlocked(const DeleteBitmap&
delete_bitmap) {
_delete_bitmap = std::make_unique<DeleteBitmap>(tablet_id());
- for (auto rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
DeleteBitmap rs_bm(tablet_id());
delete_bitmap.subset({rs->rowset_id(), 0, 0}, {rs->rowset_id(),
UINT32_MAX, INT64_MAX},
&rs_bm);
_delete_bitmap->merge(rs_bm);
}
- for (auto rs : _stale_rs_metas) {
+ for (const auto& [_, rs] : _stale_rs_metas) {
DeleteBitmap rs_bm(tablet_id());
delete_bitmap.subset({rs->rowset_id(), 0, 0}, {rs->rowset_id(),
UINT32_MAX, INT64_MAX},
&rs_bm);
@@ -1048,30 +1043,19 @@ void TabletMeta::revise_delete_bitmap_unlocked(const
DeleteBitmap& delete_bitmap
}
void TabletMeta::delete_stale_rs_meta_by_version(const Version& version) {
- auto it = _stale_rs_metas.begin();
- while (it != _stale_rs_metas.end()) {
- if ((*it)->version() == version) {
- it = _stale_rs_metas.erase(it);
- } else {
- it++;
- }
- }
+ _stale_rs_metas.erase(version);
}
RowsetMetaSharedPtr TabletMeta::acquire_rs_meta_by_version(const Version&
version) const {
- for (auto it : _rs_metas) {
- if (it->version() == version) {
- return it;
- }
+ if (auto it = _rs_metas.find(version); it != _rs_metas.end()) {
+ return it->second;
}
return nullptr;
}
RowsetMetaSharedPtr TabletMeta::acquire_stale_rs_meta_by_version(const
Version& version) const {
- for (auto it : _stale_rs_metas) {
- if (it->version() == version) {
- return it;
- }
+ if (auto it = _stale_rs_metas.find(version); it != _stale_rs_metas.end()) {
+ return it->second;
}
return nullptr;
}
@@ -1107,10 +1091,10 @@ void
TabletMeta::_check_mow_rowset_cache_version_size(size_t rowset_cache_versio
std::set<std::string> tablet_rowset_ids;
{
std::shared_lock rlock(_meta_lock);
- for (auto& rs_meta : _rs_metas) {
+ for (const auto& [_, rs_meta] : _rs_metas) {
tablet_rowset_ids.emplace(rs_meta->rowset_id().to_string());
}
- for (auto& rs_meta : _stale_rs_metas) {
+ for (const auto& [_, rs_meta] : _stale_rs_metas) {
tablet_rowset_ids.emplace(rs_meta->rowset_id().to_string());
}
}
@@ -1143,10 +1127,7 @@ bool operator==(const TabletMeta& a, const TabletMeta&
b) {
if (a._tablet_type != b._tablet_type) return false;
if (a._tablet_state != b._tablet_state) return false;
if (*a._schema != *b._schema) return false;
- if (a._rs_metas.size() != b._rs_metas.size()) return false;
- for (int i = 0; i < a._rs_metas.size(); ++i) {
- if (a._rs_metas[i] != b._rs_metas[i]) return false;
- }
+ if (a._rs_metas != b._rs_metas) return false;
if (a._in_restore_mode != b._in_restore_mode) return false;
if (a._preferred_rowset_type != b._preferred_rowset_type) return false;
if (a._storage_policy_id != b._storage_policy_id) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index d858d577bc3..a8f35520782 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -193,8 +193,8 @@ public:
TabletSchema* mutable_tablet_schema();
- const std::vector<RowsetMetaSharedPtr>& all_rs_metas() const;
- std::vector<RowsetMetaSharedPtr>& all_mutable_rs_metas();
+ const RowsetMetaMapContainer& all_rs_metas() const;
+ RowsetMetaMapContainer& all_mutable_rs_metas();
Status add_rs_meta(const RowsetMetaSharedPtr& rs_meta);
void delete_rs_meta_by_version(const Version& version,
std::vector<RowsetMetaSharedPtr>*
deleted_rs_metas);
@@ -206,7 +206,7 @@ public:
void revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas);
void revise_delete_bitmap_unlocked(const DeleteBitmap& delete_bitmap);
- const std::vector<RowsetMetaSharedPtr>& all_stale_rs_metas() const;
+ const RowsetMetaMapContainer& all_stale_rs_metas() const;
RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version)
const;
void delete_stale_rs_meta_by_version(const Version& version);
RowsetMetaSharedPtr acquire_stale_rs_meta_by_version(const Version&
version) const;
@@ -336,11 +336,11 @@ private:
TabletSchemaSPtr _schema;
Cache::Handle* _handle = nullptr;
- std::vector<RowsetMetaSharedPtr> _rs_metas;
+ RowsetMetaMapContainer _rs_metas;
// This variable _stale_rs_metas is used to record these rowsets‘ meta
which are be compacted.
// These stale rowsets meta are been removed when rowsets' pathVersion is
expired,
// this policy is judged and computed by TimestampedVersionTracker.
- std::vector<RowsetMetaSharedPtr> _stale_rs_metas;
+ RowsetMetaMapContainer _stale_rs_metas;
bool _in_restore_mode = false;
RowsetTypePB _preferred_rowset_type = BETA_ROWSET;
@@ -670,7 +670,7 @@ inline void TabletMeta::set_cumulative_layer_point(int64_t
new_point) {
inline size_t TabletMeta::num_rows() const {
size_t num_rows = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
num_rows += rs->num_rows();
}
return num_rows;
@@ -678,7 +678,7 @@ inline size_t TabletMeta::num_rows() const {
inline size_t TabletMeta::tablet_footprint() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
total_size += rs->total_disk_size();
}
return total_size;
@@ -686,7 +686,7 @@ inline size_t TabletMeta::tablet_footprint() const {
inline size_t TabletMeta::tablet_local_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (rs->is_local()) {
total_size += rs->total_disk_size();
}
@@ -696,7 +696,7 @@ inline size_t TabletMeta::tablet_local_size() const {
inline size_t TabletMeta::tablet_remote_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (!rs->is_local()) {
total_size += rs->total_disk_size();
}
@@ -706,7 +706,7 @@ inline size_t TabletMeta::tablet_remote_size() const {
inline size_t TabletMeta::tablet_local_index_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (rs->is_local()) {
total_size += rs->index_disk_size();
}
@@ -716,7 +716,7 @@ inline size_t TabletMeta::tablet_local_index_size() const {
inline size_t TabletMeta::tablet_local_segment_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (rs->is_local()) {
total_size += rs->data_disk_size();
}
@@ -726,7 +726,7 @@ inline size_t TabletMeta::tablet_local_segment_size() const
{
inline size_t TabletMeta::tablet_remote_index_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (!rs->is_local()) {
total_size += rs->index_disk_size();
}
@@ -736,7 +736,7 @@ inline size_t TabletMeta::tablet_remote_index_size() const {
inline size_t TabletMeta::tablet_remote_segment_size() const {
size_t total_size = 0;
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (!rs->is_local()) {
total_size += rs->data_disk_size();
}
@@ -776,25 +776,25 @@ inline TabletSchema* TabletMeta::mutable_tablet_schema() {
return _schema.get();
}
-inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_rs_metas()
const {
+inline const RowsetMetaMapContainer& TabletMeta::all_rs_metas() const {
return _rs_metas;
}
-inline std::vector<RowsetMetaSharedPtr>& TabletMeta::all_mutable_rs_metas() {
+inline RowsetMetaMapContainer& TabletMeta::all_mutable_rs_metas() {
return _rs_metas;
}
-inline const std::vector<RowsetMetaSharedPtr>&
TabletMeta::all_stale_rs_metas() const {
+inline const RowsetMetaMapContainer& TabletMeta::all_stale_rs_metas() const {
return _stale_rs_metas;
}
inline bool TabletMeta::all_beta() const {
- for (auto& rs : _rs_metas) {
+ for (const auto& [_, rs] : _rs_metas) {
if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
return false;
}
}
- for (auto& rs : _stale_rs_metas) {
+ for (const auto& [_, rs] : _stale_rs_metas) {
if (rs->rowset_type() != RowsetTypePB::BETA_ROWSET) {
return false;
}
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index db0ad79f51f..a3d6c5acb66 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -960,7 +960,7 @@ Status EngineCloneTask::_finish_full_clone(Tablet* tablet,
}
to_add.reserve(cloned_tablet_meta->all_rs_metas().size());
- for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : cloned_tablet_meta->all_rs_metas()) {
RowsetSharedPtr rs;
RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs));
to_add.push_back(std::move(rs));
diff --git a/be/src/olap/version_graph.cpp b/be/src/olap/version_graph.cpp
index 1894e0953d3..8c10dfe2ee8 100644
--- a/be/src/olap/version_graph.cpp
+++ b/be/src/olap/version_graph.cpp
@@ -27,6 +27,7 @@
#include <memory>
#include <optional>
#include <ostream>
+#include <ranges>
#include <utility>
#include "common/logging.h"
@@ -36,146 +37,14 @@ namespace doris {
using namespace ErrorCode;
-void TimestampedVersionTracker::_construct_versioned_tracker(
- const std::vector<RowsetMetaSharedPtr>& rs_metas) {
- int64_t max_version = 0;
-
- // construct the rowset graph
- _version_graph.reconstruct_version_graph(rs_metas, &max_version);
-}
-
void TimestampedVersionTracker::construct_versioned_tracker(
- const std::vector<RowsetMetaSharedPtr>& rs_metas) {
- if (rs_metas.empty()) {
- VLOG_NOTICE << "there is no version in the header.";
- return;
- }
- _stale_version_path_map.clear();
- _next_path_id = 1;
- _construct_versioned_tracker(rs_metas);
+ const RowsetMetaMapContainer& rs_metas) {
+ construct_versioned_tracker(std::views::values(rs_metas));
}
void TimestampedVersionTracker::construct_versioned_tracker(
- const std::vector<RowsetMetaSharedPtr>& rs_metas,
- const std::vector<RowsetMetaSharedPtr>& stale_metas) {
- if (rs_metas.empty()) {
- VLOG_NOTICE << "there is no version in the header.";
- return;
- }
- _stale_version_path_map.clear();
- _next_path_id = 1;
- _construct_versioned_tracker(rs_metas);
-
- // Init `_stale_version_path_map`.
- _init_stale_version_path_map(rs_metas, stale_metas);
-}
-
-void TimestampedVersionTracker::_init_stale_version_path_map(
- const std::vector<RowsetMetaSharedPtr>& rs_metas,
- const std::vector<RowsetMetaSharedPtr>& stale_metas) {
- if (stale_metas.empty()) {
- return;
- }
-
- // Sort stale meta by version diff (second version - first version).
- std::list<RowsetMetaSharedPtr> sorted_stale_metas;
- for (auto& rs : stale_metas) {
- sorted_stale_metas.emplace_back(rs);
- }
-
- // 1. sort the existing rowsets by version in ascending order.
- sorted_stale_metas.sort([](const RowsetMetaSharedPtr& a, const
RowsetMetaSharedPtr& b) {
- // Compare by version diff between `version.first` and
`version.second`.
- int64_t a_diff = a->version().second - a->version().first;
- int64_t b_diff = b->version().second - b->version().first;
-
- int64_t diff = a_diff - b_diff;
- if (diff < 0) {
- return true;
- } else if (diff > 0) {
- return false;
- }
- // When the version diff is equal, compare the rowset`s stale time
- return a->stale_at() < b->stale_at();
- });
-
- // first_version -> (second_version -> rowset_meta)
- std::unordered_map<int64_t, std::unordered_map<int64_t,
RowsetMetaSharedPtr>> stale_map;
-
- // 2. generate stale path from stale_metas. traverse sorted_stale_metas
and each time add stale_meta to stale_map.
- // when a stale path in stale_map can replace stale_meta in
sorted_stale_metas, stale_map remove rowset_metas of a stale path
- // and add the path to `_stale_version_path_map`.
- for (auto& stale_meta : sorted_stale_metas) {
- std::vector<RowsetMetaSharedPtr> stale_path;
- // 2.1 find a path in `stale_map` can replace current `stale_meta`
version.
- bool r = _find_path_from_stale_map(stale_map,
stale_meta->start_version(),
- stale_meta->end_version(),
&stale_path);
-
- // 2.2 add version to `version_graph`.
- Version stale_meta_version = stale_meta->version();
- add_version(stale_meta_version);
-
- // 2.3 find the path.
- if (r) {
- // Add the path to `_stale_version_path_map`.
- add_stale_path_version(stale_path);
- // Remove `stale_path` from `stale_map`.
- for (auto stale_item : stale_path) {
-
stale_map[stale_item->start_version()].erase(stale_item->end_version());
-
- if (stale_map[stale_item->start_version()].empty()) {
- stale_map.erase(stale_item->start_version());
- }
- }
- }
-
- // 2.4 add `stale_meta` to `stale_map`.
- auto start_iter = stale_map.find(stale_meta->start_version());
- if (start_iter != stale_map.end()) {
- start_iter->second[stale_meta->end_version()] = stale_meta;
- } else {
- std::unordered_map<int64_t, RowsetMetaSharedPtr> item;
- item[stale_meta->end_version()] = stale_meta;
- stale_map[stale_meta->start_version()] = std::move(item);
- }
- }
-
- // 3. generate stale path from `rs_metas`.
- for (auto& stale_meta : rs_metas) {
- std::vector<RowsetMetaSharedPtr> stale_path;
- // 3.1 find a path in stale_map can replace current `stale_meta`
version.
- bool r = _find_path_from_stale_map(stale_map,
stale_meta->start_version(),
- stale_meta->end_version(),
&stale_path);
-
- // 3.2 find the path.
- if (r) {
- // Add the path to `_stale_version_path_map`.
- add_stale_path_version(stale_path);
- // Remove `stale_path` from `stale_map`.
- for (auto stale_item : stale_path) {
-
stale_map[stale_item->start_version()].erase(stale_item->end_version());
-
- if (stale_map[stale_item->start_version()].empty()) {
- stale_map.erase(stale_item->start_version());
- }
- }
- }
- }
-
- // 4. process remain stale `rowset_meta` in `stale_map`.
- auto map_iter = stale_map.begin();
- while (map_iter != stale_map.end()) {
- auto second_iter = map_iter->second.begin();
- while (second_iter != map_iter->second.end()) {
- // Each remain stale `rowset_meta` generate a stale path.
- std::vector<RowsetMetaSharedPtr> stale_path;
- stale_path.push_back(second_iter->second);
- add_stale_path_version(stale_path);
-
- second_iter++;
- }
- map_iter++;
- }
+ const RowsetMetaMapContainer& rs_metas, const RowsetMetaMapContainer&
stale_metas) {
+ construct_versioned_tracker(std::views::values(rs_metas),
std::views::values(stale_metas));
}
bool TimestampedVersionTracker::_find_path_from_stale_map(
@@ -451,64 +320,14 @@ std::vector<TimestampedVersionSharedPtr>&
TimestampedVersionPathContainer::times
return _timestamped_versions_container;
}
-void VersionGraph::construct_version_graph(const
std::vector<RowsetMetaSharedPtr>& rs_metas,
+void VersionGraph::construct_version_graph(const RowsetMetaMapContainer&
rs_metas,
int64_t* max_version) {
- if (rs_metas.empty()) {
- VLOG_NOTICE << "there is no version in the header.";
- return;
- }
-
- // Distill vertex values from versions in TabletMeta.
- std::vector<int64_t> vertex_values;
- vertex_values.reserve(2 * rs_metas.size());
-
- for (size_t i = 0; i < rs_metas.size(); ++i) {
- vertex_values.push_back(rs_metas[i]->start_version());
- vertex_values.push_back(rs_metas[i]->end_version() + 1);
- if (max_version != nullptr and *max_version <
rs_metas[i]->end_version()) {
- *max_version = rs_metas[i]->end_version();
- }
- }
- std::sort(vertex_values.begin(), vertex_values.end());
-
- // Items in `vertex_values` are sorted, but not unique.
- // we choose unique items in `vertex_values` to create vertexes.
- int64_t last_vertex_value = -1;
- for (size_t i = 0; i < vertex_values.size(); ++i) {
- if (i != 0 && vertex_values[i] == last_vertex_value) {
- continue;
- }
-
- // Add vertex to graph.
- _add_vertex_to_graph(vertex_values[i]);
- last_vertex_value = vertex_values[i];
- }
- // Create edges for version graph according to TabletMeta's versions.
- for (size_t i = 0; i < rs_metas.size(); ++i) {
- // Versions in header are unique.
- // We ensure `_vertex_index_map` has its `start_version`.
- int64_t start_vertex_index =
_vertex_index_map[rs_metas[i]->start_version()];
- int64_t end_vertex_index =
_vertex_index_map[rs_metas[i]->end_version() + 1];
- // Add one edge from `start_version` to `end_version`.
- _version_graph[start_vertex_index].edges.push_front(end_vertex_index);
- // Add reverse edge from `end_version` to `start_version`.
- _version_graph[end_vertex_index].edges.push_front(start_vertex_index);
- }
-
- // Sort edges by version in descending order.
- for (auto& vertex : _version_graph) {
- vertex.edges.sort([this](const int& vertex_idx_a, const int&
vertex_idx_b) {
- return _version_graph[vertex_idx_a].value >
_version_graph[vertex_idx_b].value;
- });
- }
+ construct_version_graph(std::views::values(rs_metas), max_version);
}
-void VersionGraph::reconstruct_version_graph(const
std::vector<RowsetMetaSharedPtr>& rs_metas,
+void VersionGraph::reconstruct_version_graph(const RowsetMetaMapContainer&
rs_metas,
int64_t* max_version) {
- _version_graph.clear();
- _vertex_index_map.clear();
-
- construct_version_graph(rs_metas, max_version);
+ reconstruct_version_graph(std::views::values(rs_metas), max_version);
}
void VersionGraph::add_version_to_graph(const Version& version) {
diff --git a/be/src/olap/version_graph.h b/be/src/olap/version_graph.h
index 4c65d920861..a61a7446add 100644
--- a/be/src/olap/version_graph.h
+++ b/be/src/olap/version_graph.h
@@ -32,6 +32,10 @@
#include "olap/rowset/rowset_meta.h"
namespace doris {
+
+template <typename R>
+concept RowsetMetaSPtrRange = std::same_as<std::ranges::range_value_t<R>,
RowsetMetaSharedPtr> &&
+ std::ranges::sized_range<R> &&
std::ranges::forward_range<R>;
/// VersionGraph class which is implemented to build and maintain total
versions of rowsets.
/// This class use adjacency-matrix represent rowsets version and links. A
vertex is a version
/// and a link is the _version object of a rowset (from start version to end
version + 1).
@@ -41,11 +45,11 @@ class VersionGraph {
public:
/// Use rs_metas to construct the graph including vertex and edges, and
return the
/// max_version in metas.
- void construct_version_graph(const std::vector<RowsetMetaSharedPtr>&
rs_metas,
- int64_t* max_version);
+ void construct_version_graph(RowsetMetaSPtrRange auto&& rs_metas, int64_t*
max_version);
+ void construct_version_graph(const RowsetMetaMapContainer& rs_metas,
int64_t* max_version);
/// Reconstruct the graph, begin construction the vertex vec and edges
list will be cleared.
- void reconstruct_version_graph(const std::vector<RowsetMetaSharedPtr>&
rs_metas,
- int64_t* max_version);
+ void reconstruct_version_graph(RowsetMetaSPtrRange auto&&, int64_t*
max_version);
+ void reconstruct_version_graph(const RowsetMetaMapContainer& rs_metas,
int64_t* max_version);
/// Add a version to this graph, graph will add the version and edge in
version.
void add_version_to_graph(const Version& version);
/// Delete a version from graph. Notice that this del operation only
remove this edges and
@@ -171,11 +175,14 @@ using PathVersionListSharedPtr =
std::shared_ptr<TimestampedVersionPathContainer
class TimestampedVersionTracker {
public:
/// Construct rowsets version tracker by main path rowset meta.
- void construct_versioned_tracker(const std::vector<RowsetMetaSharedPtr>&
rs_metas);
+ void construct_versioned_tracker(RowsetMetaSPtrRange auto&& rs_metas);
+ void construct_versioned_tracker(const RowsetMetaMapContainer& rs_metas);
/// Construct rowsets version tracker by main path rowset meta and stale
rowset meta.
- void construct_versioned_tracker(const std::vector<RowsetMetaSharedPtr>&
rs_metas,
- const std::vector<RowsetMetaSharedPtr>&
stale_metas);
+ void construct_versioned_tracker(RowsetMetaSPtrRange auto&& rs_metas,
+ RowsetMetaSPtrRange auto&& stale_metas);
+ void construct_versioned_tracker(const RowsetMetaMapContainer& rs_metas,
+ const RowsetMetaMapContainer&
stale_metas);
/// Recover rowsets version tracker from stale version path map. When
delete operation fails, the
/// tracker can be recovered from deleted stale_version_path_map.
@@ -258,11 +265,11 @@ public:
private:
/// Construct rowsets version tracker with main path rowset meta.
- void _construct_versioned_tracker(const std::vector<RowsetMetaSharedPtr>&
rs_metas);
+ void _construct_versioned_tracker(RowsetMetaSPtrRange auto&& rs_metas);
/// init stale_version_path_map by main path rowset meta and stale rowset
meta.
- void _init_stale_version_path_map(const std::vector<RowsetMetaSharedPtr>&
rs_metas,
- const std::vector<RowsetMetaSharedPtr>&
stale_metas);
+ void _init_stale_version_path_map(RowsetMetaSPtrRange auto&& rs_metas,
+ RowsetMetaSPtrRange auto&& stale_metas);
/// find a path in stale_map from first_version to second_version,
stale_path is used as result.
bool _find_path_from_stale_map(
@@ -283,4 +290,202 @@ private:
VersionGraph _version_graph;
};
+void VersionGraph::reconstruct_version_graph(RowsetMetaSPtrRange auto&&
rs_metas,
+ int64_t* max_version) {
+ _version_graph.clear();
+ _vertex_index_map.clear();
+
+ construct_version_graph(rs_metas, max_version);
+}
+
+void VersionGraph::construct_version_graph(RowsetMetaSPtrRange auto&& rs_metas,
+ int64_t* max_version) {
+ if (std::ranges::empty(rs_metas)) {
+ VLOG_NOTICE << "there is no version in the header.";
+ return;
+ }
+
+ // Distill vertex values from versions in TabletMeta.
+ std::vector<int64_t> vertex_values;
+ vertex_values.reserve(2 * std::ranges::size(rs_metas));
+
+ for (const auto& rs : rs_metas) {
+ vertex_values.push_back(rs->start_version());
+ vertex_values.push_back(rs->end_version() + 1);
+ if (max_version != nullptr and *max_version < rs->end_version()) {
+ *max_version = rs->end_version();
+ }
+ }
+ std::sort(vertex_values.begin(), vertex_values.end());
+
+ // Items in `vertex_values` are sorted, but not unique.
+ // we choose unique items in `vertex_values` to create vertexes.
+ int64_t last_vertex_value = -1;
+ for (size_t i = 0; i < vertex_values.size(); ++i) {
+ if (i != 0 && vertex_values[i] == last_vertex_value) {
+ continue;
+ }
+
+ // Add vertex to graph.
+ _add_vertex_to_graph(vertex_values[i]);
+ last_vertex_value = vertex_values[i];
+ }
+ // Create edges for version graph according to TabletMeta's versions.
+ for (const auto& rs : rs_metas) {
+ // Versions in header are unique.
+ // We ensure `_vertex_index_map` has its `start_version`.
+ int64_t start_vertex_index = _vertex_index_map[rs->start_version()];
+ int64_t end_vertex_index = _vertex_index_map[rs->end_version() + 1];
+ // Add one edge from `start_version` to `end_version`.
+ _version_graph[start_vertex_index].edges.push_front(end_vertex_index);
+ // Add reverse edge from `end_version` to `start_version`.
+ _version_graph[end_vertex_index].edges.push_front(start_vertex_index);
+ }
+
+ // Sort edges by version in descending order.
+ for (auto& vertex : _version_graph) {
+ vertex.edges.sort([this](const int& vertex_idx_a, const int&
vertex_idx_b) {
+ return _version_graph[vertex_idx_a].value >
_version_graph[vertex_idx_b].value;
+ });
+ }
+}
+
+void
TimestampedVersionTracker::_construct_versioned_tracker(RowsetMetaSPtrRange
auto&& rs_metas) {
+ int64_t max_version = 0;
+
+ // construct the rowset graph
+ _version_graph.reconstruct_version_graph(rs_metas, &max_version);
+}
+
+void
TimestampedVersionTracker::construct_versioned_tracker(RowsetMetaSPtrRange
auto&& rs_metas) {
+ if (std::ranges::empty(rs_metas)) {
+ VLOG_NOTICE << "there is no version in the header.";
+ return;
+ }
+ _stale_version_path_map.clear();
+ _next_path_id = 1;
+ _construct_versioned_tracker(rs_metas);
+}
+
+void TimestampedVersionTracker::construct_versioned_tracker(
+ RowsetMetaSPtrRange auto&& rs_metas, RowsetMetaSPtrRange auto&&
stale_metas) {
+ if (std::ranges::empty(rs_metas)) {
+ VLOG_NOTICE << "there is no version in the header.";
+ return;
+ }
+ _stale_version_path_map.clear();
+ _next_path_id = 1;
+ _construct_versioned_tracker(rs_metas);
+
+ // Init `_stale_version_path_map`.
+ _init_stale_version_path_map(rs_metas, stale_metas);
+}
+
+void TimestampedVersionTracker::_init_stale_version_path_map(
+ RowsetMetaSPtrRange auto&& rs_metas, RowsetMetaSPtrRange auto&&
stale_metas) {
+ if (std::ranges::empty(stale_metas)) {
+ return;
+ }
+
+ // Sort stale meta by version diff (second version - first version).
+ std::list<RowsetMetaSharedPtr> sorted_stale_metas;
+ for (const auto& rs : stale_metas) {
+ sorted_stale_metas.emplace_back(rs);
+ }
+
+ // 1. sort the existing rowsets by version in ascending order.
+ sorted_stale_metas.sort([](const RowsetMetaSharedPtr& a, const
RowsetMetaSharedPtr& b) {
+ // Compare by version diff between `version.first` and
`version.second`.
+ int64_t a_diff = a->version().second - a->version().first;
+ int64_t b_diff = b->version().second - b->version().first;
+
+ int64_t diff = a_diff - b_diff;
+ if (diff < 0) {
+ return true;
+ } else if (diff > 0) {
+ return false;
+ }
+ // When the version diff is equal, compare the rowset`s stale time
+ return a->stale_at() < b->stale_at();
+ });
+
+ // first_version -> (second_version -> rowset_meta)
+ std::unordered_map<int64_t, std::unordered_map<int64_t,
RowsetMetaSharedPtr>> stale_map;
+
+ // 2. generate stale path from stale_metas. traverse sorted_stale_metas
and each time add stale_meta to stale_map.
+ // when a stale path in stale_map can replace stale_meta in
sorted_stale_metas, stale_map remove rowset_metas of a stale path
+ // and add the path to `_stale_version_path_map`.
+ for (auto& stale_meta : sorted_stale_metas) {
+ std::vector<RowsetMetaSharedPtr> stale_path;
+ // 2.1 find a path in `stale_map` can replace current `stale_meta`
version.
+ bool r = _find_path_from_stale_map(stale_map,
stale_meta->start_version(),
+ stale_meta->end_version(),
&stale_path);
+
+ // 2.2 add version to `version_graph`.
+ Version stale_meta_version = stale_meta->version();
+ add_version(stale_meta_version);
+
+ // 2.3 find the path.
+ if (r) {
+ // Add the path to `_stale_version_path_map`.
+ add_stale_path_version(stale_path);
+ // Remove `stale_path` from `stale_map`.
+ for (auto stale_item : stale_path) {
+
stale_map[stale_item->start_version()].erase(stale_item->end_version());
+
+ if (stale_map[stale_item->start_version()].empty()) {
+ stale_map.erase(stale_item->start_version());
+ }
+ }
+ }
+
+ // 2.4 add `stale_meta` to `stale_map`.
+ auto start_iter = stale_map.find(stale_meta->start_version());
+ if (start_iter != stale_map.end()) {
+ start_iter->second[stale_meta->end_version()] = stale_meta;
+ } else {
+ std::unordered_map<int64_t, RowsetMetaSharedPtr> item;
+ item[stale_meta->end_version()] = stale_meta;
+ stale_map[stale_meta->start_version()] = std::move(item);
+ }
+ }
+
+ // 3. generate stale path from `rs_metas`.
+ for (const auto& stale_meta : rs_metas) {
+ std::vector<RowsetMetaSharedPtr> stale_path;
+ // 3.1 find a path in stale_map can replace current `stale_meta`
version.
+ bool r = _find_path_from_stale_map(stale_map,
stale_meta->start_version(),
+ stale_meta->end_version(),
&stale_path);
+
+ // 3.2 find the path.
+ if (r) {
+ // Add the path to `_stale_version_path_map`.
+ add_stale_path_version(stale_path);
+ // Remove `stale_path` from `stale_map`.
+ for (auto stale_item : stale_path) {
+
stale_map[stale_item->start_version()].erase(stale_item->end_version());
+
+ if (stale_map[stale_item->start_version()].empty()) {
+ stale_map.erase(stale_item->start_version());
+ }
+ }
+ }
+ }
+
+ // 4. process remain stale `rowset_meta` in `stale_map`.
+ auto map_iter = stale_map.begin();
+ while (map_iter != stale_map.end()) {
+ auto second_iter = map_iter->second.begin();
+ while (second_iter != map_iter->second.end()) {
+ // Each remain stale `rowset_meta` generate a stale path.
+ std::vector<RowsetMetaSharedPtr> stale_path;
+ stale_path.push_back(second_iter->second);
+ add_stale_path_version(stale_path);
+
+ second_iter++;
+ }
+ map_iter++;
+ }
+}
+
} // namespace doris
diff --git a/be/test/cloud/cloud_meta_mgr_test.cpp
b/be/test/cloud/cloud_meta_mgr_test.cpp
index 39e21105d2b..df3df147959 100644
--- a/be/test/cloud/cloud_meta_mgr_test.cpp
+++ b/be/test/cloud/cloud_meta_mgr_test.cpp
@@ -145,7 +145,7 @@ TEST_F(CloudMetaMgrTest, test_fill_version_holes_no_holes) {
// Verify tablet still has the same number of rowsets (no holes to fill)
EXPECT_EQ(tablet->tablet_meta()->all_rs_metas().size(), 5);
// Verify rows number is correct
- for (const auto& rs_meta : tablet->tablet_meta()->all_rs_metas()) {
+ for (const auto& [_, rs_meta] : tablet->tablet_meta()->all_rs_metas()) {
EXPECT_EQ(rs_meta->num_rows(), 100);
}
}
@@ -203,7 +203,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_with_holes) {
// Verify all versions are present
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
std::set<int64_t> found_versions;
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
found_versions.insert(rs_meta->version().first);
}
EXPECT_EQ(found_versions.size(), 5);
@@ -214,7 +214,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_with_holes) {
EXPECT_TRUE(found_versions.contains(4));
// Verify the hole rowsets (versions 1 and 3) are empty
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
if (rs_meta->version().first == 1 || rs_meta->version().first == 3) {
EXPECT_TRUE(rs_meta->empty());
EXPECT_EQ(rs_meta->num_rows(), 0);
@@ -379,7 +379,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_trailing_holes) {
// Verify all versions are present
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
std::set<int64_t> found_versions;
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
found_versions.insert(rs_meta->version().first);
}
EXPECT_EQ(found_versions.size(), 6);
@@ -388,7 +388,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_trailing_holes) {
}
// Verify the trailing hole rowsets (versions 3, 4, 5) are empty
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
if (rs_meta->version().first >= 3) {
EXPECT_TRUE(rs_meta->empty())
<< "Version " << rs_meta->version().first << " should be
empty";
@@ -454,7 +454,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_single_hole) {
// Verify all versions are present
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
std::set<int64_t> found_versions;
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
found_versions.insert(rs_meta->version().first);
}
EXPECT_EQ(found_versions.size(), 3);
@@ -463,7 +463,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_single_hole) {
EXPECT_TRUE(found_versions.contains(2));
// Verify the hole rowset (version 1) is empty
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
if (rs_meta->version().first == 1) {
EXPECT_TRUE(rs_meta->empty());
EXPECT_EQ(rs_meta->num_rows(), 0);
@@ -527,7 +527,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_multiple_consecutive_holes) {
// Verify all versions are present
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
std::set<int64_t> found_versions;
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
found_versions.insert(rs_meta->version().first);
}
EXPECT_EQ(found_versions.size(), 6);
@@ -536,7 +536,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_multiple_consecutive_holes) {
}
// Verify the hole rowsets (versions 1, 2, 3, 4) are empty
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
if (rs_meta->version().first >= 1 && rs_meta->version().first <= 4) {
EXPECT_TRUE(rs_meta->empty())
<< "Version " << rs_meta->version().first << " should be
empty";
@@ -602,7 +602,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_mixed_holes) {
// Verify all versions are present
auto rs_metas = tablet->tablet_meta()->all_rs_metas();
std::set<int64_t> found_versions;
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
found_versions.insert(rs_meta->version().first);
}
EXPECT_EQ(found_versions.size(), 9);
@@ -613,7 +613,7 @@ TEST_F(CloudMetaMgrTest,
test_fill_version_holes_mixed_holes) {
// Verify the hole rowsets (versions 1, 3, 4, 7, 8) are empty
std::set<int64_t> original_versions = {0, 2, 5, 6};
std::set<int64_t> hole_versions = {1, 3, 4, 7, 8};
- for (const auto& rs_meta : rs_metas) {
+ for (const auto& [_, rs_meta] : rs_metas) {
int64_t version = rs_meta->version().first;
if (hole_versions.contains(version)) {
EXPECT_TRUE(rs_meta->empty()) << "Version " << version << " should
be empty";
diff --git a/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
b/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
index f20609a6d03..7f6c79595be 100644
--- a/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
+++ b/be/test/olap/compaction_delete_bitmap_calculator_test.cpp
@@ -240,7 +240,7 @@ TEST_F(CompactionDeleteBitmapCalculatorTest, test) {
// publish rowset 1
DeleteBitmapPtr delete_bitmap =
std::make_shared<DeleteBitmap>(_tablet->tablet_id());
RowsetIdUnorderedSet set;
- set.insert(_tablet_meta->all_rs_metas()[0]->rowset_id());
+ set.insert(_rowset1->rowset_id());
engine_ref->txn_manager()->set_txn_related_delete_bitmap(partition_id, 1,
_tablet->tablet_id(),
_tablet->tablet_uid(), true,
delete_bitmap,
set, nullptr);
@@ -254,7 +254,7 @@ TEST_F(CompactionDeleteBitmapCalculatorTest, test) {
status = engine_ref->txn_manager()->commit_txn(_meta, partition_id, 2,
tablet_id, _tablet_uid,
load_id, _rowset2, {},
false);
EXPECT_TRUE(status == Status::OK());
- set.insert(_tablet_meta->all_rs_metas()[1]->rowset_id());
+ set.insert(_rowset2->rowset_id());
engine_ref->txn_manager()->set_txn_related_delete_bitmap(
partition_id, 2, tablet_id, _tablet_uid, true, delete_bitmap, set,
nullptr);
diff --git a/be/test/olap/delete_handler_test.cpp
b/be/test/olap/delete_handler_test.cpp
index d615be62bc4..c9d94a36b61 100644
--- a/be/test/olap/delete_handler_test.cpp
+++ b/be/test/olap/delete_handler_test.cpp
@@ -1037,7 +1037,7 @@ protected:
std::vector<RowsetMetaSharedPtr> get_delete_predicates() {
std::vector<RowsetMetaSharedPtr> delete_preds;
- for (auto&& rs_meta : tablet->tablet_meta()->_rs_metas) {
+ for (auto&& [_, rs_meta] : tablet->tablet_meta()->_rs_metas) {
if (rs_meta->has_delete_predicate()) {
delete_preds.push_back(rs_meta);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
