This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 33dce03a4d5 [Enhancement](compaction) enable the compaction producer
to generate multiple compaction tasks in a single run (#45411) (#46164)
33dce03a4d5 is described below
commit 33dce03a4d5fbdb31b4e4834258e23dec7143cab
Author: Luwei <[email protected]>
AuthorDate: Fri Jan 3 17:34:43 2025 +0800
[Enhancement](compaction) enable the compaction producer to generate
multiple compaction tasks in a single run (#45411) (#46164)
pick #45411
---
be/src/common/config.cpp | 2 +
be/src/common/config.h | 2 +
be/src/olap/tablet_manager.cpp | 42 +++++++++++++++++++--
be/test/olap/tablet_mgr_test.cpp | 79 +++++++++++++++++++++++++++++++++++++++-
4 files changed, 120 insertions(+), 5 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 342638786ee..e467558128a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1402,6 +1402,8 @@ DEFINE_Bool(enable_table_size_correctness_check, "false");
DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
+DEFINE_mInt32(compaction_num_per_round, "1");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 5706a38b4b0..25127f222e2 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1488,6 +1488,8 @@ DECLARE_Bool(enable_table_size_correctness_check);
// Enable sleep 5s between delete cumulative compaction.
DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
+DECLARE_mInt32(compaction_num_per_round);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 9c31a1cc83d..99d6e53e0ac 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -742,6 +742,11 @@ void TabletManager::get_tablet_stat(TTabletStatResult*
result) {
result->__set_tablet_stat_list(*local_cache);
}
+struct TabletScore {
+ TabletSharedPtr tablet_ptr;
+ int score;
+};
+
std::vector<TabletSharedPtr> TabletManager::find_best_tablets_to_compaction(
CompactionType compaction_type, DataDir* data_dir,
const std::unordered_set<TabletSharedPtr>&
tablet_submitted_compaction, uint32_t* score,
@@ -755,6 +760,9 @@ std::vector<TabletSharedPtr>
TabletManager::find_best_tablets_to_compaction(
uint32_t single_compact_highest_score = 0;
TabletSharedPtr best_tablet;
TabletSharedPtr best_single_compact_tablet;
+ auto cmp = [](TabletScore left, TabletScore right) { return left.score >
right.score; };
+ std::priority_queue<TabletScore, std::vector<TabletScore>, decltype(cmp)>
top_tablets(cmp);
+
auto handler = [&](const TabletSharedPtr& tablet_ptr) {
if
(tablet_ptr->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
LOG_EVERY_N(INFO, 500) << "Tablet " << tablet_ptr->tablet_id()
@@ -818,10 +826,26 @@ std::vector<TabletSharedPtr>
TabletManager::find_best_tablets_to_compaction(
best_single_compact_tablet = tablet_ptr;
}
- // tablet should do cumu or base compaction
- if (current_compaction_score > highest_score &&
!tablet_ptr->should_fetch_from_peer()) {
- highest_score = current_compaction_score;
- best_tablet = tablet_ptr;
+ if (config::compaction_num_per_round > 1 &&
!tablet_ptr->should_fetch_from_peer()) {
+ TabletScore ts;
+ ts.score = current_compaction_score;
+ ts.tablet_ptr = tablet_ptr;
+ if ((top_tablets.size() >= config::compaction_num_per_round &&
+ current_compaction_score > top_tablets.top().score) ||
+ top_tablets.size() < config::compaction_num_per_round) {
+ top_tablets.push(ts);
+ if (top_tablets.size() > config::compaction_num_per_round) {
+ top_tablets.pop();
+ }
+ if (current_compaction_score > highest_score) {
+ highest_score = current_compaction_score;
+ }
+ }
+ } else {
+ if (current_compaction_score > highest_score &&
!tablet_ptr->should_fetch_from_peer()) {
+ highest_score = current_compaction_score;
+ best_tablet = tablet_ptr;
+ }
}
};
@@ -836,6 +860,16 @@ std::vector<TabletSharedPtr>
TabletManager::find_best_tablets_to_compaction(
picked_tablet.emplace_back(std::move(best_tablet));
}
+ std::vector<TabletSharedPtr> reverse_top_tablets;
+ while (!top_tablets.empty()) {
+ reverse_top_tablets.emplace_back(top_tablets.top().tablet_ptr);
+ top_tablets.pop();
+ }
+
+ for (auto it = reverse_top_tablets.rbegin(); it !=
reverse_top_tablets.rend(); ++it) {
+ picked_tablet.emplace_back(*it);
+ }
+
// pick single compaction tablet needs the highest score
if (best_single_compact_tablet != nullptr && single_compact_highest_score
>= highest_score) {
VLOG_CRITICAL << "Found the best tablet for single compaction. "
diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp
index 02df327fd5f..02a197d43e0 100644
--- a/be/test/olap/tablet_mgr_test.cpp
+++ b/be/test/olap/tablet_mgr_test.cpp
@@ -83,6 +83,7 @@ public:
SAFE_DELETE(_data_dir);
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok());
_tablet_mgr = nullptr;
+ config::compaction_num_per_round = 1;
}
std::unique_ptr<StorageEngine> k_engine;
@@ -463,11 +464,87 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) {
ASSERT_EQ(score, 24);
// drop all tablets
- for (int64_t id = 1; id <= 20; ++id) {
+ for (int64_t id = 1; id <= 21; ++id) {
Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
ASSERT_TRUE(drop_st.ok()) << drop_st;
}
+ {
+ config::compaction_num_per_round = 10;
+ for (int64_t i = 1; i <= 100; ++i) {
+ create_tablet(10000 + i, false, i);
+ }
+
+ compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+ CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set,
&score,
+ cumulative_compaction_policies);
+ ASSERT_EQ(compact_tablets.size(), 10);
+ int index = 0;
+ for (auto t : compact_tablets) {
+ ASSERT_EQ(t->tablet_id(), 10100 - index);
+ ASSERT_EQ(t->calc_compaction_score(), 100 - index);
+ index++;
+ }
+ config::compaction_num_per_round = 1;
+ // drop all tablets
+ for (int64_t id = 10001; id <= 10100; ++id) {
+ Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+ ASSERT_TRUE(drop_st.ok()) << drop_st;
+ }
+ }
+
+ {
+ config::compaction_num_per_round = 10;
+ for (int64_t i = 1; i <= 100; ++i) {
+ create_tablet(20000 + i, false, i);
+ }
+ create_tablet(20102, true, 200);
+
+ compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+ CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set,
&score,
+ cumulative_compaction_policies);
+ ASSERT_EQ(compact_tablets.size(), 11);
+ for (int i = 0; i < 10; ++i) {
+ ASSERT_EQ(compact_tablets[i]->tablet_id(), 20100 - i);
+ ASSERT_EQ(compact_tablets[i]->calc_compaction_score(), 100 - i);
+ }
+ ASSERT_EQ(compact_tablets[10]->tablet_id(), 20102);
+ ASSERT_EQ(compact_tablets[10]->calc_compaction_score(), 200);
+
+ config::compaction_num_per_round = 1;
+ // drop all tablets
+ for (int64_t id = 20001; id <= 20100; ++id) {
+ Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+ ASSERT_TRUE(drop_st.ok()) << drop_st;
+ }
+
+ Status drop_st = _tablet_mgr->drop_tablet(20102, 20102 * 10, false);
+ ASSERT_TRUE(drop_st.ok()) << drop_st;
+ }
+
+ {
+ config::compaction_num_per_round = 10;
+ for (int64_t i = 1; i <= 5; ++i) {
+ create_tablet(30000 + i, false, i + 5);
+ }
+
+ compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+ CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set,
&score,
+ cumulative_compaction_policies);
+ ASSERT_EQ(compact_tablets.size(), 5);
+ for (int i = 0; i < 5; ++i) {
+ ASSERT_EQ(compact_tablets[i]->tablet_id(), 30000 + 5 - i);
+ ASSERT_EQ(compact_tablets[i]->calc_compaction_score(), 10 - i);
+ }
+
+ config::compaction_num_per_round = 1;
+ // drop all tablets
+ for (int64_t id = 30001; id <= 30005; ++id) {
+ Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+ ASSERT_TRUE(drop_st.ok()) << drop_st;
+ }
+ }
+
Status trash_st = _tablet_mgr->start_trash_sweep();
ASSERT_TRUE(trash_st.ok()) << trash_st;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]