This is an automated email from the ASF dual-hosted git repository.

gavinchou pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new 33dce03a4d5 [Enhancement](compaction) enable the compaction producer 
to generate multiple compaction tasks in a single run (#45411) (#46164)
33dce03a4d5 is described below

commit 33dce03a4d5fbdb31b4e4834258e23dec7143cab
Author: Luwei <[email protected]>
AuthorDate: Fri Jan 3 17:34:43 2025 +0800

    [Enhancement](compaction) enable the compaction producer to generate 
multiple compaction tasks in a single run (#45411) (#46164)
    
    pick #45411
---
 be/src/common/config.cpp         |  2 +
 be/src/common/config.h           |  2 +
 be/src/olap/tablet_manager.cpp   | 42 +++++++++++++++++++--
 be/test/olap/tablet_mgr_test.cpp | 79 +++++++++++++++++++++++++++++++++++++++-
 4 files changed, 120 insertions(+), 5 deletions(-)

diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 342638786ee..e467558128a 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1402,6 +1402,8 @@ DEFINE_Bool(enable_table_size_correctness_check, "false");
 DEFINE_Bool(force_regenerate_rowsetid_on_start_error, "false");
 DEFINE_mBool(enable_sleep_between_delete_cumu_compaction, "false");
 
+DEFINE_mInt32(compaction_num_per_round, "1");
+
 // clang-format off
 #ifdef BE_TEST
 // test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 5706a38b4b0..25127f222e2 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1488,6 +1488,8 @@ DECLARE_Bool(enable_table_size_correctness_check);
 // Enable sleep 5s between delete cumulative compaction.
 DECLARE_mBool(enable_sleep_between_delete_cumu_compaction);
 
+DECLARE_mInt32(compaction_num_per_round);
+
 #ifdef BE_TEST
 // test s3
 DECLARE_String(test_s3_resource);
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 9c31a1cc83d..99d6e53e0ac 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -742,6 +742,11 @@ void TabletManager::get_tablet_stat(TTabletStatResult* 
result) {
     result->__set_tablet_stat_list(*local_cache);
 }
 
+struct TabletScore {
+    TabletSharedPtr tablet_ptr;
+    int score;
+};
+
 std::vector<TabletSharedPtr> TabletManager::find_best_tablets_to_compaction(
         CompactionType compaction_type, DataDir* data_dir,
         const std::unordered_set<TabletSharedPtr>& 
tablet_submitted_compaction, uint32_t* score,
@@ -755,6 +760,9 @@ std::vector<TabletSharedPtr> 
TabletManager::find_best_tablets_to_compaction(
     uint32_t single_compact_highest_score = 0;
     TabletSharedPtr best_tablet;
     TabletSharedPtr best_single_compact_tablet;
+    auto cmp = [](TabletScore left, TabletScore right) { return left.score > 
right.score; };
+    std::priority_queue<TabletScore, std::vector<TabletScore>, decltype(cmp)> 
top_tablets(cmp);
+
     auto handler = [&](const TabletSharedPtr& tablet_ptr) {
         if 
(tablet_ptr->tablet_meta()->tablet_schema()->disable_auto_compaction()) {
             LOG_EVERY_N(INFO, 500) << "Tablet " << tablet_ptr->tablet_id()
@@ -818,10 +826,26 @@ std::vector<TabletSharedPtr> 
TabletManager::find_best_tablets_to_compaction(
             best_single_compact_tablet = tablet_ptr;
         }
 
-        // tablet should do cumu or base compaction
-        if (current_compaction_score > highest_score && 
!tablet_ptr->should_fetch_from_peer()) {
-            highest_score = current_compaction_score;
-            best_tablet = tablet_ptr;
+        if (config::compaction_num_per_round > 1 && 
!tablet_ptr->should_fetch_from_peer()) {
+            TabletScore ts;
+            ts.score = current_compaction_score;
+            ts.tablet_ptr = tablet_ptr;
+            if ((top_tablets.size() >= config::compaction_num_per_round &&
+                 current_compaction_score > top_tablets.top().score) ||
+                top_tablets.size() < config::compaction_num_per_round) {
+                top_tablets.push(ts);
+                if (top_tablets.size() > config::compaction_num_per_round) {
+                    top_tablets.pop();
+                }
+                if (current_compaction_score > highest_score) {
+                    highest_score = current_compaction_score;
+                }
+            }
+        } else {
+            if (current_compaction_score > highest_score && 
!tablet_ptr->should_fetch_from_peer()) {
+                highest_score = current_compaction_score;
+                best_tablet = tablet_ptr;
+            }
         }
     };
 
@@ -836,6 +860,16 @@ std::vector<TabletSharedPtr> 
TabletManager::find_best_tablets_to_compaction(
         picked_tablet.emplace_back(std::move(best_tablet));
     }
 
+    std::vector<TabletSharedPtr> reverse_top_tablets;
+    while (!top_tablets.empty()) {
+        reverse_top_tablets.emplace_back(top_tablets.top().tablet_ptr);
+        top_tablets.pop();
+    }
+
+    for (auto it = reverse_top_tablets.rbegin(); it != 
reverse_top_tablets.rend(); ++it) {
+        picked_tablet.emplace_back(*it);
+    }
+
     // pick single compaction tablet needs the highest score
     if (best_single_compact_tablet != nullptr && single_compact_highest_score 
>= highest_score) {
         VLOG_CRITICAL << "Found the best tablet for single compaction. "
diff --git a/be/test/olap/tablet_mgr_test.cpp b/be/test/olap/tablet_mgr_test.cpp
index 02df327fd5f..02a197d43e0 100644
--- a/be/test/olap/tablet_mgr_test.cpp
+++ b/be/test/olap/tablet_mgr_test.cpp
@@ -83,6 +83,7 @@ public:
         SAFE_DELETE(_data_dir);
         
EXPECT_TRUE(io::global_local_filesystem()->delete_directory(_engine_data_path).ok());
         _tablet_mgr = nullptr;
+        config::compaction_num_per_round = 1;
     }
     std::unique_ptr<StorageEngine> k_engine;
 
@@ -463,11 +464,87 @@ TEST_F(TabletMgrTest, FindTabletWithCompact) {
     ASSERT_EQ(score, 24);
 
     // drop all tablets
-    for (int64_t id = 1; id <= 20; ++id) {
+    for (int64_t id = 1; id <= 21; ++id) {
         Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
         ASSERT_TRUE(drop_st.ok()) << drop_st;
     }
 
+    {
+        config::compaction_num_per_round = 10;
+        for (int64_t i = 1; i <= 100; ++i) {
+            create_tablet(10000 + i, false, i);
+        }
+
+        compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+                CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, 
&score,
+                cumulative_compaction_policies);
+        ASSERT_EQ(compact_tablets.size(), 10);
+        int index = 0;
+        for (auto t : compact_tablets) {
+            ASSERT_EQ(t->tablet_id(), 10100 - index);
+            ASSERT_EQ(t->calc_compaction_score(), 100 - index);
+            index++;
+        }
+        config::compaction_num_per_round = 1;
+        // drop all tablets
+        for (int64_t id = 10001; id <= 10100; ++id) {
+            Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+            ASSERT_TRUE(drop_st.ok()) << drop_st;
+        }
+    }
+
+    {
+        config::compaction_num_per_round = 10;
+        for (int64_t i = 1; i <= 100; ++i) {
+            create_tablet(20000 + i, false, i);
+        }
+        create_tablet(20102, true, 200);
+
+        compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+                CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, 
&score,
+                cumulative_compaction_policies);
+        ASSERT_EQ(compact_tablets.size(), 11);
+        for (int i = 0; i < 10; ++i) {
+            ASSERT_EQ(compact_tablets[i]->tablet_id(), 20100 - i);
+            ASSERT_EQ(compact_tablets[i]->calc_compaction_score(), 100 - i);
+        }
+        ASSERT_EQ(compact_tablets[10]->tablet_id(), 20102);
+        ASSERT_EQ(compact_tablets[10]->calc_compaction_score(), 200);
+
+        config::compaction_num_per_round = 1;
+        // drop all tablets
+        for (int64_t id = 20001; id <= 20100; ++id) {
+            Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+            ASSERT_TRUE(drop_st.ok()) << drop_st;
+        }
+
+        Status drop_st = _tablet_mgr->drop_tablet(20102, 20102 * 10, false);
+        ASSERT_TRUE(drop_st.ok()) << drop_st;
+    }
+
+    {
+        config::compaction_num_per_round = 10;
+        for (int64_t i = 1; i <= 5; ++i) {
+            create_tablet(30000 + i, false, i + 5);
+        }
+
+        compact_tablets = _tablet_mgr->find_best_tablets_to_compaction(
+                CompactionType::CUMULATIVE_COMPACTION, _data_dir, cumu_set, 
&score,
+                cumulative_compaction_policies);
+        ASSERT_EQ(compact_tablets.size(), 5);
+        for (int i = 0; i < 5; ++i) {
+            ASSERT_EQ(compact_tablets[i]->tablet_id(), 30000 + 5 - i);
+            ASSERT_EQ(compact_tablets[i]->calc_compaction_score(), 10 - i);
+        }
+
+        config::compaction_num_per_round = 1;
+        // drop all tablets
+        for (int64_t id = 30001; id <= 30005; ++id) {
+            Status drop_st = _tablet_mgr->drop_tablet(id, id * 10, false);
+            ASSERT_TRUE(drop_st.ok()) << drop_st;
+        }
+    }
+
     Status trash_st = _tablet_mgr->start_trash_sweep();
     ASSERT_TRUE(trash_st.ok()) << trash_st;
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to