This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 178f4cc83e3 branch-4.1: [fix](compaction) Keep latest 10 versions
unmerged during schema change #66337 (#67589)
178f4cc83e3 is described below
commit 178f4cc83e3a1a79ed21a9414d63789916e0b373
Author: Jamie <[email protected]>
AuthorDate: Thu Sep 10 07:43:48 2026 +0800
branch-4.1: [fix](compaction) Keep latest 10 versions unmerged during
schema change #66337 (#67589)
### What problem does this PR solve?
Issue Number: None
Related PR: #66337
Problem Summary: Backport #66337 to `branch-4.1`.
While a schema change target tablet is `TABLET_NOTREADY`, cumulative
compaction should merge only older rowsets and leave the newest 10
versions unmerged. The filter used the inverse comparison, so a
compaction output could cross the base tablet's maximum version and
prevent incremental schema-change conversion with
`VERSION_ALREADY_MERGED`.
This change reverses the comparison in both local and Cloud size-based
cumulative compaction policies. It also adds focused unit-test coverage
adapted to the branch-4.1 test layout.
### Release note
Fix schema changes that could fail when cumulative compaction merged the
latest versions on the new tablet.
### Check List (For Author)
- Test
- [ ] Regression test
- [x] Unit Test
- Focused local and Cloud unit tests are included; hosted CI is
requested with `run buildall`.
- `build-support/check-format.sh`: passed
- `git diff --check`: passed
- [ ] Manual test
- [ ] No need to test or manual test
- Behavior changed: Yes. `TABLET_NOTREADY` cumulative compaction now
leaves the latest 10 versions unmerged.
- Does this need documentation? No
---------
Co-authored-by: Siyang Tang <[email protected]>
---
.../cloud/cloud_cumulative_compaction_policy.cpp | 2 +-
.../compaction/cumulative_compaction_policy.cpp | 2 +-
.../cloud_cumulative_compaction_policy_test.cpp | 34 ++++++++++++++++
.../cumulative_compaction_policy_test.cpp | 47 ++++++++++++++++++++++
4 files changed, 83 insertions(+), 2 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
index 53d336e5c70..0f7371307c2 100644
--- a/be/src/cloud/cloud_cumulative_compaction_policy.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction_policy.cpp
@@ -160,7 +160,7 @@ int64_t
CloudSizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
if (tablet->tablet_state() == TABLET_NOTREADY) {
// If tablet under alter, keep latest 10 version so that base
tablet max version
// not merged in new tablet, and then we can copy data from base
tablet
- if (rowset->version().second < max_version - 10) {
+ if (rowset->version().second > max_version - 10) {
continue;
}
}
diff --git a/be/src/storage/compaction/cumulative_compaction_policy.cpp
b/be/src/storage/compaction/cumulative_compaction_policy.cpp
index 5326d25bd5f..ade1fe46acb 100644
--- a/be/src/storage/compaction/cumulative_compaction_policy.cpp
+++ b/be/src/storage/compaction/cumulative_compaction_policy.cpp
@@ -304,7 +304,7 @@ int SizeBasedCumulativeCompactionPolicy::pick_input_rowsets(
if (tablet->tablet_state() == TABLET_NOTREADY) {
// If tablet under alter, keep latest 10 version so that base
tablet max version
// not merged in new tablet, and then we can copy data from base
tablet
- if (rowset->version().second < max_version - 10) {
+ if (rowset->version().second > max_version - 10) {
continue;
}
}
diff --git a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
index 29358eaaee0..6e9fa1e1aaa 100644
--- a/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
+++ b/be/test/cloud/cloud_cumulative_compaction_policy_test.cpp
@@ -151,6 +151,40 @@ TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
new_cumulative_point) {
EXPECT_EQ(policy.new_cumulative_point(&_tablet, output_rowset, version,
2), 6);
}
+TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
+ pick_input_rowsets_notready_keeps_latest_versions) {
+ auto base_rowset = create_rowset(Version(0, 1), 1, false, 1024 * 1024 *
1024);
+ ASSERT_NE(nullptr, base_rowset);
+ ASSERT_TRUE(_tablet_meta->add_rs_meta(base_rowset->rowset_meta()).ok());
+
+ std::vector<RowsetSharedPtr> candidate_rowsets;
+ for (int64_t version = 2; version <= 20; ++version) {
+ auto rowset = create_rowset(Version(version, version), 1, true, 1024 *
1024);
+ ASSERT_NE(nullptr, rowset);
+ ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset->rowset_meta()).ok());
+ candidate_rowsets.push_back(rowset);
+ }
+
+ CloudTablet tablet(_engine, _tablet_meta);
+ ASSERT_TRUE(tablet.set_tablet_state(TABLET_NOTREADY).ok());
+ tablet._base_size = 1024L * 1024 * 1024;
+
+ std::vector<RowsetSharedPtr> input_rowsets;
+ Version last_delete_version {-1, -1};
+ size_t compaction_score = 0;
+
+ CloudSizeBasedCumulativeCompactionPolicy policy;
+ auto picked_size = policy.pick_input_rowsets(&tablet, candidate_rowsets,
100, 5, &input_rowsets,
+ &last_delete_version,
&compaction_score, true);
+
+ EXPECT_EQ(9, picked_size);
+ ASSERT_EQ(9, input_rowsets.size());
+ EXPECT_EQ(9, compaction_score);
+ EXPECT_EQ(2, input_rowsets.front()->start_version());
+ EXPECT_EQ(10, input_rowsets.back()->end_version());
+ EXPECT_EQ(Version(-1, -1), last_delete_version);
+}
+
// Test case: Empty rowset compaction with skip_trim
TEST_F(TestCloudSizeBasedCumulativeCompactionPolicy,
pick_input_rowsets_empty_rowset_compaction) {
// Save original config values
diff --git a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
index 81a2d1705fa..6bff25b9ff1 100644
--- a/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
+++ b/be/test/storage/compaction/cumulative_compaction_policy_test.cpp
@@ -476,6 +476,53 @@ TEST_F(TestSizeBasedCumulativeCompactionPolicy,
pick_input_rowsets_normal) {
EXPECT_EQ(-1, last_delete_version.second);
}
+TEST_F(TestSizeBasedCumulativeCompactionPolicy,
pick_input_rowsets_notready_keeps_latest_versions) {
+ std::vector<RowsetMetaSharedPtr> rs_metas;
+
+ RowsetMetaSharedPtr base_rowset(new RowsetMeta());
+ init_rs_meta(base_rowset, 0, 1);
+ base_rowset->set_total_disk_size(1024L * 1024 * 1024);
+ base_rowset->set_segments_overlap(NONOVERLAPPING);
+ rs_metas.push_back(base_rowset);
+
+ for (int64_t version = 2; version <= 20; ++version) {
+ RowsetMetaSharedPtr rowset(new RowsetMeta());
+ init_rs_meta(rowset, version, version);
+ rowset->set_total_disk_size(1 * 1024 * 1024);
+ rowset->set_num_segments(1);
+ rowset->set_segments_overlap(OVERLAPPING);
+ rs_metas.push_back(rowset);
+ }
+
+ for (const auto& rowset : rs_metas) {
+ ASSERT_TRUE(_tablet_meta->add_rs_meta(rowset).ok());
+ }
+
+ TabletSharedPtr tablet(
+ new Tablet(_engine, _tablet_meta, nullptr,
CUMULATIVE_SIZE_BASED_POLICY));
+ ASSERT_TRUE(tablet->init().ok());
+ ASSERT_TRUE(tablet->set_tablet_state(TABLET_NOTREADY).ok());
+ tablet->calculate_cumulative_point();
+
+ EXPECT_EQ(2, tablet->cumulative_layer_point());
+ auto candidate_rowsets =
tablet->pick_candidate_rowsets_to_cumulative_compaction();
+ ASSERT_EQ(19, candidate_rowsets.size());
+
+ std::vector<RowsetSharedPtr> input_rowsets;
+ Version last_delete_version {-1, -1};
+ size_t compaction_score = 0;
+ auto picked_size =
tablet->_cumulative_compaction_policy->pick_input_rowsets(
+ tablet.get(), candidate_rowsets, 100, 5, &input_rowsets,
&last_delete_version,
+ &compaction_score, config::enable_delete_when_cumu_compaction);
+
+ EXPECT_EQ(9, picked_size);
+ ASSERT_EQ(9, input_rowsets.size());
+ EXPECT_EQ(9, compaction_score);
+ EXPECT_EQ(2, input_rowsets.front()->start_version());
+ EXPECT_EQ(10, input_rowsets.back()->end_version());
+ EXPECT_EQ(Version(-1, -1), last_delete_version);
+}
+
TEST_F(TestSizeBasedCumulativeCompactionPolicy, pick_input_rowsets_big_base) {
std::vector<RowsetMetaSharedPtr> rs_metas;
init_rs_meta_big_base(&rs_metas);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
