This is an automated email from the ASF dual-hosted git repository.
luwei16 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 85420c68187 [fix](compaction) Use row count for cumulative task
classification (#66706)
85420c68187 is described below
commit 85420c6818748faf1d9faa2712fbb50ad2ca988f
Author: dzr171712 <[email protected]>
AuthorDate: Wed Aug 19 15:10:21 2026 +0800
[fix](compaction) Use row count for cumulative task classification (#66706)
Problem Summary:
`CompactionMixin::calc_input_rowsets_row_num()` calculated input rowset
disk size instead of row count. As a result, cumulative compaction task
classification compared bytes with
`large_cumu_compaction_task_row_num_threshold`, which could incorrectly
treat compactions with small row count but disk size over the threshold
value as large tasks.
This PR fixes the calculation to sum `rowset->num_rows()` and adds a BE
unit test to verify row-count calculation is independent from total-size
calculation.
### Release note
None
### Check List (For Author)
- Test
- [ ] Regression test
- [✅] Unit Test
- [ ] Manual test
- [ ] No need to test or manual test.
- Behavior changed:
- [✅] No.
- [ ] Yes.
- Does this need documentation?
- [✅] No.
- [ ] Yes.
---
be/src/storage/compaction/compaction.cpp | 4 +-
.../compaction/cumulative_compaction_test.cpp | 58 ++++++++++++++++++++++
2 files changed, 59 insertions(+), 3 deletions(-)
diff --git a/be/src/storage/compaction/compaction.cpp
b/be/src/storage/compaction/compaction.cpp
index 3e411f1d388..ec735fac7ea 100644
--- a/be/src/storage/compaction/compaction.cpp
+++ b/be/src/storage/compaction/compaction.cpp
@@ -2029,9 +2029,7 @@ int64_t CompactionMixin::calc_input_rowsets_total_size()
const {
int64_t CompactionMixin::calc_input_rowsets_row_num() const {
int64_t input_rowsets_row_num = 0;
for (const auto& rowset : _input_rowsets) {
- const auto& rowset_meta = rowset->rowset_meta();
- auto total_size = rowset_meta->total_disk_size();
- input_rowsets_row_num += total_size;
+ input_rowsets_row_num += rowset->num_rows();
}
return input_rowsets_row_num;
}
diff --git a/be/test/storage/compaction/cumulative_compaction_test.cpp
b/be/test/storage/compaction/cumulative_compaction_test.cpp
index 44f5c10cfaa..93fcb8ba57c 100644
--- a/be/test/storage/compaction/cumulative_compaction_test.cpp
+++ b/be/test/storage/compaction/cumulative_compaction_test.cpp
@@ -66,6 +66,29 @@ static RowsetSharedPtr create_rowset(Version version, int
num_segments, bool ove
return rowset;
}
+class TestableCumulativeCompactionMixin : public CompactionMixin {
+public:
+ TestableCumulativeCompactionMixin(StorageEngine& engine, TabletSharedPtr
tablet)
+ : CompactionMixin(engine, tablet,
"TestableCumulativeCompactionMixin") {}
+
+ void set_input_rowsets(const std::vector<RowsetSharedPtr>& rowsets) {
+ _input_rowsets = rowsets;
+ }
+
+ Status prepare_compact() override { return Status::OK(); }
+
+ Status execute_compact() override { return Status::OK(); }
+
+ ReaderType compaction_type() const override { return
ReaderType::READER_CUMULATIVE_COMPACTION; }
+
+ std::string_view compaction_name() const override { return "testable
cumulative compaction"; }
+
+protected:
+ Status construct_output_rowset_writer(RowsetWriterContext&) override {
return Status::OK(); }
+
+ Status update_delete_bitmap() override { return Status::OK(); }
+};
+
TEST_F(CumulativeCompactionTest, TestConsecutiveVersion) {
EngineOptions options;
StorageEngine storage_engine(options);
@@ -294,4 +317,39 @@ TEST_F(CumulativeCompactionTest, TestShouldDelayLargeTask)
{
EXPECT_EQ(storage_engine._should_delay_large_task(), true);
}
+TEST_F(CumulativeCompactionTest, TestCalcInputRowsetsRowNumUsesRowCount) {
+ EngineOptions options;
+ StorageEngine storage_engine(options);
+
+ TabletMetaSharedPtr tablet_meta;
+ tablet_meta.reset(new TabletMeta(1, 2, 15673, 15674, 4, 5,
TTabletSchema(), 6, {{7, 8}},
+ UniqueId(9, 10),
TTabletType::TABLET_TYPE_DISK,
+ TCompressionType::LZ4F));
+ TabletSharedPtr tablet(
+ new Tablet(storage_engine, tablet_meta, nullptr,
CUMULATIVE_SIZE_BASED_POLICY));
+
+ TestableCumulativeCompactionMixin compaction(storage_engine, tablet);
+
+ std::vector<RowsetSharedPtr> rowsets;
+ auto rowset1 = create_rowset({1, 1}, 1, false, 1024);
+ ASSERT_NE(rowset1, nullptr);
+ rowset1->rowset_meta()->set_num_rows(10);
+ rowsets.push_back(rowset1);
+
+ auto rowset2 = create_rowset({2, 2}, 1, false, 2048);
+ ASSERT_NE(rowset2, nullptr);
+ rowset2->rowset_meta()->set_num_rows(20);
+ rowsets.push_back(rowset2);
+
+ auto rowset3 = create_rowset({3, 3}, 1, false, 4096);
+ ASSERT_NE(rowset3, nullptr);
+ rowset3->rowset_meta()->set_num_rows(30);
+ rowsets.push_back(rowset3);
+
+ compaction.set_input_rowsets(rowsets);
+
+ EXPECT_EQ(compaction.calc_input_rowsets_row_num(), 60);
+ EXPECT_EQ(compaction.calc_input_rowsets_total_size(), 7168);
+}
+
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
