This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9cc0e9526a5 [enhancement](merge-on-write) consider version count on
size-based cu compaction policy (#25352)
9cc0e9526a5 is described below
commit 9cc0e9526a52a7e9c5e7525ca951579e8d5b654f
Author: zhannngchen <[email protected]>
AuthorDate: Fri Oct 13 14:52:21 2023 +0800
[enhancement](merge-on-write) consider version count on size-based cu
compaction policy (#25352)
---
be/src/common/config.cpp | 6 ++++++
be/src/common/config.h | 6 ++++++
be/src/olap/cumulative_compaction_policy.cpp | 12 +++++++++++-
be/src/olap/cumulative_compaction_policy.h | 3 +++
4 files changed, 26 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 1a7da1471e3..93708312705 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -378,6 +378,12 @@ DEFINE_mDouble(compaction_promotion_ratio, "0.05");
// rowset will be not given to base compaction. The unit is m byte.
DEFINE_mInt64(compaction_promotion_min_size_mbytes, "128");
+// When output rowset of cumulative compaction total version count
(end_version - start_version)
+// exceed this config count, the rowset will be moved to base compaction
+// NOTE: this config will work for unique key merge-on-write table only, to
reduce version count
+// related cost on delete bitmap more effectively.
+DEFINE_mInt64(compaction_promotion_version_count, "1000");
+
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
DEFINE_mInt64(compaction_min_size_mbytes, "64");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 224ba26f67e..466f9919f09 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -431,6 +431,12 @@ DECLARE_mDouble(compaction_promotion_ratio);
// rowset will be not given to base compaction. The unit is m byte.
DECLARE_mInt64(compaction_promotion_min_size_mbytes);
+// When output rowset of cumulative compaction total version count
(end_version - start_version)
+// exceed this config count, the rowset will be moved to base compaction
+// NOTE: this config will work for unique key merge-on-write table only, to
reduce version count
+// related cost on delete bitmap more effectively.
+DECLARE_mInt64(compaction_promotion_version_count);
+
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
DECLARE_mInt64(compaction_min_size_mbytes);
diff --git a/be/src/olap/cumulative_compaction_policy.cpp
b/be/src/olap/cumulative_compaction_policy.cpp
index 193be99c37c..8587b29f512 100644
--- a/be/src/olap/cumulative_compaction_policy.cpp
+++ b/be/src/olap/cumulative_compaction_policy.cpp
@@ -33,10 +33,11 @@ namespace doris {
SizeBasedCumulativeCompactionPolicy::SizeBasedCumulativeCompactionPolicy(
int64_t promotion_size, double promotion_ratio, int64_t
promotion_min_size,
- int64_t compaction_min_size)
+ int64_t promotion_version_count, int64_t compaction_min_size)
: _promotion_size(promotion_size),
_promotion_ratio(promotion_ratio),
_promotion_min_size(promotion_min_size),
+ _promotion_version_count(promotion_version_count),
_compaction_min_size(compaction_min_size) {}
void SizeBasedCumulativeCompactionPolicy::calculate_cumulative_point(
@@ -152,6 +153,15 @@ void
SizeBasedCumulativeCompactionPolicy::update_cumulative_point(
size_t total_size = output_rowset->rowset_meta()->total_disk_size();
if (total_size >= tablet->cumulative_promotion_size()) {
tablet->set_cumulative_layer_point(output_rowset->end_version() +
1);
+ } else if (tablet->enable_unique_key_merge_on_write() &&
+ output_rowset->end_version() -
output_rowset->start_version() >
+ _promotion_version_count) {
+ // for MoW table, if there's too many versions, the delete bitmap
will grow to
+ // a very big size, which may cause the tablet meta too big and
the `save_meta`
+ // operation too slow.
+ // if the rowset should not promotion according to it's disk size,
we should also
+ // consider it's version count here.
+ tablet->set_cumulative_layer_point(output_rowset->end_version() +
1);
}
}
}
diff --git a/be/src/olap/cumulative_compaction_policy.h
b/be/src/olap/cumulative_compaction_policy.h
index e5608685a54..b9b3bb46c0f 100644
--- a/be/src/olap/cumulative_compaction_policy.h
+++ b/be/src/olap/cumulative_compaction_policy.h
@@ -114,6 +114,7 @@ public:
int64_t promotion_size = config::compaction_promotion_size_mbytes
* 1024 * 1024,
double promotion_ratio = config::compaction_promotion_ratio,
int64_t promotion_min_size =
config::compaction_promotion_min_size_mbytes * 1024 * 1024,
+ int64_t promotion_version_count =
config::compaction_promotion_version_count,
int64_t compaction_min_size = config::compaction_min_size_mbytes *
1024 * 1024);
/// Destructor function of SizeBasedCumulativeCompactionPolicy.
@@ -169,6 +170,8 @@ private:
double _promotion_ratio;
/// cumulative compaction promotion min size, unit is byte.
int64_t _promotion_min_size;
+ // cululative compaction promotion version count, only works for unique
key MoW table
+ int64_t _promotion_version_count;
/// lower bound size to do compaction compaction.
int64_t _compaction_min_size;
};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
