dataroaring commented on code in PR #13495:
URL: https://github.com/apache/doris/pull/13495#discussion_r1000659159
##########
docs/en/docs/admin-manual/config/be-config.md:
##########
@@ -117,13 +117,7 @@ Default: true
Whether to enable vectorized compaction
-### `base_compaction_interval_seconds_since_last_operation`
-
-Default: 86400
-
-One of the triggering conditions of BaseCompaction: the interval since the
last BaseCompaction
-
-### `base_compaction_num_cumulative_deltas`
+### `base_compaction_min_rowset_num`
Default: 5
Review Comment:
The limit of the number of Cumulative files to be reached. After reaching
this limit, BaseCompaction will be triggered. Clumulative files == rowsets?
##########
be/src/common/config.h:
##########
@@ -253,59 +253,41 @@ CONF_Bool(enable_vectorized_compaction, "true");
// whether enable vectorized schema change/material-view/rollup task.
CONF_Bool(enable_vectorized_alter_table, "true");
-// check the configuration of auto compaction in seconds when auto compaction
disabled
-CONF_mInt32(check_auto_compaction_interval_seconds, "5");
+// This config can be set to limit thread number in compaction thread pool.
+CONF_mInt32(max_base_compaction_threads, "4");
+CONF_mInt32(max_cumu_compaction_threads, "10");
-CONF_mInt64(base_compaction_num_cumulative_deltas, "5");
-CONF_mDouble(base_cumulative_delta_ratio, "0.3");
-CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
-CONF_mInt32(base_compaction_write_mbytes_per_sec, "5");
CONF_Bool(enable_base_compaction_idle_sched, "true");
-
-// dup key not compaction big files
-CONF_Bool(enable_dup_key_base_compaction_skip_big_file, "true");
+CONF_mInt64(base_compaction_min_rowset_num, "5");
+CONF_mDouble(base_compaction_min_data_ratio, "0.3");
CONF_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config size,
+// output rowset of cumulative compaction total disk size exceed this config
size,
// this rowset will be given to base compaction, unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_size_mbytes, "1024");
+CONF_mInt64(compaction_promotion_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config ratio of
+// output rowset of cumulative compaction total disk size exceed this config
ratio of
// base rowset's total disk size, this rowset will be given to base
compaction. The value must be between
// 0 and 1.
-CONF_mDouble(cumulative_size_based_promotion_ratio, "0.05");
+CONF_mDouble(compaction_promotion_ratio, "0.05");
-// In size_based policy, the smallest size of rowset promotion. When the
rowset is less than this config, this
+// the smallest size of rowset promotion. When the rowset is less than this
config, this
// rowset will be not given to base compaction. The unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
+CONF_mInt64(compaction_promotion_min_size_mbytes, "64");
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
-CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
+CONF_mInt64(compaction_min_size_mbytes, "64");
// cumulative compaction policy: min and max delta file's number
-CONF_mInt64(min_cumulative_compaction_num_singleton_deltas, "5");
-CONF_mInt64(max_cumulative_compaction_num_singleton_deltas, "1000");
-
-// if compaction of a tablet failed, this tablet should not be chosen to
-// compaction until this interval passes.
-CONF_mInt64(min_compaction_failure_interval_sec, "5"); // 5 seconds
-
-// This config can be set to limit thread number in compaction thread pool.
-CONF_mInt32(max_base_compaction_threads, "4");
-CONF_mInt32(max_cumu_compaction_threads, "10");
-
-// This config can be set to limit thread number in smallcompaction thread
pool.
-CONF_mInt32(quick_compaction_max_threads, "10");
-
-// Thread count to do tablet meta checkpoint, -1 means use the data
directories count.
-CONF_Int32(max_meta_checkpoint_threads, "-1");
+CONF_mInt64(cumulative_compaction_min_deltas, "5");
+CONF_mInt64(cumulative_compaction_max_deltas, "1000");
Review Comment:
Can we rename deltas to segments?
##########
be/src/common/config.h:
##########
@@ -253,59 +253,41 @@ CONF_Bool(enable_vectorized_compaction, "true");
// whether enable vectorized schema change/material-view/rollup task.
CONF_Bool(enable_vectorized_alter_table, "true");
-// check the configuration of auto compaction in seconds when auto compaction
disabled
-CONF_mInt32(check_auto_compaction_interval_seconds, "5");
+// This config can be set to limit thread number in compaction thread pool.
+CONF_mInt32(max_base_compaction_threads, "4");
+CONF_mInt32(max_cumu_compaction_threads, "10");
-CONF_mInt64(base_compaction_num_cumulative_deltas, "5");
-CONF_mDouble(base_cumulative_delta_ratio, "0.3");
-CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
-CONF_mInt32(base_compaction_write_mbytes_per_sec, "5");
CONF_Bool(enable_base_compaction_idle_sched, "true");
-
-// dup key not compaction big files
-CONF_Bool(enable_dup_key_base_compaction_skip_big_file, "true");
+CONF_mInt64(base_compaction_min_rowset_num, "5");
+CONF_mDouble(base_compaction_min_data_ratio, "0.3");
CONF_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config size,
+// output rowset of cumulative compaction total disk size exceed this config
size,
// this rowset will be given to base compaction, unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_size_mbytes, "1024");
+CONF_mInt64(compaction_promotion_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config ratio of
+// output rowset of cumulative compaction total disk size exceed this config
ratio of
// base rowset's total disk size, this rowset will be given to base
compaction. The value must be between
// 0 and 1.
-CONF_mDouble(cumulative_size_based_promotion_ratio, "0.05");
+CONF_mDouble(compaction_promotion_ratio, "0.05");
-// In size_based policy, the smallest size of rowset promotion. When the
rowset is less than this config, this
+// the smallest size of rowset promotion. When the rowset is less than this
config, this
// rowset will be not given to base compaction. The unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
+CONF_mInt64(compaction_promotion_min_size_mbytes, "64");
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
-CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
+CONF_mInt64(compaction_min_size_mbytes, "64");
Review Comment:
And the config is strange, if we insert small data each, e.g. 1row, then all
rowset would not be compacted until total size reached 64MB, however the
version num is limited by 2000?
##########
be/src/olap/cumulative_compaction.cpp:
##########
@@ -143,8 +142,7 @@ Status CumulativeCompaction::pick_rowsets_to_compact() {
int64_t last_cumu = _tablet->last_cumu_compaction_success_time();
int64_t last_base = _tablet->last_base_compaction_success_time();
if (last_cumu != 0 || last_base != 0) {
- int64_t interval_threshold =
-
config::base_compaction_interval_seconds_since_last_operation * 1000;
Review Comment:
we should define a const instead of using 86400 in different places.
##########
be/src/common/config.h:
##########
@@ -253,59 +253,41 @@ CONF_Bool(enable_vectorized_compaction, "true");
// whether enable vectorized schema change/material-view/rollup task.
CONF_Bool(enable_vectorized_alter_table, "true");
-// check the configuration of auto compaction in seconds when auto compaction
disabled
-CONF_mInt32(check_auto_compaction_interval_seconds, "5");
+// This config can be set to limit thread number in compaction thread pool.
+CONF_mInt32(max_base_compaction_threads, "4");
+CONF_mInt32(max_cumu_compaction_threads, "10");
-CONF_mInt64(base_compaction_num_cumulative_deltas, "5");
-CONF_mDouble(base_cumulative_delta_ratio, "0.3");
-CONF_mInt64(base_compaction_interval_seconds_since_last_operation, "86400");
-CONF_mInt32(base_compaction_write_mbytes_per_sec, "5");
CONF_Bool(enable_base_compaction_idle_sched, "true");
-
-// dup key not compaction big files
-CONF_Bool(enable_dup_key_base_compaction_skip_big_file, "true");
+CONF_mInt64(base_compaction_min_rowset_num, "5");
+CONF_mDouble(base_compaction_min_data_ratio, "0.3");
CONF_mInt64(base_compaction_dup_key_max_file_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config size,
+// output rowset of cumulative compaction total disk size exceed this config
size,
// this rowset will be given to base compaction, unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_size_mbytes, "1024");
+CONF_mInt64(compaction_promotion_size_mbytes, "1024");
-// In size_based policy, output rowset of cumulative compaction total disk
size exceed this config ratio of
+// output rowset of cumulative compaction total disk size exceed this config
ratio of
// base rowset's total disk size, this rowset will be given to base
compaction. The value must be between
// 0 and 1.
-CONF_mDouble(cumulative_size_based_promotion_ratio, "0.05");
+CONF_mDouble(compaction_promotion_ratio, "0.05");
-// In size_based policy, the smallest size of rowset promotion. When the
rowset is less than this config, this
+// the smallest size of rowset promotion. When the rowset is less than this
config, this
// rowset will be not given to base compaction. The unit is m byte.
-CONF_mInt64(cumulative_size_based_promotion_min_size_mbytes, "64");
+CONF_mInt64(compaction_promotion_min_size_mbytes, "64");
// The lower bound size to do cumulative compaction. When total disk size of
candidate rowsets is less than
// this size, size_based policy may not do to cumulative compaction. The unit
is m byte.
-CONF_mInt64(cumulative_size_based_compaction_lower_size_mbytes, "64");
+CONF_mInt64(compaction_min_size_mbytes, "64");
Review Comment:
If it is used for cu, we'd better use cumulative_compaction_min_size_mbytes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
