github-actions[bot] commented on code in PR #16681:
URL: https://github.com/apache/doris/pull/16681#discussion_r1104768074
##########
be/src/olap/cumulative_compaction_policy.cpp:
##########
@@ -219,10 +216,10 @@ void
SizeBasedCumulativeCompactionPolicy::calc_cumulative_compaction_score(
// if current level less then remain level, score contains current
rowset
// and process return; otherwise, score does not contains current
rowset.
if (current_level <= remain_level) {
- return;
+ return score;
}
total_size -= rs_meta->total_disk_size();
- *score -= rs_meta->get_compaction_score();
+ score -= rs_meta->get_compaction_score();
}
}
Review Comment:
warning: non-void function does not return a value in all control paths
[clang-diagnostic-return-type]
```cpp
}
^
```
##########
be/src/olap/olap_server.cpp:
##########
@@ -755,6 +766,107 @@
}
}
+void StorageEngine::_cold_data_compaction_producer_callback() {
+ std::unordered_set<int64_t> tablet_submitted;
+ std::mutex tablet_submitted_mtx;
+
+ while (!_stop_background_threads_latch.wait_for(
+ std::chrono::seconds(config::cold_data_compaction_interval_sec))) {
+ if (config::disable_auto_compaction) {
+ continue;
+ }
+
+ std::unordered_set<int64_t> copied_tablet_submitted;
+ {
+ std::lock_guard lock(tablet_submitted_mtx);
+ copied_tablet_submitted = tablet_submitted;
+ }
+ int n = config::cold_data_compaction_thread_num -
copied_tablet_submitted.size();
+ if (n <= 0) {
+ continue;
+ }
+ auto tablets =
_tablet_manager->get_all_tablet([&copied_tablet_submitted](Tablet* t) {
+ return t->tablet_meta()->cooldown_meta_id().initialized() &&
t->is_used() &&
+ t->tablet_state() == TABLET_RUNNING &&
+ !copied_tablet_submitted.count(t->tablet_id()) &&
+
!t->tablet_meta()->tablet_schema()->disable_auto_compaction();
+ });
+ std::vector<std::pair<TabletSharedPtr, int64_t>> tablet_to_compact;
+ tablet_to_compact.reserve(n + 1);
+ std::vector<std::pair<TabletSharedPtr, int64_t>> tablet_to_follow;
+ tablet_to_follow.reserve(n + 1);
+
+ for (auto& t : tablets) {
+ if (t->replica_id() == t->cooldown_replica_id()) {
+ auto score = t->calc_cold_data_compaction_score();
+ if (score < 4) {
+ continue;
+ }
+ tablet_to_compact.emplace_back(t, score);
+ std::sort(tablet_to_compact.begin(), tablet_to_compact.end(),
+ [](auto& a, auto& b) { return a.second > b.second;
});
+ if (tablet_to_compact.size() > n) tablet_to_compact.pop_back();
+ continue;
+ }
+ // else, need to follow
+ {
+ std::lock_guard lock(_running_cooldown_mutex);
+ if (_running_cooldown_tablets.count(t->table_id())) {
+ // already in cooldown queue
+ continue;
+ }
+ }
+ // TODO(plat1ko): some avoidance strategy if failed to follow
+ auto score = t->calc_cold_data_compaction_score();
+ tablet_to_follow.emplace_back(t, score);
+ std::sort(tablet_to_follow.begin(), tablet_to_follow.end(),
+ [](auto& a, auto& b) { return a.second > b.second; });
+ if (tablet_to_follow.size() > n) tablet_to_follow.pop_back();
Review Comment:
warning: statement should be inside braces
[readability-braces-around-statements]
```suggestion
if (tablet_to_follow.size() > n) { tablet_to_follow.pop_back();
}
```
##########
be/src/olap/olap_server.cpp:
##########
@@ -755,6 +766,107 @@ void
StorageEngine::_remove_unused_remote_files_callback() {
}
}
+void StorageEngine::_cold_data_compaction_producer_callback() {
+ std::unordered_set<int64_t> tablet_submitted;
+ std::mutex tablet_submitted_mtx;
+
+ while (!_stop_background_threads_latch.wait_for(
+ std::chrono::seconds(config::cold_data_compaction_interval_sec))) {
+ if (config::disable_auto_compaction) {
+ continue;
+ }
+
+ std::unordered_set<int64_t> copied_tablet_submitted;
+ {
+ std::lock_guard lock(tablet_submitted_mtx);
+ copied_tablet_submitted = tablet_submitted;
+ }
+ int n = config::cold_data_compaction_thread_num -
copied_tablet_submitted.size();
+ if (n <= 0) {
+ continue;
+ }
+ auto tablets =
_tablet_manager->get_all_tablet([&copied_tablet_submitted](Tablet* t) {
+ return t->tablet_meta()->cooldown_meta_id().initialized() &&
t->is_used() &&
+ t->tablet_state() == TABLET_RUNNING &&
+ !copied_tablet_submitted.count(t->tablet_id()) &&
+
!t->tablet_meta()->tablet_schema()->disable_auto_compaction();
+ });
+ std::vector<std::pair<TabletSharedPtr, int64_t>> tablet_to_compact;
+ tablet_to_compact.reserve(n + 1);
+ std::vector<std::pair<TabletSharedPtr, int64_t>> tablet_to_follow;
+ tablet_to_follow.reserve(n + 1);
+
+ for (auto& t : tablets) {
+ if (t->replica_id() == t->cooldown_replica_id()) {
+ auto score = t->calc_cold_data_compaction_score();
+ if (score < 4) {
+ continue;
+ }
+ tablet_to_compact.emplace_back(t, score);
+ std::sort(tablet_to_compact.begin(), tablet_to_compact.end(),
+ [](auto& a, auto& b) { return a.second > b.second;
});
+ if (tablet_to_compact.size() > n) tablet_to_compact.pop_back();
Review Comment:
warning: statement should be inside braces
[readability-braces-around-statements]
```suggestion
if (tablet_to_compact.size() > n) {
tablet_to_compact.pop_back();
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
