github-actions[bot] commented on code in PR #31317:
URL: https://github.com/apache/doris/pull/31317#discussion_r1508834637
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1064,200 @@
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr
delete_bitmap,
+ int64_t max_version,
int64_t txn_id,
+ const RowsetIdUnorderedSet&
rowset_ids,
+
std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid,
DeleteBitmap::INVALID_SEGMENT_ID,
+
DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value,
required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value,
required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value,
missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr,
root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr,
root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish
phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr
delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end =
delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' exceeds recommended size/complexity
thresholds [readability-function-size]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1144:** 115 lines including whitespace and
comments (threshold 80)
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
^
```
</details>
##########
be/src/olap/base_tablet.cpp:
##########
@@ -1059,4 +1064,200 @@ Status BaseTablet::_capture_consistent_rowsets_unlocked(
return Status::OK();
}
+Status BaseTablet::check_delete_bitmap_correctness(DeleteBitmapPtr
delete_bitmap,
+ int64_t max_version,
int64_t txn_id,
+ const RowsetIdUnorderedSet&
rowset_ids,
+
std::vector<RowsetSharedPtr>* rowsets) {
+ RowsetIdUnorderedSet missing_ids;
+ for (const auto& rowsetid : rowset_ids) {
+ if (!delete_bitmap->delete_bitmap.contains({rowsetid,
DeleteBitmap::INVALID_SEGMENT_ID,
+
DeleteBitmap::TEMP_VERSION_COMMON})) {
+ missing_ids.insert(rowsetid);
+ }
+ }
+
+ if (!missing_ids.empty()) {
+ LOG(WARNING) << "[txn_id:" << txn_id << "][tablet_id:" << tablet_id()
+ << "][max_version: " << max_version
+ << "] check delete bitmap correctness failed!";
+ rapidjson::Document root;
+ root.SetObject();
+ rapidjson::Document required_rowsets_arr;
+ required_rowsets_arr.SetArray();
+ rapidjson::Document missing_rowsets_arr;
+ missing_rowsets_arr.SetArray();
+
+ if (rowsets != nullptr) {
+ for (const auto& rowset : *rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value,
required_rowsets_arr.GetAllocator());
+ }
+ } else {
+ std::vector<RowsetSharedPtr> rowsets;
+ {
+ std::shared_lock meta_rlock(_meta_lock);
+ rowsets = get_rowset_by_ids(&rowset_ids);
+ }
+ for (const auto& rowset : rowsets) {
+ rapidjson::Value value;
+ std::string version_str = rowset->get_rowset_info_str();
+ value.SetString(version_str.c_str(), version_str.length(),
+ required_rowsets_arr.GetAllocator());
+ required_rowsets_arr.PushBack(value,
required_rowsets_arr.GetAllocator());
+ }
+ }
+ for (const auto& missing_rowset_id : missing_ids) {
+ rapidjson::Value miss_value;
+ std::string rowset_id_str = missing_rowset_id.to_string();
+ miss_value.SetString(rowset_id_str.c_str(), rowset_id_str.length(),
+ missing_rowsets_arr.GetAllocator());
+ missing_rowsets_arr.PushBack(miss_value,
missing_rowsets_arr.GetAllocator());
+ }
+
+ root.AddMember("required_rowsets", required_rowsets_arr,
root.GetAllocator());
+ root.AddMember("missing_rowsets", missing_rowsets_arr,
root.GetAllocator());
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ std::string rowset_status_string = std::string(strbuf.GetString());
+ LOG_EVERY_SECOND(WARNING) << rowset_status_string;
+ // let it crash if correctness check failed in Debug mode
+ DCHECK(false) << "delete bitmap correctness check failed in publish
phase!";
+ return Status::InternalError("check delete bitmap failed!");
+ }
+ return Status::OK();
+}
+
+void BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr
delete_bitmap) {
+ for (auto it = delete_bitmap->delete_bitmap.begin(), end =
delete_bitmap->delete_bitmap.end();
+ it != end;) {
+ if (std::get<1>(it->first) == DeleteBitmap::INVALID_SEGMENT_ID) {
+ it = delete_bitmap->delete_bitmap.erase(it);
+ } else {
+ ++it;
+ }
+ }
+}
+
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
Review Comment:
warning: function 'update_delete_bitmap' has cognitive complexity of 56
(threshold 50) [readability-function-cognitive-complexity]
```cpp
Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
^
```
<details>
<summary>Additional context</summary>
**be/src/olap/base_tablet.cpp:1162:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (txn_info->partial_update_info &&
txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1162:** +1
```cpp
if (txn_info->partial_update_info &&
txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1168:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1168:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(std::dynamic_pointer_cast<BetaRowset>(rowset)->load_segments(&segments));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1174:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (self->tablet_state() == TABLET_NOTREADY) {
^
```
**be/src/olap/base_tablet.cpp:1179:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1,
&cur_rowset_ids));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1179:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->get_all_rs_id_unlocked(cur_version - 1,
&cur_rowset_ids));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1198:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (segments.size() <= 1) {
^
```
**be/src/olap/base_tablet.cpp:1199:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments,
specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1199:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments,
specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1202:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/olap/base_tablet.cpp:1204:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments,
specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1204:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
RETURN_IF_ERROR(calc_delete_bitmap(self, rowset, segments,
specified_rowsets, delete_bitmap,
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1206:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1206:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
RETURN_IF_ERROR(token->wait());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1210:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (watch.get_elapse_time_us() < 1 * 1000 * 1000) {
^
```
**be/src/olap/base_tablet.cpp:1212:** +1, nesting level increased to 1
```cpp
} else {
^
```
**be/src/olap/base_tablet.cpp:1220:** nesting level increased to 1
```cpp
[](size_t sum, const segment_v2::SegmentSharedPtr& s) { return
sum += s->num_rows(); });
^
```
**be/src/olap/base_tablet.cpp:1227:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (config::enable_merge_on_write_correctness_check &&
rowset->num_rows() != 0) {
^
```
**be/src/olap/base_tablet.cpp:1227:** +1
```cpp
if (config::enable_merge_on_write_correctness_check &&
rowset->num_rows() != 0) {
^
```
**be/src/olap/base_tablet.cpp:1232:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
if (!st.ok()) {
^
```
**be/src/olap/base_tablet.cpp:1238:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
if (txn_info->partial_update_info &&
txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1238:** +1
```cpp
if (txn_info->partial_update_info &&
txn_info->partial_update_info->is_partial_update) {
^
```
**be/src/olap/base_tablet.cpp:1239:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail",
{
^
```
**be/src/util/debug_points.h:34:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (UNLIKELY(config::enable_debug_points)) {
\
^
```
**be/src/olap/base_tablet.cpp:1239:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
DBUG_EXECUTE_IF("Tablet.update_delete_bitmap.partial_update_write_rowset_fail",
{
^
```
**be/src/util/debug_points.h:36:** expanded from macro 'DBUG_EXECUTE_IF'
```cpp
if (dp) {
\
^
```
**be/src/olap/base_tablet.cpp:1240:** +4, including nesting penalty of 3,
nesting level increased to 4
```cpp
if (rand() % 100 < (100 * dp->param("percent", 0.5))) {
^
```
**be/src/olap/base_tablet.cpp:1248:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1248:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->flush());
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1250:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1250:** +3, including nesting penalty of 2,
nesting level increased to 3
```cpp
RETURN_IF_ERROR(rowset_writer->build(transient_rowset));
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/olap/base_tablet.cpp:1257:** +1, including nesting penalty of 0,
nesting level increased to 1
```cpp
RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id,
delete_bitmap, rowset_writer.get(),
^
```
**be/src/common/status.h:541:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/olap/base_tablet.cpp:1257:** +2, including nesting penalty of 1,
nesting level increased to 2
```cpp
RETURN_IF_ERROR(self->save_delete_bitmap(txn_info, txn_id,
delete_bitmap, rowset_writer.get(),
^
```
**be/src/common/status.h:543:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
