This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9a82c9f7288 [fix](partial update) duplicate key occur when BE restart
after conflict concurrent partial update (#35739)
9a82c9f7288 is described below
commit 9a82c9f728898b184c65c4372f06476bbe50c2cf
Author: zhannngchen <[email protected]>
AuthorDate: Sat Jun 1 22:59:13 2024 +0800
[fix](partial update) duplicate key occur when BE restart after conflict
concurrent partial update (#35739)
## Proposed changes
Issue Number: close #xxx
1. In #30366 , in order to avoid that some incomplete delete bitmap left
in `txn_info->delete_bitmap` when publish failed, we make a copy of
`txn_info->delete_bitmap` before we start to compute the delete bitmap
2. this copy is not updated back to `txn_info->delete_bitmap` after
`rowset->rowset_meta()->merge_rowset_meta()` is successful
3. `txnManager::publish_txn()` saves the contents of
`txn_info->delete_bitmap` to RocksDB after the call to
`update_delete_bitmap()`, due to the issue in step 2, bitmap generated
during publish is not saved to RocksDB, so if BE restarts at this point,
this part of the incremental delete bitmap will be lost
4. it will result in duplicated keys on querying
---
be/src/olap/base_tablet.cpp | 4 +++-
be/src/olap/base_tablet.h | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 21563890a40..726f47ea5b8 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -1118,7 +1118,7 @@ void
BaseTablet::_remove_sentinel_mark_from_delete_bitmap(DeleteBitmapPtr delete
}
}
-Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
+Status BaseTablet::update_delete_bitmap(const BaseTabletSPtr& self,
TabletTxnInfo* txn_info,
int64_t txn_id, int64_t
txn_expiration) {
SCOPED_BVAR_LATENCY(g_tablet_update_delete_bitmap_latency);
RowsetIdUnorderedSet cur_rowset_ids;
@@ -1226,6 +1226,8 @@ Status BaseTablet::update_delete_bitmap(const
BaseTabletSPtr& self, const Tablet
RowsetSharedPtr transient_rowset;
RETURN_IF_ERROR(transient_rs_writer->build(transient_rowset));
rowset->rowset_meta()->merge_rowset_meta(*transient_rowset->rowset_meta());
+ // update the shared_ptr to new bitmap, which is consistent with
current rowset.
+ txn_info->delete_bitmap = delete_bitmap;
// erase segment cache cause we will add a segment to rowset
SegmentLoader::instance()->erase_segments(rowset->rowset_id(),
rowset->num_segments());
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 58b72f2ccba..8bef4860ee9 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -210,7 +210,7 @@ public:
const Rowset& rowset, std::shared_ptr<PartialUpdateInfo>
partial_update_info,
int64_t txn_expiration = 0) = 0;
- static Status update_delete_bitmap(const BaseTabletSPtr& self, const
TabletTxnInfo* txn_info,
+ static Status update_delete_bitmap(const BaseTabletSPtr& self,
TabletTxnInfo* txn_info,
int64_t txn_id, int64_t txn_expiration
= 0);
virtual Status save_delete_bitmap(const TabletTxnInfo* txn_info, int64_t
txn_id,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
