This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 3379cbbbe7 [fix](merge-on-write) cherry pick prs from master (#23575)
3379cbbbe7 is described below
commit 3379cbbbe76828cc1c05cc7ad0a016f384945c72
Author: Xin Liao <[email protected]>
AuthorDate: Mon Aug 28 19:59:23 2023 +0800
[fix](merge-on-write) cherry pick prs from master (#23575)
cherry-pick from: #22044 #22154 #22437
---
be/src/olap/delta_writer.cpp | 29 +++++++++++++++++++----------
be/src/olap/schema_change.cpp | 8 +++++++-
be/src/olap/tablet_meta.cpp | 20 +++++++++++++-------
3 files changed, 39 insertions(+), 18 deletions(-)
diff --git a/be/src/olap/delta_writer.cpp b/be/src/olap/delta_writer.cpp
index fb26c010fb..ab17abd6c2 100644
--- a/be/src/olap/delta_writer.cpp
+++ b/be/src/olap/delta_writer.cpp
@@ -113,7 +113,13 @@ Status DeltaWriter::init() {
if (_tablet->enable_unique_key_merge_on_write()) {
std::lock_guard<std::shared_mutex> lck(_tablet->get_header_lock());
_cur_max_version = _tablet->max_version_unlocked().second;
- _rowset_ids = _tablet->all_rs_id(_cur_max_version);
+ // tablet is under alter process. The delete bitmap will be calculated
after conversion.
+ if (_tablet->tablet_state() == TABLET_NOTREADY &&
+ SchemaChangeHandler::tablet_in_converting(_tablet->tablet_id())) {
+ _rowset_ids.clear();
+ } else {
+ _rowset_ids = _tablet->all_rs_id(_cur_max_version);
+ }
}
// check tablet version number
@@ -399,19 +405,22 @@ Status DeltaWriter::close_wait(const PSlaveTabletNodes&
slave_tablet_nodes,
return res;
}
if (_tablet->enable_unique_key_merge_on_write()) {
- auto beta_rowset = reinterpret_cast<BetaRowset*>(_cur_rowset.get());
- std::vector<segment_v2::SegmentSharedPtr> segments;
- RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
// tablet is under alter process. The delete bitmap will be calculated
after conversion.
if (_tablet->tablet_state() == TABLET_NOTREADY &&
SchemaChangeHandler::tablet_in_converting(_tablet->tablet_id())) {
- return Status::OK();
- }
- if (segments.size() > 1) {
-
RETURN_IF_ERROR(_tablet->calc_delete_bitmap(beta_rowset->rowset_id(), segments,
nullptr,
- _delete_bitmap,
_cur_max_version, true));
+ LOG(INFO) << "tablet is under alter process, delete bitmap will be
calculated later, "
+ "tablet_id: "
+ << _tablet->tablet_id() << " txn_id: " << _req.txn_id;
+ } else {
+ auto beta_rowset =
reinterpret_cast<BetaRowset*>(_cur_rowset.get());
+ std::vector<segment_v2::SegmentSharedPtr> segments;
+ RETURN_IF_ERROR(beta_rowset->load_segments(&segments));
+ if (segments.size() > 1) {
+
RETURN_IF_ERROR(_tablet->calc_delete_bitmap(beta_rowset->rowset_id(), segments,
+ nullptr,
_delete_bitmap,
+ _cur_max_version,
true));
+ }
}
-
_storage_engine->txn_manager()->set_txn_related_delete_bitmap(
_req.partition_id, _req.txn_id, _tablet->tablet_id(),
_tablet->schema_hash(),
_tablet->tablet_uid(), true, _delete_bitmap, _rowset_ids);
diff --git a/be/src/olap/schema_change.cpp b/be/src/olap/schema_change.cpp
index dd81df84d3..b22ba4f32d 100644
--- a/be/src/olap/schema_change.cpp
+++ b/be/src/olap/schema_change.cpp
@@ -2067,7 +2067,7 @@ Status
SchemaChangeHandler::_do_process_alter_tablet_v2(const TAlterTabletReqV2&
std::lock_guard<std::mutex>
rwlock(new_tablet->get_rowset_update_lock());
std::lock_guard<std::shared_mutex>
new_wlock(new_tablet->get_header_lock());
SCOPED_SIMPLE_TRACE_IF_TIMEOUT(TRACE_TABLET_LOCK_THRESHOLD);
- int64_t new_max_version = new_tablet->max_version().second;
+ int64_t new_max_version =
new_tablet->max_version_unlocked().second;
rowsets.clear();
if (max_version < new_max_version) {
LOG(INFO)
@@ -2377,6 +2377,12 @@ Status SchemaChangeHandler::_parse_request(const
SchemaChangeParams& sc_params,
return Status::OK();
}
+ if (new_tablet->enable_unique_key_merge_on_write() &&
+ new_tablet->num_key_columns() > base_tablet_schema->num_key_columns())
{
+ *sc_directly = true;
+ return Status::OK();
+ }
+
if (base_tablet_schema->num_short_key_columns() !=
new_tablet->num_short_key_columns()) {
// the number of short_keys changed, can't do linked schema change
*sc_directly = true;
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 8fc037b85e..a7029913a8 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -487,6 +487,10 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
// init _schema
_schema->init_from_pb(tablet_meta_pb.schema());
+ if (tablet_meta_pb.has_enable_unique_key_merge_on_write()) {
+ _enable_unique_key_merge_on_write =
tablet_meta_pb.enable_unique_key_merge_on_write();
+ }
+
// init _rs_metas
for (auto& it : tablet_meta_pb.rs_metas()) {
RowsetMetaSharedPtr rs_meta(new RowsetMeta());
@@ -494,10 +498,15 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
_rs_metas.push_back(std::move(rs_meta));
}
- for (auto& it : tablet_meta_pb.stale_rs_metas()) {
- RowsetMetaSharedPtr rs_meta(new RowsetMeta());
- rs_meta->init_from_pb(it);
- _stale_rs_metas.push_back(std::move(rs_meta));
+ // For mow table, delete bitmap of stale rowsets has not been persisted.
+ // When be restart, query should not read the stale rowset, otherwise
duplicate keys
+ // will be read out. Therefore, we don't add them to _stale_rs_meta for
mow table.
+ if (!_enable_unique_key_merge_on_write) {
+ for (auto& it : tablet_meta_pb.stale_rs_metas()) {
+ RowsetMetaSharedPtr rs_meta(new RowsetMeta());
+ rs_meta->init_from_pb(it);
+ _stale_rs_metas.push_back(std::move(rs_meta));
+ }
}
if (tablet_meta_pb.has_in_restore_mode()) {
@@ -509,9 +518,6 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
}
_storage_policy = tablet_meta_pb.storage_policy();
- if (tablet_meta_pb.has_enable_unique_key_merge_on_write()) {
- _enable_unique_key_merge_on_write =
tablet_meta_pb.enable_unique_key_merge_on_write();
- }
if (tablet_meta_pb.has_delete_bitmap()) {
int rst_ids_size = tablet_meta_pb.delete_bitmap().rowset_ids_size();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
