This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new aef3b54db5b branch-3.0-pick: [Fix](partial update) Fix incorrect
result when partial update include delete sign columns #46194 (#46336)
aef3b54db5b is described below
commit aef3b54db5b82cee406d59433ed198495079b75f
Author: bobhan1 <[email protected]>
AuthorDate: Tue Jan 7 15:15:17 2025 +0800
branch-3.0-pick: [Fix](partial update) Fix incorrect result when partial
update include delete sign columns #46194 (#46336)
pick https://github.com/apache/doris/pull/46194
---
be/src/olap/base_tablet.cpp | 14 +--
be/src/olap/partial_update_info.cpp | 37 +++---
be/src/olap/partial_update_info.h | 5 +-
...t_partial_update_with_delete_col_in_publish.out | 15 +++
.../test_partial_update_seq_col_delete.out | Bin 1530 -> 1533 bytes
.../test_partial_update_seq_type_delete.out | Bin 2967 -> 2981 bytes
.../test_partial_update_merge_type.out | 32 ++---
.../test_partial_update_seq_col_delete.out | 4 +-
.../test_partial_update_seq_type_delete.out | 4 +-
.../partial_update/test_with_delete_sign_col.out | 133 +++++++++++++++++++++
.../partial_update/with_delete1.csv | 6 +
...artial_update_with_delete_col_in_publish.groovy | 104 ++++++++++++++++
.../test_with_delete_sign_col.groovy | 98 +++++++++++++++
13 files changed, 408 insertions(+), 44 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index a5df20d6383..8cad5b87f5b 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -931,7 +931,7 @@ Status BaseTablet::generate_new_block_for_partial_update(
// read current rowset first, if a row in the current rowset has delete
sign mark
// we don't need to read values from old block
RETURN_IF_ERROR(read_plan_update.read_columns_by_plan(
- *rowset_schema, update_cids, rsid_to_rowset, update_block,
&read_index_update));
+ *rowset_schema, update_cids, rsid_to_rowset, update_block,
&read_index_update, false));
size_t update_rows = read_index_update.size();
for (auto i = 0; i < update_cids.size(); ++i) {
for (auto idx = 0; idx < update_rows; ++idx) {
@@ -951,19 +951,17 @@ Status BaseTablet::generate_new_block_for_partial_update(
// rowid in the final block(start from 0, increase, may not continuous
becasue we skip to read some rows) -> rowid to read in old_block
std::map<uint32_t, uint32_t> read_index_old;
RETURN_IF_ERROR(read_plan_ori.read_columns_by_plan(*rowset_schema,
missing_cids, rsid_to_rowset,
- old_block,
&read_index_old,
+ old_block,
&read_index_old, true,
new_block_delete_signs));
size_t old_rows = read_index_old.size();
const auto* __restrict old_block_delete_signs =
get_delete_sign_column_data(old_block, old_rows);
-
+ DCHECK(old_block_delete_signs != nullptr);
// build default value block
auto default_value_block = old_block.clone_empty();
- if (old_block_delete_signs != nullptr || new_block_delete_signs !=
nullptr) {
- RETURN_IF_ERROR(BaseTablet::generate_default_value_block(
- *rowset_schema, missing_cids,
partial_update_info->default_values, old_block,
- default_value_block));
- }
+ RETURN_IF_ERROR(BaseTablet::generate_default_value_block(*rowset_schema,
missing_cids,
+
partial_update_info->default_values,
+ old_block,
default_value_block));
auto mutable_default_value_columns = default_value_block.mutate_columns();
CHECK(update_rows >= old_rows);
diff --git a/be/src/olap/partial_update_info.cpp
b/be/src/olap/partial_update_info.cpp
index 247353103df..a1ae19a6aa7 100644
--- a/be/src/olap/partial_update_info.cpp
+++ b/be/src/olap/partial_update_info.cpp
@@ -24,7 +24,6 @@
#include "olap/olap_common.h"
#include "olap/rowset/rowset.h"
#include "olap/rowset/rowset_writer_context.h"
-#include "olap/tablet_meta.h"
#include "olap/tablet_schema.h"
#include "olap/utils.h"
#include "util/bitmap_value.h"
@@ -206,9 +205,21 @@ void PartialUpdateReadPlan::prepare_to_read(const
RowLocation& row_location, siz
// read columns by read plan
// read_index: ori_pos-> block_idx
Status PartialUpdateReadPlan::read_columns_by_plan(
- const TabletSchema& tablet_schema, const std::vector<uint32_t>
cids_to_read,
+ const TabletSchema& tablet_schema, std::vector<uint32_t> cids_to_read,
const std::map<RowsetId, RowsetSharedPtr>& rsid_to_rowset,
vectorized::Block& block,
- std::map<uint32_t, uint32_t>* read_index, const signed char*
__restrict skip_map) const {
+ std::map<uint32_t, uint32_t>* read_index, bool
force_read_old_delete_signs,
+ const signed char* __restrict cur_delete_signs) const {
+ if (force_read_old_delete_signs) {
+ // always read delete sign column from historical data
+ if (const vectorized::ColumnWithTypeAndName* old_delete_sign_column =
+ block.try_get_by_name(DELETE_SIGN);
+ old_delete_sign_column == nullptr) {
+ auto del_col_cid = tablet_schema.field_index(DELETE_SIGN);
+ cids_to_read.emplace_back(del_col_cid);
+ block.swap(tablet_schema.create_block_by_cids(cids_to_read));
+ }
+ }
+
bool has_row_column = tablet_schema.has_row_store_for_all_columns();
auto mutable_columns = block.mutate_columns();
size_t read_idx = 0;
@@ -218,7 +229,7 @@ Status PartialUpdateReadPlan::read_columns_by_plan(
CHECK(rowset_iter != rsid_to_rowset.end());
std::vector<uint32_t> rids;
for (auto [rid, pos] : mappings) {
- if (skip_map && skip_map[pos]) {
+ if (cur_delete_signs && cur_delete_signs[pos]) {
continue;
}
rids.emplace_back(rid);
@@ -263,17 +274,15 @@ Status PartialUpdateReadPlan::fill_missing_columns(
// record real pos, key is input line num, value is old_block line num
std::map<uint32_t, uint32_t> read_index;
RETURN_IF_ERROR(read_columns_by_plan(tablet_schema, missing_cids,
rsid_to_rowset,
- old_value_block, &read_index,
nullptr));
-
- const auto* delete_sign_column_data =
BaseTablet::get_delete_sign_column_data(old_value_block);
+ old_value_block, &read_index, true,
nullptr));
+ const auto* old_delete_signs =
BaseTablet::get_delete_sign_column_data(old_value_block);
+ DCHECK(old_delete_signs != nullptr);
// build default value columns
auto default_value_block = old_value_block.clone_empty();
- if (has_default_or_nullable || delete_sign_column_data != nullptr) {
- RETURN_IF_ERROR(BaseTablet::generate_default_value_block(
- tablet_schema, missing_cids,
rowset_ctx->partial_update_info->default_values,
- old_value_block, default_value_block));
- }
+ RETURN_IF_ERROR(BaseTablet::generate_default_value_block(
+ tablet_schema, missing_cids,
rowset_ctx->partial_update_info->default_values,
+ old_value_block, default_value_block));
auto mutable_default_value_columns = default_value_block.mutate_columns();
// fill all missing value from mutable_old_columns, need to consider
default value and null value
@@ -285,8 +294,8 @@ Status PartialUpdateReadPlan::fill_missing_columns(
// read values from old rows for missing values in this occasion. So
we should read the DELETE_SIGN column
// to check if a row REALLY exists in the table.
auto pos_in_old_block = read_index[idx + segment_start_pos];
- if (use_default_or_null_flag[idx] || (delete_sign_column_data !=
nullptr &&
-
delete_sign_column_data[pos_in_old_block] != 0)) {
+ if (use_default_or_null_flag[idx] ||
+ (old_delete_signs != nullptr && old_delete_signs[pos_in_old_block]
!= 0)) {
for (auto i = 0; i < missing_cids.size(); ++i) {
// if the column has default value, fill it with default value
// otherwise, if the column is nullable, fill it with null
value
diff --git a/be/src/olap/partial_update_info.h
b/be/src/olap/partial_update_info.h
index 278b027942e..88fac1a3e9c 100644
--- a/be/src/olap/partial_update_info.h
+++ b/be/src/olap/partial_update_info.h
@@ -80,10 +80,11 @@ class PartialUpdateReadPlan {
public:
void prepare_to_read(const RowLocation& row_location, size_t pos);
Status read_columns_by_plan(const TabletSchema& tablet_schema,
- const std::vector<uint32_t> cids_to_read,
+ std::vector<uint32_t> cids_to_read,
const std::map<RowsetId, RowsetSharedPtr>&
rsid_to_rowset,
vectorized::Block& block, std::map<uint32_t,
uint32_t>* read_index,
- const signed char* __restrict skip_map =
nullptr) const;
+ bool force_read_old_delete_signs,
+ const signed char* __restrict cur_delete_signs
= nullptr) const;
Status fill_missing_columns(RowsetWriterContext* rowset_ctx,
const std::map<RowsetId, RowsetSharedPtr>&
rsid_to_rowset,
const TabletSchema& tablet_schema,
vectorized::Block& full_block,
diff --git
a/regression-test/data/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.out
b/regression-test/data/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.out
new file mode 100644
index 00000000000..39b43afd898
--- /dev/null
+++
b/regression-test/data/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.out
@@ -0,0 +1,15 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+
+-- !sql --
+1 1 1 987 987
+2 \N \N 987 987
+3 3 3 3 3
+4 -1 -1 987 987
+5 \N \N 987 987
+
diff --git
a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.out
b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.out
index afa7ccfc9bb..ad704cac251 100644
Binary files
a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.out
and
b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_col_delete.out
differ
diff --git
a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.out
b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.out
index 05f0c5dab4e..8621761c3a6 100644
Binary files
a/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.out
and
b/regression-test/data/unique_with_mow_c_p0/partial_update/test_partial_update_seq_type_delete.out
differ
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
index 04e4b07f2dc..d21693cda10 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_merge_type.out
@@ -28,7 +28,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !sql_2_1 --
0 0 0 0
@@ -36,7 +36,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !sql_2_2 --
0 0 0 0
@@ -90,7 +90,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !inspect --
0 0 0 0 1 0
@@ -109,7 +109,7 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
+11 \N \N \N 3 0
-- !sql_4_1 --
0 0 0 0
@@ -117,7 +117,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !inspect --
0 0 0 0 1 0
@@ -138,7 +138,7 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
+11 \N \N \N 3 0
-- !sql_4_2 --
0 0 0 0
@@ -166,8 +166,8 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
-11 \N 888 \N 5 1
+11 \N \N \N 3 0
+11 \N \N \N 5 1
-- !sql --
0 0 0 0
@@ -198,7 +198,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !sql_2_1 --
0 0 0 0
@@ -206,7 +206,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !sql_2_2 --
0 0 0 0
@@ -260,7 +260,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !inspect --
0 0 0 0 1 0
@@ -279,7 +279,7 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
+11 \N \N \N 3 0
-- !sql_4_1 --
0 0 0 0
@@ -287,7 +287,7 @@
7 7 7 7
8 8 8 8
10 \N 999 \N
-11 \N 888 \N
+11 \N \N \N
-- !inspect --
0 0 0 0 1 0
@@ -308,7 +308,7 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
+11 \N \N \N 3 0
-- !sql_4_2 --
0 0 0 0
@@ -336,6 +336,6 @@
8 8 8 8 1 0
10 \N 999 \N 2 0
11 \N 888 \N 2 1
-11 \N 888 \N 3 0
-11 \N 888 \N 5 1
+11 \N \N \N 3 0
+11 \N \N \N 5 1
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
index 3818f661202..ab2c14cf8bc 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_col_delete.out
@@ -13,7 +13,7 @@
-- !partial_update_with_seq_hidden_columns --
1 doris 200 123 1 2023-01-01 1 3
2023-01-01
-2 doris2 2600 223 1 2023-07-20 1 4
2023-07-20
+2 unknown 2600 \N 4321 2023-07-20 1 4
2023-07-20
3 unknown 1500 \N 4321 2022-07-20 1 4
2022-07-20
-- !select_default --
@@ -30,6 +30,6 @@
-- !partial_update_with_seq_hidden_columns --
1 doris 200 123 1 2023-01-01 1 3
2023-01-01
-2 doris2 2600 223 1 2023-07-20 1 4
2023-07-20
+2 unknown 2600 \N 4321 2023-07-20 1 4
2023-07-20
3 unknown 1500 \N 4321 2022-07-20 1 4
2022-07-20
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
index 90ea414b995..266ebff5714 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_seq_type_delete.out
@@ -13,7 +13,7 @@
-- !partial_update_with_seq_test --
-- !partial_update_with_seq_test_hidden --
-1 doris 2300 2300 1 2021-05-19 1 4 2300
+1 unknown 2300 2300 4321 2021-05-19 1 4 2300
2 doris2 3600 2400 1 2019-01-23 1 3 3600
3 unknown 1500 2500 4321 2022-03-31 1 4 2500
@@ -41,7 +41,7 @@
-- !partial_update_with_seq_test --
-- !partial_update_with_seq_test_hidden --
-1 doris 2300 2300 1 2021-05-19 1 4 2300
+1 unknown 2300 2300 4321 2021-05-19 1 4 2300
2 doris2 3600 2400 1 2019-01-23 1 3 3600
3 unknown 1500 2500 4321 2022-03-31 1 4 2500
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_with_delete_sign_col.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_with_delete_sign_col.out
new file mode 100644
index 00000000000..66960554a61
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_with_delete_sign_col.out
@@ -0,0 +1,133 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql_1 --
+0 0 0 0 0
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+7 7 7 7 7
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 10 10
+
+-- !sql_1 --
+0 0 0 0 0
+1 1 1 1 1
+4 4 4 4 4
+5 5 5 55 55
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 1010 1010
+
+-- !sql_1 --
+0 0 0 0 0
+1 11 11 1 1
+2 22 22 888 777
+3 33 33 888 777
+4 4 4 4 4
+5 5 5 55 55
+6 66 66 888 777
+8 8 8 8 8
+10 10 10 1010 1010
+
+-- !sql_2 --
+0 0 0 0 0
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+7 7 7 7 7
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 10 10
+
+-- !sql_2 --
+0 0 0 0 0
+1 1 1 1 1
+4 4 4 4 4
+5 5 5 55 55
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 1010 1010
+
+-- !sql_1 --
+0 0 0 0 0
+1 11 11 1 1
+2 22 22 888 777
+3 33 33 888 777
+4 4 4 4 4
+5 5 5 55 55
+6 66 66 888 777
+8 8 8 8 8
+10 10 10 1010 1010
+
+-- !sql_1 --
+0 0 0 0 0
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+7 7 7 7 7
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 10 10
+
+-- !sql_1 --
+0 0 0 0 0
+1 1 1 1 1
+4 4 4 4 4
+5 5 5 55 55
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 1010 1010
+
+-- !sql_1 --
+0 0 0 0 0
+1 11 11 1 1
+2 22 22 888 777
+3 33 33 888 777
+4 4 4 4 4
+5 5 5 55 55
+6 66 66 888 777
+8 8 8 8 8
+10 10 10 1010 1010
+
+-- !sql_2 --
+0 0 0 0 0
+1 1 1 1 1
+2 2 2 2 2
+3 3 3 3 3
+4 4 4 4 4
+5 5 5 5 5
+6 6 6 6 6
+7 7 7 7 7
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 10 10
+
+-- !sql_2 --
+0 0 0 0 0
+1 1 1 1 1
+4 4 4 4 4
+5 5 5 55 55
+8 8 8 8 8
+9 9 9 9 9
+10 10 10 1010 1010
+
+-- !sql_1 --
+0 0 0 0 0
+1 11 11 1 1
+2 22 22 888 777
+3 33 33 888 777
+4 4 4 4 4
+5 5 5 55 55
+6 66 66 888 777
+8 8 8 8 8
+10 10 10 1010 1010
+
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/with_delete1.csv
b/regression-test/data/unique_with_mow_p0/partial_update/with_delete1.csv
new file mode 100644
index 00000000000..c13ead6ffad
--- /dev/null
+++ b/regression-test/data/unique_with_mow_p0/partial_update/with_delete1.csv
@@ -0,0 +1,6 @@
+2,22,22,1
+3,33,33,1
+5,55,55,0
+6,66,66,1
+7,77,77,1
+10,1010,1010,0
diff --git
a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy
b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy
new file mode 100644
index 00000000000..caae2c59af8
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_with_delete_col_in_publish.groovy
@@ -0,0 +1,104 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite("test_partial_update_with_delete_col_in_publish", "nonConcurrent") {
+
+ def tableName = "test_partial_update_with_delete_col_in_publish"
+ sql """ DROP TABLE IF EXISTS ${tableName} force;"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` int(11) NULL,
+ `v1` BIGINT NULL,
+ `v2` BIGINT NULL,
+ `v3` BIGINT NULL,
+ `v4` BIGINT NULL,
+ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "store_row_column" = "false"); """
+ def show_res = sql "show create table ${tableName}"
+ sql """insert into ${tableName}
values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3),(4,4,4,4,4),(5,5,5,5,5);"""
+ qt_sql "select * from ${tableName} order by k;"
+
+ def enable_publish_spin_wait = {
+ if (isCloudMode()) {
+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait")
+ } else {
+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait")
+ }
+ }
+
+ def enable_block_in_publish = {
+ if (isCloudMode()) {
+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block")
+ } else {
+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block")
+ }
+ }
+
+ def disable_block_in_publish = {
+ if (isCloudMode()) {
+
GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block")
+ } else {
+
GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block")
+ }
+ }
+
+ try {
+ GetDebugPoint().clearDebugPointsForAllFEs()
+ GetDebugPoint().clearDebugPointsForAllBEs()
+
+ // block the partial update in publish phase
+ enable_publish_spin_wait()
+ enable_block_in_publish()
+
+ def threads = []
+
+ threads << Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false"
+ sql "sync;"
+ sql "insert into ${tableName}(k,v1,v2,__DORIS_DELETE_SIGN__)
values(2,222,222,1),(4,-1,-1,0),(5,555,555,1);"
+ }
+
+ Thread.sleep(500)
+
+ threads << Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "sync;"
+ sql "insert into ${tableName}(k,v3,v4)
values(1,987,987),(2,987,987),(4,987,987),(5,987,987);"
+ }
+
+ Thread.sleep(500)
+
+ disable_block_in_publish()
+ threads.each { t -> t.join() }
+
+ qt_sql "select * from ${tableName} order by k;"
+ } catch(Exception e) {
+ logger.info(e.getMessage())
+ throw e
+ } finally {
+ GetDebugPoint().clearDebugPointsForAllFEs()
+ GetDebugPoint().clearDebugPointsForAllBEs()
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_with_delete_sign_col.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_with_delete_sign_col.groovy
new file mode 100644
index 00000000000..ee2a56f7ee5
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_with_delete_sign_col.groovy
@@ -0,0 +1,98 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_with_delete_sign_col", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ def inspectRows = { sqlStr ->
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_inspect sqlStr
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect( context.config.jdbcUser, context.config.jdbcPassword,
context.config.jdbcUrl) {
+ sql "use ${db};"
+ def table1 = "test_with_delete_sign_col"
+ sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+ sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int default "999",
+ `c3` int default "888",
+ `c4` int default "777"
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1"); """
+
+ sql """insert into ${table1} select
number,number,number,number,number from numbers("number"="11");"""
+ qt_sql_1 "select * from ${table1} order by k1;"
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+
+ sql "insert into ${table1}(k1,c3,c4,__DORIS_DELETE_SIGN__)
values(2,22,22,1),(3,33,33,1),(5,55,55,0),(6,66,66,1),(7,77,77,1),(10,1010,1010,0);"
+ qt_sql_1 "select * from ${table1} order by k1;"
+
+ sql "insert into ${table1}(k1,c1,c2,__DORIS_DELETE_SIGN__)
values(1,11,11,0),(2,22,22,0),(3,33,33,0),(6,66,66,0),(9,99,99,1);"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "sync;"
+ qt_sql_1 "select * from ${table1} order by k1;"
+
+
+ sql "truncate table ${table1};"
+ sql """insert into ${table1} select
number,number,number,number,number from numbers("number"="11");"""
+ qt_sql_2 "select * from ${table1} order by k1;"
+
+ streamLoad {
+ table "${table1}"
+ set 'column_separator', ','
+ set 'format', 'csv'
+ set 'columns', 'k1,c3,c4,del'
+ set 'partial_columns', 'true'
+ set 'merge_type', 'MERGE'
+ set 'delete', 'del=1'
+ file 'with_delete1.csv'
+ time 10000
+ }
+ qt_sql_2 "select * from ${table1} order by k1;"
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2,__DORIS_DELETE_SIGN__)
values(1,11,11,0),(2,22,22,0),(3,33,33,0),(6,66,66,0),(9,99,99,1);"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "sync;"
+ qt_sql_1 "select * from ${table1} order by k1;"
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]