This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 21802294a81 branch-3.0: [Opt](partial update) Add some cases for
partial update #47900 (#48129)
21802294a81 is described below
commit 21802294a81985dbb1b33b6951cd1e1bc4906aa0
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Feb 20 15:22:06 2025 +0800
branch-3.0: [Opt](partial update) Add some cases for partial update #47900
(#48129)
Cherry-picked from #47900
Co-authored-by: bobhan1 <[email protected]>
---
be/src/olap/base_tablet.cpp | 2 +
.../test_partial_update_publish_seq.out | Bin 0 -> 1714 bytes
.../test_partial_update_default_value.out | Bin 259 -> 1003 bytes
.../test_partial_update_lookup_row_key.out | Bin 0 -> 779 bytes
.../test_partial_update_row_store.out | Bin 0 -> 847 bytes
.../test_partial_update_publish_seq.groovy | 180 +++++++++++++++++++++
.../test_partial_update_default_value.groovy | 25 ++-
.../test_partial_update_lookup_row_key.groovy | 81 ++++++++++
.../test_partial_update_row_store.groovy | 103 ++++++++++++
9 files changed, 390 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 25398e84346..895899b9ca6 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -707,6 +707,8 @@ Status
BaseTablet::calc_segment_delete_bitmap(RowsetSharedPtr rowset,
row_id);
++conflict_rows;
continue;
+ // NOTE: for partial update which doesn't specify the sequence
column, we can't use the sequence column value filled in flush phase
+ // as its final value. Otherwise it may cause inconsistency
between replicas.
}
if (is_partial_update && rowset_writer != nullptr) {
// In publish version, record rows to be deleted for
concurrent update
diff --git
a/regression-test/data/fault_injection_p0/partial_update/test_partial_update_publish_seq.out
b/regression-test/data/fault_injection_p0/partial_update/test_partial_update_publish_seq.out
new file mode 100644
index 00000000000..a7fa43e0ad9
Binary files /dev/null and
b/regression-test/data/fault_injection_p0/partial_update/test_partial_update_publish_seq.out
differ
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_default_value.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_default_value.out
index edd3326a752..231696c1d6e 100644
Binary files
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_default_value.out
and
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_default_value.out
differ
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.out
new file mode 100644
index 00000000000..2250bd3535a
Binary files /dev/null and
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.out
differ
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_row_store.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_row_store.out
new file mode 100644
index 00000000000..59b96bcbd7c
Binary files /dev/null and
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_row_store.out
differ
diff --git
a/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_seq.groovy
b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_seq.groovy
new file mode 100644
index 00000000000..19639998da4
--- /dev/null
+++
b/regression-test/suites/fault_injection_p0/partial_update/test_partial_update_publish_seq.groovy
@@ -0,0 +1,180 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite("test_partial_update_publish_seq", "nonConcurrent") {
+
+ def enable_block_in_publish = {
+ if (isCloudMode()) {
+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait")
+
GetDebugPoint().enableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block")
+ } else {
+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait")
+
GetDebugPoint().enableDebugPointForAllBEs("EnginePublishVersionTask::execute.block")
+ }
+ }
+
+ def disable_block_in_publish = {
+ if (isCloudMode()) {
+
GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.enable_spin_wait")
+
GetDebugPoint().disableDebugPointForAllFEs("CloudGlobalTransactionMgr.getDeleteBitmapUpdateLock.block")
+ } else {
+
GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.enable_spin_wait")
+
GetDebugPoint().disableDebugPointForAllBEs("EnginePublishVersionTask::execute.block")
+ }
+ }
+
+ def inspect_rows = { sqlStr ->
+ sql "set skip_delete_sign=true;"
+ sql "set skip_delete_bitmap=true;"
+ sql "sync"
+ qt_inspect sqlStr
+ sql "set skip_delete_sign=false;"
+ sql "set skip_delete_bitmap=false;"
+ sql "sync"
+ }
+
+
+ try {
+ GetDebugPoint().clearDebugPointsForAllFEs()
+ GetDebugPoint().clearDebugPointsForAllBEs()
+
+ def table1 = "test_partial_update_publish_seq_map"
+ sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+ sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int,
+ `c4` int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "function_column.sequence_col" = "c1",
+ "replication_num" = "1"); """
+
+ sql "insert into ${table1} values(1,1,1,1,1),(2,2,2,2,2),(3,3,3,3,3);"
+ sql "sync;"
+ qt_seq_map_0 "select * from ${table1} order by k1;"
+
+
+ // with seq map val, >/=/< conflicting seq val
+ enable_block_in_publish()
+ def t1 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2)
values(1,10,99),(2,10,99),(3,10,99);"
+ }
+ Thread.sleep(500)
+ def t2 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c3)
values(1,20,88),(2,10,88),(3,5,88);"
+ }
+ Thread.sleep(1000)
+ disable_block_in_publish()
+ t1.join()
+ t2.join()
+ qt_seq_map_1 "select * from ${table1} order by k1;"
+ inspect_rows "select
*,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__ from
${table1} order by k1,__DORIS_VERSION_COL__;"
+
+ // without seq map val, the filled seq val >/=/< conflicting seq val
+ enable_block_in_publish()
+ t1 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2)
values(1,9,77),(2,10,77),(3,50,77);"
+ }
+ Thread.sleep(500)
+ t2 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c4) values(1,33),(2,33),(3,33);"
+ }
+ Thread.sleep(1000)
+ disable_block_in_publish()
+ t1.join()
+ t2.join()
+ qt_seq_map_2 "select * from ${table1} order by k1;"
+ inspect_rows "select
*,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__ from
${table1} order by k1,__DORIS_VERSION_COL__;"
+
+ // with delete sign and seq col val, >/=/< conflicting seq val
+ enable_block_in_publish()
+ t1 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2)
values(1,80,66),(2,100,66),(3,120,66);"
+ }
+ Thread.sleep(500)
+ t2 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,__DORIS_DELETE_SIGN__)
values(1,100,1),(2,100,1),(3,100,1);"
+ }
+ Thread.sleep(1000)
+ disable_block_in_publish()
+ t1.join()
+ t2.join()
+ qt_seq_map_3 "select * from ${table1} order by k1;"
+ inspect_rows "select
*,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__ from
${table1} order by k1,__DORIS_VERSION_COL__;"
+
+
+ sql "truncate table ${table1};"
+ sql "insert into ${table1}
values(1,10,1,1,1),(2,10,2,2,2),(3,10,3,3,3);"
+ sql "sync;"
+ // with delete sign and without seq col val, >/=/< conflicting seq val
+ enable_block_in_publish()
+ t1 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2)
values(1,20,55),(2,100,55),(3,120,55);"
+ }
+ Thread.sleep(500)
+ t2 = Thread.start {
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c4,__DORIS_DELETE_SIGN__)
values(1,100,1),(2,100,1),(3,100,1);"
+ }
+ Thread.sleep(1000)
+ disable_block_in_publish()
+ t1.join()
+ t2.join()
+ qt_seq_map_4 "select * from ${table1} order by k1;"
+ inspect_rows "select
*,__DORIS_DELETE_SIGN__,__DORIS_SEQUENCE_COL__,__DORIS_VERSION_COL__ from
${table1} order by k1,__DORIS_VERSION_COL__;"
+
+
+ } catch(Exception e) {
+ logger.info(e.getMessage())
+ throw e
+ } finally {
+ GetDebugPoint().clearDebugPointsForAllFEs()
+ GetDebugPoint().clearDebugPointsForAllBEs()
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
index 28d1d0ed42d..fb12c840280 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_default_value.groovy
@@ -68,8 +68,31 @@ suite("test_primary_key_partial_update_default_value", "p0")
{
select * from ${tableName} order by id;
"""
- // drop drop
+ // test special default values
+ tableName = "test_primary_key_partial_update_default_value2"
+ // create table
sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """ CREATE TABLE ${tableName} (
+ k int,
+ c1 int,
+ c2 bitmap NOT NULL DEFAULT bitmap_empty,
+ c3 double DEFAULT PI,
+ c4 double DEFAULT E,
+ c5 array<int> NOT NULL DEFAULT "[]"
+ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES("replication_num" = "1",
"enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${tableName}(k,c1) values(1,1),(2,2),(3,3);"
+ sql "sync;"
+ qt_sql "select k,c1,bitmap_to_string(c2),c3,c4,ARRAY_SIZE(c5) from
${tableName} order by k;"
+
+ sql "insert into ${tableName}(k,c1)
values(1,10),(2,20),(4,40),(5,50);"
+ sql "sync;"
+ qt_sql "select k,c1,bitmap_to_string(c2),c3,c4,ARRAY_SIZE(c5) from
${tableName} order by k;"
}
}
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.groovy
new file mode 100644
index 00000000000..dfd8f43a6b3
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_lookup_row_key.groovy
@@ -0,0 +1,81 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_lookup_row_key", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect( context.config.jdbcUser, context.config.jdbcPassword,
context.config.jdbcUrl) {
+ sql "use ${db};"
+ sql "sync;"
+
+ def tableName = "test_partial_update_publish_conflict_seq"
+ sql """ DROP TABLE IF EXISTS ${tableName} force;"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` int(11) NULL,
+ `v1` BIGINT NULL,
+ `v2` BIGINT NULL,
+ `v3` BIGINT NULL,
+ `v4` BIGINT NULL,
+ ) UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "disable_auto_compaction" = "true",
+ "function_column.sequence_col" = "v1",
+ "store_row_column" = "${use_row_store}"); """
+
+ sql """ insert into ${tableName} values
+ (1,400,1,1,1),(2,100,2,2,2),(3,30,3,3,3),(4,300,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,100,1,1,1),(2,400,2,2,2),(3,100,3,3,3),(4,200,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,200,1,1,1),(2,200,2,2,2),(3,300,3,3,3),(4,400,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,300,1,1,1),(2,300,2,2,2),(3,400,3,3,3),(4,100,4,4,4);"""
+ qt_1 "select * from ${tableName} order by k;"
+ // lookup_row_key will find key rowset with highest version to
lowest version
+ // the index of valid segment for each key will be in the search
seqeuence
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${tableName}(k,v2)
values(1,99),(2,99),(3,99),(4,99),(5,99),(6,99);"
+ qt_1 "select *,__DORIS_SEQUENCE_COL__ from ${tableName} order by
k;"
+
+
+ sql "truncate table ${tableName};"
+ sql """ insert into ${tableName} values
+ (1,400,1,1,1),(2,100,2,2,2),(3,30,3,3,3),(4,300,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,100,1,1,1),(2,400,2,2,2),(3,100,3,3,3),(4,200,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,200,1,1,1),(2,200,2,2,2),(3,300,3,3,3),(4,400,4,4,4);"""
+ sql """ insert into ${tableName} values
+ (1,300,1,1,1),(2,300,2,2,2),(3,400,3,3,3),(4,100,4,4,4);"""
+ qt_2 "select * from ${tableName} order by k;"
+
+ sql "insert into ${tableName}(k,v1,v3)
values(1,500,88),(2,500,88),(3,300,88),(4,200,88),(5,200,88),(6,200,88);"
+ qt_2 "select *,__DORIS_SEQUENCE_COL__ from ${tableName} order by
k;"
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_row_store.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_row_store.groovy
new file mode 100644
index 00000000000..30102f9064d
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_row_store.groovy
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.junit.Assert
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite("test_partial_update_row_store", "nonConcurrent") {
+
+ def table1 = "test_partial_update_row_store"
+ sql "DROP TABLE IF EXISTS ${table1} FORCE;"
+ sql """ CREATE TABLE IF NOT EXISTS ${table1} (
+ `k1` int NOT NULL,
+ `c1` int,
+ `c2` int,
+ c3 int
+ )UNIQUE KEY(k1)
+ DISTRIBUTED BY HASH(k1) BUCKETS 1
+ PROPERTIES (
+ "enable_mow_light_delete" = "false",
+ "disable_auto_compaction" = "true",
+ "replication_num" = "1",
+ "store_row_column" = "false"); """
+
+ sql "insert into ${table1} values(1,1,1,1),(2,2,2,2),(3,3,3,3);"
+ sql "insert into ${table1} values(4,4,4,4),(5,5,5,5),(6,6,6,6);"
+ sql "sync;"
+ qt_1 "select * from ${table1} order by k1;"
+
+ def doSchemaChange = { cmd ->
+ sql cmd
+ waitForSchemaChangeDone {
+ sql """SHOW ALTER TABLE COLUMN WHERE IndexName='${table1}' ORDER
BY createtime DESC LIMIT 1"""
+ time 2000
+ }
+ }
+
+ // turn on row_store_column, but only store part of columns
+ doSchemaChange """alter table ${table1} set ("store_row_column" =
"true")"""
+ doSchemaChange """alter table ${table1} set ("row_store_columns" =
"k1,c2")"""
+ sql "insert into ${table1} values(7,7,7,7),(8,8,8,8),(9,9,9,9);"
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c1,c2)
values(1,10,10),(2,20,20),(5,50,50),(7,70,70),(100,100,100);"
+ qt_2 "select *, LENGTH(__DORIS_ROW_STORE_COL__) from ${table1} order by
k1;"
+ sql "insert into ${table1}(k1,c3)
values(1,99),(3,99),(6,99),(8,99),(200,200);"
+ qt_2 "select *, LENGTH(__DORIS_ROW_STORE_COL__) from ${table1} order by
k1;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "sync;"
+
+
+ sql "truncate table ${table1};"
+ sql "insert into ${table1} values(1,1,1,1),(2,2,2,2),(3,3,3,3);"
+ sql "insert into ${table1} values(4,4,4,4),(5,5,5,5),(6,6,6,6);"
+ sql "sync;"
+ qt_3 "select * from ${table1} order by k1;"
+
+
+ // turn on full row store column
+ doSchemaChange """alter table ${table1} set ("store_row_column" =
"true")"""
+ sql "insert into ${table1} values(7,7,7,7),(8,8,8,8),(9,9,9,9);"
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c2)
values(2,777),(3,777),(10,777),(21,777),(8,777);"
+ qt_4 "select *,LENGTH(__DORIS_ROW_STORE_COL__) from ${table1} order by k1;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "sync;"
+
+ // from row store to part columns row store
+ doSchemaChange """alter table ${table1} set ("row_store_columns" =
"k1,c2")"""
+ sql "insert into ${table1} values(11,11,11,11),(20,20,20,20);"
+
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+ sql "insert into ${table1}(k1,c3) values(1,987),(2,987),(11,987),(22,987);"
+ qt_4 "select *,LENGTH(__DORIS_ROW_STORE_COL__) from ${table1} order by k1;"
+ sql "set enable_unique_key_partial_update=false;"
+ sql "set enable_insert_strict=true;"
+ sql "sync;"
+
+ // Can not alter store_row_column from true to false currently, should add
related case if supported
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
