This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ea4d166edbe [branch-2.1] Picks "[opt](partial update) Allow to only
specify key columns in partial update #40736" (#40863)
ea4d166edbe is described below
commit ea4d166edbe995a2c1c362d3d7541b27f029a6b6
Author: bobhan1 <[email protected]>
AuthorDate: Sun Sep 15 18:57:43 2024 +0800
[branch-2.1] Picks "[opt](partial update) Allow to only specify key columns
in partial update #40736" (#40863)
picks https://github.com/apache/doris/pull/40736
---
be/src/olap/rowset/segment_v2/segment_writer.cpp | 7 +--
.../rowset/segment_v2/vertical_segment_writer.cpp | 2 +-
.../schema_change/load_with_key_column.csv | 2 +-
.../test_partial_update_only_keys.out | 59 ++++++++++++++++++++
...artial_update_insert_light_schema_change.groovy | 4 +-
...test_partial_update_insert_schema_change.groovy | 4 +-
.../test_partial_update_only_keys.groovy | 62 ++++++++++++++++++++++
.../test_partial_update_schema_change.groovy | 8 +--
...t_partial_update_schema_change_row_store.groovy | 8 +--
9 files changed, 138 insertions(+), 18 deletions(-)
diff --git a/be/src/olap/rowset/segment_v2/segment_writer.cpp
b/be/src/olap/rowset/segment_v2/segment_writer.cpp
index b9a46ec97ff..5028b596677 100644
--- a/be/src/olap/rowset/segment_v2/segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_writer.cpp
@@ -359,13 +359,8 @@ void
SegmentWriter::_serialize_block_to_row_column(vectorized::Block& block) {
// 3. set columns to data convertor and then write all columns
Status SegmentWriter::append_block_with_partial_content(const
vectorized::Block* block,
size_t row_pos, size_t
num_rows) {
- if constexpr (!std::is_same_v<ExecEnv::Engine, StorageEngine>) {
- // TODO(plat1ko): cloud mode
- return Status::NotSupported("append_block_with_partial_content");
- }
-
auto* tablet = static_cast<Tablet*>(_tablet.get());
- if (block->columns() <= _tablet_schema->num_key_columns() ||
+ if (block->columns() < _tablet_schema->num_key_columns() ||
block->columns() >= _tablet_schema->num_columns()) {
return Status::InternalError(
fmt::format("illegal partial update block columns: {}, num key
columns: {}, total "
diff --git a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
index c2418194ed6..4ee0decabd2 100644
--- a/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
+++ b/be/src/olap/rowset/segment_v2/vertical_segment_writer.cpp
@@ -693,7 +693,7 @@ Status VerticalSegmentWriter::batch_block(const
vectorized::Block* block, size_t
_opts.rowset_ctx->partial_update_info->is_partial_update &&
_opts.write_type == DataWriteType::TYPE_DIRECT &&
!_opts.rowset_ctx->is_transient_rowset_writer) {
- if (block->columns() <= _tablet_schema->num_key_columns() ||
+ if (block->columns() < _tablet_schema->num_key_columns() ||
block->columns() >= _tablet_schema->num_columns()) {
return Status::InternalError(fmt::format(
"illegal partial update block columns: {}, num key
columns: {}, total "
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
index 128ecf3838d..75456df9f6c 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
+++
b/regression-test/data/unique_with_mow_p0/partial_update/schema_change/load_with_key_column.csv
@@ -1 +1 @@
-1, 1
\ No newline at end of file
+2, 2
\ No newline at end of file
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
new file mode 100644
index 00000000000..9ec27274af4
--- /dev/null
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_only_keys.out
@@ -0,0 +1,59 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
+-- !sql --
+0 0 0 0
+1 1 1 1
+2 2 2 2
+4 \N \N \N
+5 \N \N \N
+6 \N \N \N
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
index 6f05843138b..4d09757122d 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_light_schema_change.groovy
@@ -273,11 +273,11 @@ suite("test_partial_update_insert_light_schema_change",
"p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
index 55a9786afe8..395b83a60da 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_insert_schema_change.groovy
@@ -258,11 +258,11 @@ suite("test_partial_update_insert_schema_change", "p0") {
sql "sync"
// test insert data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
sql "set enable_unique_key_partial_update=true;"
test {
sql "insert into ${tableName}(c0,c1) values(1, 1);"
- exception "INTERNAL_ERROR"
+ exception "Insert has filtered data in strict mode"
}
sql "insert into ${tableName}(c0,c1,c2) values(1,0,10);"
sql "set enable_unique_key_partial_update=false;"
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
new file mode 100644
index 00000000000..1f56643b6e8
--- /dev/null
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_only_keys.groovy
@@ -0,0 +1,62 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_partial_update_only_keys", "p0") {
+
+ String db = context.config.getDbNameByFile(context.file)
+ sql "select 1;" // to create database
+
+ for (def use_row_store : [false, true]) {
+ logger.info("current params: use_row_store: ${use_row_store}")
+
+ connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
+ sql "use ${db};"
+ def tableName = "test_primary_key_partial_update"
+ sql """ DROP TABLE IF EXISTS ${tableName} force"""
+ sql """ CREATE TABLE ${tableName} (
+ `k` BIGINT NOT NULL,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k`) DISTRIBUTED BY HASH(`k`) BUCKETS 1
+ PROPERTIES(
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true",
+ "store_row_column" = "${use_row_store}"); """
+ sql """insert into ${tableName} select number,number,number,number
from numbers("number"="3");"""
+ qt_sql """select * from ${tableName} order by k;"""
+ // new rows will be appended
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+
+ // fail if has new rows
+ sql "set enable_insert_strict=true;"
+ sql "sync"
+ sql "insert into ${tableName}(k) values(0),(1),(4),(5),(6);"
+ qt_sql """select * from ${tableName} order by k;"""
+ test {
+ sql "insert into ${tableName}(k) values(0),(1),(10),(11);"
+ exception "Insert has filtered data in strict mode"
+ }
+ qt_sql """select * from ${tableName} order by k;"""
+ }
+ }
+}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
index 0ed7c042abf..c39dbe651cf 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change.groovy
@@ -422,12 +422,13 @@ suite("test_partial_update_schema_change", "p0") {
}
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -441,7 +442,7 @@ suite("test_partial_update_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1021,6 +1022,7 @@ suite("test_partial_update_schema_change", "p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1034,7 +1036,7 @@ suite("test_partial_update_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
index ef57c3fec40..6803d75d840 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_schema_change_row_store.groovy
@@ -426,12 +426,13 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
}
// test load data with all key column, should fail because
- // it don't have any value columns
+ // it inserts a new row in strict mode
streamLoad {
table "${tableName}"
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -445,7 +446,7 @@ suite("test_partial_update_row_store_schema_change", "p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
@@ -1031,6 +1032,7 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
set 'column_separator', ','
set 'partial_columns', 'true'
+ set 'strict_mode', 'true'
set 'columns', 'c0, c1'
file 'schema_change/load_with_key_column.csv'
@@ -1044,7 +1046,7 @@ suite("test_partial_update_row_store_schema_change",
"p0") {
def json = parseJson(result)
assertEquals("fail", json.Status.toLowerCase())
assertEquals(1, json.NumberTotalRows)
- assertEquals(0, json.NumberFilteredRows)
+ assertEquals(1, json.NumberFilteredRows)
assertEquals(0, json.NumberUnselectedRows)
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
