This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cd0cd55f11d [opt](auto-inc) Allow to miss auto-increment column and
other value columns in partial update (#44528)
cd0cd55f11d is described below
commit cd0cd55f11d21a9bf25a02c1567e378c1f672e07
Author: bobhan1 <[email protected]>
AuthorDate: Fri Nov 29 20:52:54 2024 +0800
[opt](auto-inc) Allow to miss auto-increment column and other value columns
in partial update (#44528)
Previously, if a partial update misses auto-increment key columns and
other value columns, the load will fail with error `distributed column
not found, column=xxx`. Considering the uniqueness of the generated
value of auto-increment column, this PR converts the partial update load
to upsert in this situation to make it llegal.
### Release note
Allow to miss auto-increment key column and other value columns in
partial update load
---
.../trees/plans/commands/insert/InsertUtils.java | 10 ++++++
.../test_partial_update_auto_inc.out | 8 +++++
.../test_partial_update_auto_inc.groovy | 42 +++++++++++++++++++++-
3 files changed, 59 insertions(+), 1 deletion(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
index 6b301c16f73..60e7e5bf805 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java
@@ -297,6 +297,7 @@ public class InsertUtils {
+ " with sync materialized view.");
}
boolean hasMissingColExceptAutoIncKey = false;
+ boolean hasMissingAutoIncKey = false;
for (Column col : olapTable.getFullSchema()) {
Optional<String> insertCol =
unboundLogicalSink.getColNames().stream()
.filter(c ->
c.equalsIgnoreCase(col.getName())).findFirst();
@@ -313,9 +314,18 @@ public class InsertUtils {
if (!(col.isAutoInc() && col.isKey()) &&
!insertCol.isPresent() && col.isVisible()) {
hasMissingColExceptAutoIncKey = true;
}
+ if (col.isAutoInc() && col.isKey() &&
!insertCol.isPresent()) {
+ hasMissingAutoIncKey = true;
+ }
}
if (!hasMissingColExceptAutoIncKey) {
((UnboundTableSink<? extends Plan>)
unboundLogicalSink).setPartialUpdate(false);
+ } else {
+ if (hasMissingAutoIncKey) {
+ // becuase of the uniqueness of genetaed
value of auto-increment column,
+ // we convert this load to upsert when is
misses auto-increment key column
+ ((UnboundTableSink<? extends Plan>)
unboundLogicalSink).setPartialUpdate(false);
+ }
}
}
}
diff --git
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
index 19b192c95fa..b672eb10570 100644
---
a/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
+++
b/regression-test/data/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.out
@@ -35,3 +35,11 @@ doris9
3 888 888 30
4 40 40 40
+-- !sql --
+test1 15
+test2 29
+test3 49
+
+-- !sql --
+3
+
diff --git
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
index 1ec60fbb10e..d689deeb8af 100644
---
a/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
+++
b/regression-test/suites/unique_with_mow_p0/partial_update/test_partial_update_auto_inc.groovy
@@ -103,6 +103,46 @@ suite("test_partial_update_auto_inc") {
time 10000
}
order_qt_select_6 "select * from
test_primary_key_partial_update_auto_inc2"
- sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc2
"""
+
+
+ sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc3
force; """
+ sql """ create table test_primary_key_partial_update_auto_inc3
+ (
+ `id` bigint not null
AUTO_INCREMENT,
+ `project_code` varchar(20) not null,
+ `period_num` int,
+ `c2` int
+ ) unique KEY(`id`)
+ DISTRIBUTED BY HASH(`id`) BUCKETS auto
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "enable_unique_key_merge_on_write" = "true"
+ ); """
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+
+ sql "insert into
test_primary_key_partial_update_auto_inc3(project_code,period_num) values
('test1',15),('test2',29),('test3',49);"
+ qt_sql "select project_code,period_num from
test_primary_key_partial_update_auto_inc3 order by project_code,period_num;"
+ qt_sql "select count(distinct id) from
test_primary_key_partial_update_auto_inc3;"
+
+
+ sql """ DROP TABLE IF EXISTS test_primary_key_partial_update_auto_inc4
"""
+ sql """ CREATE TABLE test_primary_key_partial_update_auto_inc4 (
+ `k1` BIGINT NOT NULL AUTO_INCREMENT,
+ `k2` int,
+ `c1` int,
+ `c2` int,
+ `c3` int)
+ UNIQUE KEY(`k1`,`k2`) DISTRIBUTED BY HASH(`k1`,`k2`)
BUCKETS 1
+ PROPERTIES("replication_num" = "1",
"enable_unique_key_merge_on_write" = "true"); """
+ sql "set enable_unique_key_partial_update=true;"
+ sql "set enable_insert_strict=false;"
+ sql "sync;"
+
+ test {
+ sql "insert into test_primary_key_partial_update_auto_inc4(c1,c2)
values(1,1),(2,2),(3,3)"
+ exception "Partial update should include all key columns, missing:
k2"
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
