This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new a8d8798e960 [fix](regression) speed up some group commit case (#40624)
a8d8798e960 is described below
commit a8d8798e960b25e49b67c115a864c02164884e98
Author: meiyi <[email protected]>
AuthorDate: Tue Sep 10 23:59:13 2024 +0800
[fix](regression) speed up some group commit case (#40624)
group commit interval is default 10s, cause the case cost too much time
---
.../load_p0/http_stream/test_group_commit_http_stream.out | 2 +-
.../load_p0/stream_load/test_group_commit_stream_load.out | 2 +-
...test_group_commit_async_wal_msg_fault_injection.groovy | 6 +++---
.../suites/insert_p0/insert_group_commit_into.groovy | 14 +++++++++-----
.../insert_group_commit_into_max_filter_ratio.groovy | 15 +--------------
.../insert_group_commit_into_unique_sync_mode.groovy | 6 +++---
.../insert_p0/insert_group_commit_with_exception.groovy | 12 ++++++------
.../insert_p0/insert_group_commit_with_large_data.groovy | 1 +
.../insert_group_commit_with_prepare_stmt.groovy | 2 ++
.../http_stream/test_group_commit_http_stream.groovy | 6 ++++--
.../test_group_commit_and_wal_back_pressure.groovy | 1 +
.../stream_load/test_group_commit_stream_load.groovy | 10 ++++++----
12 files changed, 38 insertions(+), 39 deletions(-)
diff --git
a/regression-test/data/load_p0/http_stream/test_group_commit_http_stream.out
b/regression-test/data/load_p0/http_stream/test_group_commit_http_stream.out
index 57c2525815a..e4f297347cc 100644
--- a/regression-test/data/load_p0/http_stream/test_group_commit_http_stream.out
+++ b/regression-test/data/load_p0/http_stream/test_group_commit_http_stream.out
@@ -21,5 +21,5 @@
8 f 80
-- !sql --
-2402288
+1201144
diff --git
a/regression-test/data/load_p0/stream_load/test_group_commit_stream_load.out
b/regression-test/data/load_p0/stream_load/test_group_commit_stream_load.out
index 246be06453b..1f1afae813f 100644
--- a/regression-test/data/load_p0/stream_load/test_group_commit_stream_load.out
+++ b/regression-test/data/load_p0/stream_load/test_group_commit_stream_load.out
@@ -23,5 +23,5 @@
11 a 11
-- !sql --
-2402288
+1201144
diff --git
a/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy
b/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy
index c9e22504b1b..2d0b91a01ed 100644
---
a/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy
+++
b/regression-test/suites/fault_injection_p0/test_group_commit_async_wal_msg_fault_injection.groovy
@@ -42,7 +42,7 @@
suite("test_group_commit_async_wal_msg_fault_injection","nonConcurrent") {
) engine=olap
DISTRIBUTED BY HASH(`k`)
BUCKETS 5
- properties("replication_num" = "1")
+ properties("replication_num" = "1", "group_commit_interval_ms" = "10")
"""
GetDebugPoint().clearDebugPointsForAllBEs()
@@ -79,7 +79,7 @@
suite("test_group_commit_async_wal_msg_fault_injection","nonConcurrent") {
) engine=olap
DISTRIBUTED BY HASH(`k`)
BUCKETS 5
- properties("replication_num" = "1")
+ properties("replication_num" = "1", "group_commit_interval_ms" = "10")
"""
GetDebugPoint().clearDebugPointsForAllBEs()
@@ -118,7 +118,7 @@
suite("test_group_commit_async_wal_msg_fault_injection","nonConcurrent") {
) engine=olap
DISTRIBUTED BY HASH(`k`)
BUCKETS 5
- properties("replication_num" = "1")
+ properties("replication_num" = "1", "group_commit_interval_ms" = "10")
"""
GetDebugPoint().clearDebugPointsForAllBEs()
diff --git a/regression-test/suites/insert_p0/insert_group_commit_into.groovy
b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
index dbf2bd2e18e..7af61dfc25f 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_into.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_into.groovy
@@ -116,7 +116,8 @@ suite("insert_group_commit_into") {
)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
- "replication_num" = "1"
+ "replication_num" = "1",
+ "group_commit_interval_ms" = "200"
);
"""
@@ -333,7 +334,7 @@ suite("insert_group_commit_into") {
) UNIQUE key (`teamID`,`service_id`, `start_time`)
DISTRIBUTED BY hash(`start_time`)
BUCKETS 1
- PROPERTIES ("replication_allocation" = "tag.location.default: 1")
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1",
"group_commit_interval_ms" = "200")
"""
connect(user = context.config.jdbcUser, password =
context.config.jdbcPassword, url = context.config.jdbcUrl) {
@@ -384,7 +385,8 @@ suite("insert_group_commit_into") {
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`ordernum`) BUCKETS 3
PROPERTIES (
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "group_commit_interval_ms" = "200"
);"""
sql """drop table if exists ${table_tmp};"""
sql """CREATE TABLE ${table_tmp} (
@@ -403,7 +405,8 @@ suite("insert_group_commit_into") {
COMMENT 'OLAP'
DISTRIBUTED BY HASH(`ordernum`) BUCKETS 1
PROPERTIES (
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "group_commit_interval_ms" = "200"
); """
sql """DROP MATERIALIZED VIEW IF EXISTS ods_zn_dnt_max1 ON
${table};"""
createMV("""create materialized view ods_zn_dnt_max1 as
@@ -508,7 +511,8 @@ suite("insert_group_commit_into") {
DUPLICATE KEY(`k1`)
DISTRIBUTED BY HASH(`k1`)
BUCKETS 1 PROPERTIES (
- "replication_allocation" = "tag.location.default: 1"
+ "replication_allocation" = "tag.location.default: 1",
+ "group_commit_interval_ms" = "200"
);
"""
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
index 0624f1bcf37..64ae30f8f8a 100644
---
a/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
+++
b/regression-test/suites/insert_p0/insert_group_commit_into_max_filter_ratio.groovy
@@ -41,19 +41,6 @@ suite("insert_group_commit_into_max_filter_ratio") {
}
}
- def normal_insert = { sql, expected_row_count ->
- def stmt = prepareStatement """ ${sql} """
- def result = stmt.executeUpdate()
- logger.info("insert result: " + result)
- def serverInfo = (((StatementImpl) stmt).results).getServerInfo()
- logger.info("result server info: " + serverInfo)
- if (result != expected_row_count) {
- logger.warn("insert result: " + result + ", expected_row_count: "
+ expected_row_count + ", sql: " + sql)
- }
- assertTrue(serverInfo.contains("'status':'VISIBLE'"))
- assertTrue(serverInfo.contains("'label':'label"))
- }
-
def group_commit_insert = { sql, expected_row_count ->
def stmt = prepareStatement """ ${sql} """
def result = stmt.executeUpdate()
@@ -172,7 +159,7 @@ suite("insert_group_commit_into_max_filter_ratio") {
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_num" = "1",
- "group_commit_interval_ms" = "1000"
+ "group_commit_interval_ms" = "200"
);
"""
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
b/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
index c3a1e79cba5..f58b306ab4e 100644
---
a/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
+++
b/regression-test/suites/insert_p0/insert_group_commit_into_unique_sync_mode.groovy
@@ -124,7 +124,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
"replication_num" = "1",
- "group_commit_interval_ms" = "1000"
+ "group_commit_interval_ms" = "10"
);
"""
@@ -211,7 +211,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
PROPERTIES (
"replication_num" = "1",
"function_column.sequence_col" = "score",
- "group_commit_interval_ms" = "1000"
+ "group_commit_interval_ms" = "10"
);
"""
@@ -301,7 +301,7 @@ suite("insert_group_commit_into_unique_sync_mode") {
PROPERTIES (
"replication_num" = "1",
"function_column.sequence_type" = "int",
- "group_commit_interval_ms" = "1000"
+ "group_commit_interval_ms" = "10"
);
"""
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
index 1081064d9fe..f59c9bb8b00 100644
--- a/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
+++ b/regression-test/suites/insert_p0/insert_group_commit_with_exception.groovy
@@ -20,7 +20,7 @@ import java.sql.DriverManager
import java.sql.Statement
import java.sql.PreparedStatement
-suite("insert_group_commit_with_exception", "nonConcurrent") {
+suite("insert_group_commit_with_exception") {
def table = "insert_group_commit_with_exception"
def getRowCount = { expectedRowCount ->
def retry = 0
@@ -57,6 +57,7 @@ suite("insert_group_commit_with_exception", "nonConcurrent") {
DUPLICATE KEY(`id`, `name`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
+ "group_commit_interval_ms" = "200",
"replication_num" = "1"
);
"""
@@ -65,10 +66,10 @@ suite("insert_group_commit_with_exception",
"nonConcurrent") {
if (item == "nereids") {
sql """ set enable_nereids_planner=true; """
sql """ set enable_fallback_to_original_planner=false; """
- sql "set global enable_server_side_prepared_statement = true"
+ sql "set enable_server_side_prepared_statement = true"
} else {
sql """ set enable_nereids_planner = false; """
- sql "set global enable_server_side_prepared_statement = false"
+ sql "set enable_server_side_prepared_statement = false"
}
// insert into without column
@@ -128,10 +129,10 @@ suite("insert_group_commit_with_exception",
"nonConcurrent") {
if (item == "nereids") {
statement.execute("set enable_nereids_planner=true;");
statement.execute("set
enable_fallback_to_original_planner=false;");
- sql "set global enable_server_side_prepared_statement =
true"
+ sql "set enable_server_side_prepared_statement = true"
} else {
statement.execute("set enable_nereids_planner = false;")
- sql "set global enable_server_side_prepared_statement =
false"
+ sql "set enable_server_side_prepared_statement = false"
}
// without column
try (PreparedStatement ps =
connection.prepareStatement("insert into ${table} values(?, ?, ?, ?)")) {
@@ -291,5 +292,4 @@ suite("insert_group_commit_with_exception",
"nonConcurrent") {
// try_sql("DROP TABLE ${table}")
}
}
- sql "set global enable_server_side_prepared_statement = true"
}
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
b/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
index 2af290ffc18..b66130c9e29 100644
---
a/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
+++
b/regression-test/suites/insert_p0/insert_group_commit_with_large_data.groovy
@@ -62,6 +62,7 @@ suite("insert_group_commit_with_large_data") {
DUPLICATE KEY(`id`, `name`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
+ "group_commit_interval_ms" = "40",
"replication_num" = "1"
);
"""
diff --git
a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
index 6e05513a8d6..7f2919f8118 100644
---
a/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
+++
b/regression-test/suites/insert_p0/insert_group_commit_with_prepare_stmt.groovy
@@ -144,6 +144,7 @@ suite("insert_group_commit_with_prepare_stmt") {
UNIQUE KEY(`id`, `name`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
+ "group_commit_interval_ms" = "40",
"replication_num" = "1"
);
"""
@@ -206,6 +207,7 @@ suite("insert_group_commit_with_prepare_stmt") {
DUPLICATE KEY(`id`, `name`)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
+ "group_commit_interval_ms" = "40",
"replication_num" = "1"
);
"""
diff --git
a/regression-test/suites/load_p0/http_stream/test_group_commit_http_stream.groovy
b/regression-test/suites/load_p0/http_stream/test_group_commit_http_stream.groovy
index 5f4906662d3..cb17cc82655 100644
---
a/regression-test/suites/load_p0/http_stream/test_group_commit_http_stream.groovy
+++
b/regression-test/suites/load_p0/http_stream/test_group_commit_http_stream.groovy
@@ -88,6 +88,7 @@ suite("test_group_commit_http_stream") {
)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
+ "group_commit_interval_ms" = "200",
"replication_num" = "1"
);
"""
@@ -285,6 +286,7 @@ suite("test_group_commit_http_stream") {
PARTITION p1998 VALUES [("19980101"), ("19990101")))
DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 4
PROPERTIES (
+ "group_commit_interval_ms" = "200",
"replication_num" = "1"
);
"""
@@ -307,7 +309,7 @@ suite("test_group_commit_http_stream") {
sql """ alter table ${tableName} order by (${new_columns}); """
}).start();*/
- for (int i = 0; i < 4; i++) {
+ for (int i = 0; i < 2; i++) {
streamLoad {
set 'version', '1'
@@ -334,7 +336,7 @@ suite("test_group_commit_http_stream") {
}
}
- getRowCount(2402288)
+ getRowCount(600572 * 2)
qt_sql """ select count(*) from ${tableName} """
// assertTrue(getAlterTableState())
diff --git
a/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
b/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
index bd443b6eab7..622ade420a2 100644
---
a/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_group_commit_and_wal_back_pressure.groovy
@@ -41,6 +41,7 @@ suite("test_group_commit_and_wal_back_pressure") {
UNIQUE KEY(k)
DISTRIBUTED BY HASH (k) BUCKETS 32
PROPERTIES(
+ "group_commit_interval_ms" = "100",
"replication_num" = "1"
);
"""
diff --git
a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load.groovy
b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load.groovy
index fea930f5912..be483c4dd48 100644
---
a/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_group_commit_stream_load.groovy
@@ -76,7 +76,8 @@ suite("test_group_commit_stream_load") {
)
DISTRIBUTED BY HASH(`id`) BUCKETS 1
PROPERTIES (
- "replication_num" = "1"
+ "replication_num" = "1",
+ "group_commit_interval_ms" = "200"
);
"""
@@ -250,7 +251,8 @@ suite("test_group_commit_stream_load") {
PARTITION p1998 VALUES [("19980101"), ("19990101")))
DISTRIBUTED BY HASH(`lo_orderkey`) BUCKETS 4
PROPERTIES (
- "replication_num" = "1"
+ "replication_num" = "1",
+ "group_commit_interval_ms" = "200"
);
"""
// load data
@@ -272,7 +274,7 @@ suite("test_group_commit_stream_load") {
sql """ alter table ${tableName} order by (${new_columns}); """
}).start();*/
- for (int i = 0; i < 4; i++) {
+ for (int i = 0; i < 2; i++) {
streamLoad {
table tableName
@@ -297,7 +299,7 @@ suite("test_group_commit_stream_load") {
}
}
- getRowCount(2402288)
+ getRowCount(600572 * 2)
qt_sql """ select count(*) from ${tableName} """
// assertTrue(getAlterTableState())
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
