This is an automated email from the ASF dual-hosted git repository.
HappenLee pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e2bfaa807f0 [fix](regression) Batch export_p0 generated VALUES inserts
(#65208)
e2bfaa807f0 is described below
commit e2bfaa807f09494cf5f394217ce05c11cfb68458
Author: shuke <[email protected]>
AuthorDate: Sun Jul 5 20:49:31 2026 +0800
[fix](regression) Batch export_p0 generated VALUES inserts (#65208)
## Summary
- Split large generated VALUES setup inserts in export_p0 into 500-row
batches.
- Keep the same data volume and OUTFILE validation coverage.
- Avoid single huge literal INSERT statements that can exceed the Cloud
P0 send-fragments RPC deadline during fragment preparation.
Related: DORIS-26804
## Testing
- [x] git diff --check
- [x] groovyc syntax compile for the three changed Groovy suites
- [ ] Not run Cloud P0 locally: no Doris regression cluster available in
this workspace
---
.../export_p0/outfile/native/test_outfile_native.groovy | 17 ++++++++++-------
regression-test/suites/export_p0/test_outfile.groovy | 14 +++++++-------
.../suites/export_p0/test_outfile_expr.groovy | 14 +++++++-------
3 files changed, 24 insertions(+), 21 deletions(-)
diff --git
a/regression-test/suites/export_p0/outfile/native/test_outfile_native.groovy
b/regression-test/suites/export_p0/outfile/native/test_outfile_native.groovy
index 334fc809b88..80edcdbee65 100644
--- a/regression-test/suites/export_p0/outfile/native/test_outfile_native.groovy
+++ b/regression-test/suites/export_p0/outfile/native/test_outfile_native.groovy
@@ -63,18 +63,21 @@ suite("test_outfile_native", "p0") {
DISTRIBUTED BY HASH(id) PROPERTIES("replication_num" = "1");
"""
- // Insert 10 rows of test data (the last row is all NULL)
- StringBuilder sb = new StringBuilder()
+ // Avoid a single huge VALUES statement: Cloud P0 can spend over 120s
+ // preparing the plan fragment when all 10k rows are inserted at once.
+ List<String> rows = []
int i = 1
for (; i < 10000; i ++) {
- sb.append("""
- (${i}, '2024-01-01', '2024-01-01 00:00:00', 's${i}', ${i}, ${i
% 128}, true, ${i}.${i}),
+ rows.add("""
+ (${i}, '2024-01-01', '2024-01-01 00:00:00', 's${i}', ${i}, ${i
% 128}, true, ${i}.${i})
""")
}
- sb.append("""
+ rows.add("""
(${i}, '2024-01-01', '2024-01-01 00:00:00', NULL, NULL, NULL,
NULL, NULL)
""")
- sql """ INSERT INTO ${tableName} VALUES ${sb.toString()} """
+ rows.collate(500).each { batch ->
+ sql """ INSERT INTO ${tableName} VALUES ${batch.join(",")} """
+ }
// baseline: local table query result
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY id limit
10; """
@@ -97,4 +100,4 @@ suite("test_outfile_native", "p0") {
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
}
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/export_p0/test_outfile.groovy
b/regression-test/suites/export_p0/test_outfile.groovy
index 35c5e0b681b..87908fd691d 100644
--- a/regression-test/suites/export_p0/test_outfile.groovy
+++ b/regression-test/suites/export_p0/test_outfile.groovy
@@ -86,19 +86,19 @@ suite("test_outfile") {
)
DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
"""
- StringBuilder sb = new StringBuilder()
+ List<String> rows = []
int i = 1
for (; i < 1000; i ++) {
- sb.append("""
- (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', 'Beijing', ${i}, ${i % 128}, true, ${i}, ${i}, ${i},
${i}.${i}, ${i}.${i}, 'char${i}', ${i}),
+ rows.add("""
+ (${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', 'Beijing', ${i}, ${i % 128}, true, ${i}, ${i}, ${i},
${i}.${i}, ${i}.${i}, 'char${i}', ${i})
""")
}
- sb.append("""
+ rows.add("""
(${i}, '2017-10-01', '2017-10-01 00:00:00', '2017-10-01',
'2017-10-01 00:00:00.111111', '2017-10-01 00:00:00.111111', '2017-10-01
00:00:00.111111', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL,
NULL)
""")
- sql """ INSERT INTO ${tableName} VALUES
- ${sb.toString()}
- """
+ rows.collate(500).each { batch ->
+ sql """ INSERT INTO ${tableName} VALUES ${batch.join(",")} """
+ }
qt_select_default """ SELECT * FROM ${tableName} t ORDER BY user_id;
"""
// check outfile
diff --git a/regression-test/suites/export_p0/test_outfile_expr.groovy
b/regression-test/suites/export_p0/test_outfile_expr.groovy
index 67c28f77390..fd4f1c786f8 100644
--- a/regression-test/suites/export_p0/test_outfile_expr.groovy
+++ b/regression-test/suites/export_p0/test_outfile_expr.groovy
@@ -82,19 +82,19 @@ suite("test_outfile_expr") {
)
DISTRIBUTED BY HASH(user_id) PROPERTIES("replication_num" = "1");
"""
- StringBuilder sb = new StringBuilder()
+ List<String> rows = []
int i = 1
for (; i < 1000; i ++) {
- sb.append("""
- (${i}, '2017-10-01', '2017-10-01 00:00:00', 'Beijing', ${i},
${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}', ${i}),
+ rows.add("""
+ (${i}, '2017-10-01', '2017-10-01 00:00:00', 'Beijing', ${i},
${i % 128}, true, ${i}, ${i}, ${i}, ${i}.${i}, ${i}.${i}, 'char${i}', ${i})
""")
}
- sb.append("""
+ rows.add("""
(${i}, '2017-10-01', '2017-10-01 00:00:00', NULL, NULL, NULL,
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL)
""")
- sql """ INSERT INTO ${tableName} VALUES
- ${sb.toString()}
- """
+ rows.collate(500).each { batch ->
+ sql """ INSERT INTO ${tableName} VALUES ${batch.join(",")} """
+ }
qt_select_default """ SELECT user_id+1, age+sex, repeat(char_col, 10)
FROM ${tableName} t ORDER BY user_id; """
// check outfile
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
