This is an automated email from the ASF dual-hosted git repository. suxiaogang223 pushed a commit to branch codex/forward-pick-paimon-write-master in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5871740f8917c7c9b9d2332c0357176d6b3056ac Author: suxiaogang <[email protected]> AuthorDate: Sat Sep 19 13:50:33 2026 +0800 [fix](regression) Compare Paimon partition counts across Spark versions ### What problem does this PR solve? Issue Number: close #65086 Related PR: #67395 Problem Summary: The Paimon write append-only, schema-change, and transaction suites failed in External Regression because Spark with Paimon 1.3 renders the $partitions.partition value as a struct-like string while Doris exposes a partition path. The data rows already matched. Compare the partition record-count multisets across engines and preserve exact Doris partition-path and count checks in ordered golden results, adding those checks to the transaction suite. ### Release note None ### Check List (For Author) - Test: Regression test: the three affected Paimon write suites passed locally against Paimon 1.3.1 in normal verification mode; regression framework unit tests passed (13/13). - Behavior changed: No - Does this need documentation: No --- .../paimon/write/test_paimon_write_transaction.out | 13 +++++++++++++ .../groovy/org/apache/doris/regression/suite/Suite.groovy | 9 +++++++++ .../paimon/write/test_paimon_write_append_only.groovy | 4 ++-- .../paimon/write/test_paimon_write_schema_change.groovy | 2 +- .../paimon/write/test_paimon_write_transaction.groovy | 14 ++++++++++++-- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out index 37eaca4cb5b..b2e4a106999 100644 --- a/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out +++ b/regression-test/data/external_table_p0/paimon/write/test_paimon_write_transaction.out @@ -75,6 +75,13 @@ 50 special_new a/b=c%20 2026-07-04 7 keep keep 2026-07-01 +-- !txn_static_boundary_partitions -- +region=__CUSTOM_DEFAULT_PARTITION__/dt=2026-07-01 1 +region=__CUSTOM_DEFAULT_PARTITION__/dt=2026-07-03 1 +region=a/b=c%20/dt=2026-07-04 1 +region=keep/dt=2026-07-01 1 +region=null/dt=2026-07-01 1 + -- !txn_dynamic_multi -- 10 p1_new p1 20 p2_new_a p2 @@ -82,6 +89,12 @@ 4 p3_keep p3 5 p4_keep p4 +-- !txn_dynamic_multi_partitions -- +region=p1 1 +region=p2 2 +region=p3 1 +region=p4 1 + -- !txn_dynamic_partition -- 10 east_new east 2 west_old west diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 6ef166d9459..6543dbb8d92 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -1951,6 +1951,15 @@ class Suite implements GroovyInterceptable { ResultUtils.assertSparkDorisResultEquals((args as Object[])[0] as List<List<Object>>, (args as Object[])[1] as List<List<Object>>) return null + } else if (name == "assertPaimonPartitionRecordCountsEqual") { + // Paimon 1.3 displays Spark partition values as struct-like strings, whereas Doris + // returns partition paths. Compare counts here; ordered golden results check paths. + List<List<Object>> sparkRows = (args as Object[])[0] as List<List<Object>> + List<List<Object>> dorisRows = (args as Object[])[1] as List<List<Object>> + def sparkCounts = sparkRows.collect { row -> [row[1]] }.sort { row -> row[0] } + def dorisCounts = dorisRows.collect { row -> [row[1]] }.sort { row -> row[0] } + ResultUtils.assertSparkDorisResultEquals(sparkCounts, dorisCounts) + return null } else if (name.startsWith("assert") && name.length() > "assert".length()) { // delegate to junit Assertions dynamically return Assertions."$name"(*args) // *args: spread-dot diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy index bce47f0f828..d2a25feb65a 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_append_only.groovy @@ -142,7 +142,7 @@ suite("test_paimon_write_append_only", "p0,external,paimon") { FROM t_auto_partition\$partitions ORDER BY `partition` """ - assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + assertPaimonPartitionRecordCountsEqual(sparkPartitions, dorisPartitions) order_qt_ao_auto_partition_metadata """ SELECT `partition`, record_count FROM t_auto_partition\$partitions @@ -195,7 +195,7 @@ suite("test_paimon_write_append_only", "p0,external,paimon") { FROM t_partition_default\$partitions ORDER BY `partition` """ - assertSparkDorisResultEquals(sparkDefaultPartitions, dorisDefaultPartitions) + assertPaimonPartitionRecordCountsEqual(sparkDefaultPartitions, dorisDefaultPartitions) order_qt_ao_partition_default_metadata """ SELECT `partition`, record_count FROM t_partition_default\$partitions diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy index 34195de1f5e..1b4b6de0159 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_schema_change.groovy @@ -683,7 +683,7 @@ suite("test_paimon_write_schema_change", "p0,external,paimon") { FROM `${appendTable}\$partitions` ORDER BY `partition` """ - assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + assertPaimonPartitionRecordCountsEqual(sparkPartitions, dorisPartitions) order_qt_sc_append_partitions """ SELECT `partition`, record_count FROM `${appendTable}\$partitions` diff --git a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy index a0a5fff705e..0cdbee408e9 100644 --- a/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy +++ b/regression-test/suites/external_table_p0/paimon/write/test_paimon_write_transaction.groovy @@ -319,7 +319,12 @@ suite("test_paimon_write_transaction", "p0,external,paimon") { FROM t_static_boundary\$partitions ORDER BY `partition` """ - assertSparkDorisResultEquals(sparkBoundaryPartitions, dorisBoundaryPartitions) + assertPaimonPartitionRecordCountsEqual(sparkBoundaryPartitions, dorisBoundaryPartitions) + order_qt_txn_static_boundary_partitions """ + SELECT `partition`, record_count + FROM t_static_boundary\$partitions + ORDER BY `partition` + """ // Dynamic overwrite replaces all partitions present in one input batch, // preserves untouched partitions, and publishes one overwrite snapshot. @@ -348,7 +353,12 @@ suite("test_paimon_write_transaction", "p0,external,paimon") { FROM t_dynamic_multi\$partitions ORDER BY `partition` """ - assertSparkDorisResultEquals(sparkDynamicPartitions, dorisDynamicPartitions) + assertPaimonPartitionRecordCountsEqual(sparkDynamicPartitions, dorisDynamicPartitions) + order_qt_txn_dynamic_multi_partitions """ + SELECT `partition`, record_count + FROM t_dynamic_multi\$partitions + ORDER BY `partition` + """ // FT-016: Dynamic partition overwrite replaces the partitions present in the // input while preserving existing partitions that are not touched. --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
