This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b4f03c0eb3b [charry-pick](branch-2.1) add more hudi jni test cases
#47431 (#47764)
b4f03c0eb3b is described below
commit b4f03c0eb3bfb4d8391a674a11d151dbadb22cf9
Author: Socrates <[email protected]>
AuthorDate: Tue Feb 11 22:13:53 2025 +0800
[charry-pick](branch-2.1) add more hudi jni test cases #47431 (#47764)
---
.../hudi/test_hudi_incremental.out | Bin 5687 -> 7551 bytes
.../hudi/test_hudi_schema_evolution.out | Bin 7521 -> 11234 bytes
.../external_table_p2/hudi/test_hudi_snapshot.out | Bin 696105 -> 1044838
bytes
.../hudi/test_hudi_timetravel.out | Bin 2111 -> 3119 bytes
.../hudi/test_hudi_incremental.groovy | 17 ++++++++---------
.../hudi/test_hudi_schema_evolution.groovy | 11 ++++++-----
.../hudi/test_hudi_snapshot.groovy | 15 ++++++++-------
.../hudi/test_hudi_timetravel.groovy | 15 ++++++++-------
8 files changed, 30 insertions(+), 28 deletions(-)
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out
index 50644f34961..852aebe4896 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_incremental.out differ
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
index da7273d4c14..79f36e7124c 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
and
b/regression-test/data/external_table_p2/hudi/test_hudi_schema_evolution.out
differ
diff --git a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out
index 114ca5b0a89..833ccab2ccd 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_snapshot.out differ
diff --git
a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out
index 00d15805baf..4e6d98a2aac 100644
Binary files
a/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out and
b/regression-test/data/external_table_p2/hudi/test_hudi_timetravel.out differ
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
index 845fc9ee0d9..c079d20be81 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_incremental.groovy
@@ -104,19 +104,18 @@ suite("test_hudi_incremental",
"p2,external,hudi,external_remote,external_remote
"20241114152334111",
]
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='spark';"""
+ // test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
+ // test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
+ test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
+ test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
+
+ sql """set force_jni_scanner=false;"""
test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- // disable jni scanner because the old hudi jni reader based on spark
can't read the emr hudi data
- // sql """set force_jni_scanner=true;"""
- // don't support incremental query for cow table by jni reader
- // test_hudi_incremental_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
- // test_hudi_incremental_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
- // test_hudi_incremental_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
- // test_hudi_incremental_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- // sql """set force_jni_scanner=false;"""
-
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
index 032e195f75f..50e24a3b9f5 100644
---
a/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
+++
b/regression-test/suites/external_table_p2/hudi/test_hudi_schema_evolution.groovy
@@ -35,6 +35,9 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
sql """ use regression_hudi;"""
sql """ set enable_fallback_to_original_planner=false """
+ sql """set force_jni_scanner = true;"""
+
+ sql """set hudi_jni_scanner='spark';"""
qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
// qt_deleting_simple_columns_table """ select * from
deleting_simple_columns_table order by id """
@@ -45,10 +48,9 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
// qt_deleting_complex_columns_table """ select * from
deleting_complex_columns_table order by id """
// qt_renaming_complex_columns_table """ select * from
renaming_complex_columns_table order by id """
- // disable jni scanner because the old hudi jni reader based on spark
can't read the emr hudi data
- // sql """set force_jni_scanner = true;"""
- // qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
- // qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
+ sql """set force_jni_scanner = false;"""
+ qt_adding_simple_columns_table """ select * from
adding_simple_columns_table order by id """
+ qt_altering_simple_columns_table """ select * from
altering_simple_columns_table order by id """
// qt_deleting_simple_columns_table """ select * from
deleting_simple_columns_table order by id """
// qt_renaming_simple_columns_table """ select * from
renaming_simple_columns_table order by id """
@@ -56,7 +58,6 @@ suite("test_hudi_schema_evolution",
"p2,external,hudi,external_remote,external_r
// qt_altering_complex_columns_table """ select * from
altering_complex_columns_table order by id """
// qt_deleting_complex_columns_table """ select * from
deleting_complex_columns_table order by id """
// qt_renaming_complex_columns_table """ select * from
renaming_complex_columns_table order by id """
- // sql """set force_jni_scanner = false;"""
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index 3800c962825..2b43f1e0621 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -82,18 +82,19 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
qt_q15 """SELECT user_id, array_size(purchases) AS purchase_count FROM
${table_name} ORDER BY user_id LIMIT 5;"""
}
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='spark';"""
+ test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
+ test_hudi_snapshot_querys("user_activity_log_mor_partition")
+ test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
+ test_hudi_snapshot_querys("user_activity_log_cow_partition")
+
+ sql """set force_jni_scanner=false;"""
test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
test_hudi_snapshot_querys("user_activity_log_mor_partition")
test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
test_hudi_snapshot_querys("user_activity_log_cow_partition")
- // disable jni scanner because the old hudi jni reader based on spark
can't read the emr hudi data
- // sql """set force_jni_scanner=true;"""
- // test_hudi_snapshot_querys("user_activity_log_mor_non_partition")
- // test_hudi_snapshot_querys("user_activity_log_mor_partition")
- // test_hudi_snapshot_querys("user_activity_log_cow_non_partition")
- // test_hudi_snapshot_querys("user_activity_log_cow_partition")
- // sql """set force_jni_scanner=false;"""
sql """drop catalog if exists ${catalog_name};"""
}
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
index 812ae578066..0beb2d6a6a9 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_timetravel.groovy
@@ -98,18 +98,19 @@ suite("test_hudi_timetravel",
"p2,external,hudi,external_remote,external_remote_
"20241114152334111",
]
+ sql """set force_jni_scanner=true;"""
+ sql """set hudi_jni_scanner='spark';"""
+ test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
+ test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
+ test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
+ test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
+
+ sql """set force_jni_scanner=false;"""
test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- // disable jni scanner because the old hudi jni reader based on spark
can't read the emr hudi data
- // sql """set force_jni_scanner=true;"""
- // test_hudi_timetravel_querys("user_activity_log_cow_non_partition",
timestamps_cow_non_partition)
- // test_hudi_timetravel_querys("user_activity_log_cow_partition",
timestamps_cow_partition)
- // test_hudi_timetravel_querys("user_activity_log_mor_non_partition",
timestamps_mor_non_partition)
- // test_hudi_timetravel_querys("user_activity_log_mor_partition",
timestamps_mor_partition)
- // sql """set force_jni_scanner=false;"""
sql """drop catalog if exists ${catalog_name};"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
