This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ebc63eabaaa [fix](test)(jdbc) fix some external regression test cases
and jdbc bug (#52817)
ebc63eabaaa is described below
commit ebc63eabaaaafbd26987d739311550d56d7c09fa
Author: Mingyu Chen (Rayner) <morning...@163.com>
AuthorDate: Sun Jul 6 12:34:44 2025 +0800
[fix](test)(jdbc) fix some external regression test cases and jdbc bug
(#52817)
Followup #52561
Also fix a bug introduced by #51471, which lost the function rules in
jdbc external table.
---
.../main/java/org/apache/doris/catalog/JdbcTable.java | 16 ++++++++++++++--
.../doris/datasource/iceberg/IcebergMetadataOps.java | 2 ++
.../org/apache/doris/journal/bdbje/BDBJEJournal.java | 2 +-
.../iceberg/test_iceberg_hadoop_case_sensibility.out | Bin 3792 -> 3876 bytes
.../iceberg/test_iceberg_rest_case_sensibility.out | Bin 3712 -> 3796 bytes
.../test_iceberg_hadoop_case_sensibility.groovy | 2 +-
.../iceberg/test_iceberg_rest_case_sensibility.groovy | 2 +-
.../iceberg/write/test_iceberg_create_table.groovy | 4 +---
.../external_table_p0/tvf/test_file_tvf_hdfs.groovy | 10 +++++-----
.../suites/external_table_p0/tvf/upgrade/load.groovy | 12 ++++++------
.../suites/external_table_p0/tvf/upgrade/test.groovy | 6 +++---
.../mv/external_table/part_partition_invalid.groovy | 7 ++-----
.../mv/external_table/single_external_table.groovy | 6 ++----
13 files changed, 38 insertions(+), 31 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
index 5c5d8e39a4b..738a96c07c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
@@ -36,6 +36,7 @@ import org.apache.commons.collections.map.CaseInsensitiveMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -100,6 +101,9 @@ public class JdbcTable extends Table {
private boolean connectionPoolKeepAlive;
private ExternalFunctionRules functionRules;
+ // This is used for edit log
+ @SerializedName("frs")
+ private String functionRulesString;
static {
Map<String, TOdbcTableType> tempMap = new CaseInsensitiveMap();
@@ -381,8 +385,9 @@ public class JdbcTable extends Table {
private void checkAndSetExternalFunctionRules(Map<String, String>
properties) throws DdlException {
ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES,
""));
- this.functionRules = ExternalFunctionRules.create(jdbcTypeName,
- properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""));
+ String functionRulesString =
properties.getOrDefault(JdbcResource.FUNCTION_RULES, "");
+ this.functionRules = ExternalFunctionRules.create(jdbcTypeName,
functionRulesString);
+ this.functionRulesString = functionRulesString;
}
/**
@@ -491,4 +496,11 @@ public class JdbcTable extends Table {
public ExternalFunctionRules getExternalFunctionRules() {
return functionRules;
}
+
+ @Override
+ public void gsonPostProcess() throws IOException {
+ super.gsonPostProcess();
+ functionRules = ExternalFunctionRules.create(jdbcTypeName,
Strings.nullToEmpty(functionRulesString));
+ }
}
+
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
index 059e12376fb..1fcc7083258 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java
@@ -339,6 +339,8 @@ public class IcebergMetadataOps implements
ExternalMetadataOps {
if (db.isPresent()) {
db.get().setUnInitialized(true);
}
+ LOG.info("after drop table {}.{}.{}. is db present: {}",
+ dorisCatalog.getName(), dbName, tblName, db.isPresent());
}
private void performDropTable(String remoteDbName, String remoteTblName,
boolean ifExists) throws DdlException {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
index 2a6a5b201e0..f03fac85ddb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/journal/bdbje/BDBJEJournal.java
@@ -248,7 +248,7 @@ public class BDBJEJournal implements Journal { //
CHECKSTYLE IGNORE THIS LINE: B
MetricRepo.COUNTER_CURRENT_EDIT_LOG_SIZE_BYTES.increase((long)
theData.getSize());
}
if (LOG.isDebugEnabled() || theData.getSize() > (1 << 20)) {
- LOG.info("opCode = {}, journal size = {}", op, theData.getSize());
+ LOG.info("opCode = {}, journal size = {}, log id: {}", op,
theData.getSize(), id);
if (MetricRepo.isInit) {
MetricRepo.COUNTER_LARGE_EDIT_LOG.increase(1L);
}
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out
index 2b0147259bb..953e628f309 100644
Binary files
a/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out
and
b/regression-test/data/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.out
differ
diff --git
a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out
b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out
index d6d88db3dbe..c6379666096 100644
Binary files
a/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out
and
b/regression-test/data/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.out
differ
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy
index ee8ee903e1f..26a35b86cee 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_hadoop_case_sensibility.groovy
@@ -278,7 +278,7 @@ suite("test_iceberg_hadoop_case_sensibility",
"p0,external,doris,external_docker
exception "Table [case_tbl22] does not exist in database
[iceberg_hadoop_case_db2]"
}
sql """create table iceberg_hadoop_case_db2.case_tbl22 (k1
int);""" // recreate
- sql """show tables from iceberg_hadoop_case_db2 like
"%case_tbl22%""""
+ qt_sql_show """show tables from iceberg_hadoop_case_db2 like
"%case_tbl22%""""
sql """insert into iceberg_hadoop_case_db2.case_tbl22 values(5);"""
order_qt_sql21 """select * from
iceberg_hadoop_case_db2.case_tbl22;"""
diff --git
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy
index 22161de615d..510b2c00362 100644
---
a/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/test_iceberg_rest_case_sensibility.groovy
@@ -279,7 +279,7 @@ suite("test_iceberg_rest_case_sensibility",
"p0,external,doris,external_docker,e
exception "Table [case_tbl22] does not exist in database
[iceberg_rest_case_db2]"
}
sql """create table iceberg_rest_case_db2.case_tbl22 (k1 int);"""
// recreate
- sql """show tables from iceberg_rest_case_db2 like
"%case_tbl22%""""
+ qt_sql_show """show tables from iceberg_rest_case_db2 like
"%case_tbl22%""""
sql """insert into iceberg_rest_case_db2.case_tbl22 values(5);"""
order_qt_sql21 """select * from
iceberg_rest_case_db2.case_tbl22;"""
diff --git
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy
index 22364b8bc96..e7b2bb9bafc 100644
---
a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy
+++
b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_create_table.groovy
@@ -52,9 +52,7 @@ suite("test_iceberg_create_table",
"p0,external,doris,external_docker,external_d
String tb1 = db1 + "_tb1"
String tb2 = db1 + "_tb2"
- sql """ drop table if exists ${db1}.${tb1} """
- sql """ drop table if exists ${db1}.${tb2} """
- sql """ drop database if exists ${db1} """
+ sql """ drop database if exists ${db1} force"""
sql """ create database ${db1} """
diff --git
a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
index d5d9f3ca032..b2704a2cf46 100644
--- a/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
+++ b/regression-test/suites/external_table_p0/tvf/test_file_tvf_hdfs.groovy
@@ -333,9 +333,9 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
// test create view from tvf and alter view from tvf
uri = "${defaultFS}" +
"/user/doris/preinstalled_data/csv_format_test/all_types.csv"
format = "csv"
- sql """ DROP VIEW IF EXISTS test_hdfs_tvf_create_view;"""
+ sql """ DROP VIEW IF EXISTS test_file_tvf_hdfs_create_view;"""
sql """
- create view test_hdfs_tvf_create_view as
+ create view test_file_tvf_hdfs_create_view as
select * from FILE(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
@@ -343,10 +343,10 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
"format" = "${format}") order by c1;
"""
- order_qt_create_view """ select * from test_hdfs_tvf_create_view
order by c1 limit 20; """
+ order_qt_create_view """ select * from
test_file_tvf_hdfs_create_view order by c1 limit 20; """
sql """
- alter view test_hdfs_tvf_create_view as
+ alter view test_file_tvf_hdfs_create_view as
select c1 from FILE(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
@@ -354,7 +354,7 @@
suite("test_file_tvf_hdfs","external,hive,tvf,external_docker") {
"format" = "${format}") order by c1;
"""
- order_qt_alter_view """ select * from test_hdfs_tvf_create_view
order by c1 limit 20; """
+ order_qt_alter_view """ select * from
test_file_tvf_hdfs_create_view order by c1 limit 20; """
} finally {
}
}
diff --git a/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy
b/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy
index 5fac50e64ff..1dd9bbffdd0 100644
--- a/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy
+++ b/regression-test/suites/external_table_p0/tvf/upgrade/load.groovy
@@ -30,26 +30,26 @@ suite("test_tvf_upgrade_load",
"p0,external,hive,external_docker,external_docker
// test create view from tvf and alter view from tvf
uri = "${defaultFS}" +
"/user/doris/preinstalled_data/csv_format_test/all_types.csv"
format = "csv"
- sql """ DROP VIEW IF EXISTS test_hdfs_tvf_create_view;"""
+ sql """ DROP VIEW IF EXISTS test_tvf_upgrade_load_create_view;"""
sql """
- create view test_hdfs_tvf_create_view as
+ create view test_tvf_upgrade_load_create_view as
select * from HDFS(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
"column_separator" = ",",
"format" = "${format}") order by c1;
"""
- logger.info("View test_hdfs_tvf_create_view created")
+ logger.info("View test_tvf_upgrade_load_create_view created")
sql """
- alter view test_hdfs_tvf_create_view as
+ alter view test_tvf_upgrade_load_create_view as
select c1 from HDFS(
"uri" = "${uri}",
"hadoop.username" = "${hdfsUserName}",
"column_separator" = ",",
"format" = "${format}") order by c1;
"""
- logger.info("View test_hdfs_tvf_create_view altered")
+ logger.info("View test_tvf_upgrade_load_create_view altered")
}
-}
\ No newline at end of file
+}
diff --git a/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy
b/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy
index a3b0795ef46..861fad7ddc8 100644
--- a/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy
+++ b/regression-test/suites/external_table_p0/tvf/upgrade/test.groovy
@@ -27,8 +27,8 @@ suite("test_tvf_upgrade_test",
"p0,external,hive,external_docker,external_docker
String enabled = context.config.otherConfigs.get("enableHiveTest")
if (enabled != null && enabled.equalsIgnoreCase("true")) {
- order_qt_create_view """ select * from test_hdfs_tvf_create_view order
by c1 limit 20; """
+ order_qt_create_view """ select * from
test_tvf_upgrade_load_create_view order by c1 limit 20; """
- order_qt_alter_view """ select * from test_hdfs_tvf_create_view order
by c1 limit 20; """
+ order_qt_alter_view """ select * from
test_tvf_upgrade_load_create_view order by c1 limit 20; """
}
-}
\ No newline at end of file
+}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
index f0094c849b3..8ce441caabd 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/part_partition_invalid.groovy
@@ -38,8 +38,7 @@ suite("part_partition_invalid",
"p0,external,external_docker") {
);"""
sql """switch ${hive_catalog_name};"""
- sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
- sql """ drop database if exists ${hive_database}"""
+ sql """ drop database if exists ${hive_database} force"""
sql """ create database ${hive_database}"""
sql """use ${hive_database}"""
sql """
@@ -203,9 +202,7 @@ suite("part_partition_invalid",
"p0,external,external_docker") {
mv_rewrite_success(query_sql, mv_name)
order_qt_after_add_data_and_refresh_catalog_and_mv """ ${query_sql}"""
- sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
- sql """drop table if exists ${internal_catalog}.${olap_db}.${olap_table}"""
- sql """drop database if exists ${hive_catalog_name}.${hive_database}"""
+ sql """drop database if exists ${hive_catalog_name}.${hive_database}
force"""
sql """drop materialized view if exists
${internal_catalog}.${olap_db}.${mv_name};"""
sql """drop catalog if exists ${hive_catalog_name}"""
}
diff --git
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
index 5bd7e5cf338..4ecd7065f34 100644
---
a/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/external_table/single_external_table.groovy
@@ -38,8 +38,7 @@ suite("single_external_table", "p0,external,hive") {
);"""
sql """switch ${hive_catalog_name};"""
- sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
- sql """ drop database if exists ${hive_database}"""
+ sql """ drop database if exists ${hive_database} force"""
sql """ create database ${hive_database}"""
sql """use ${hive_database}"""
sql """
@@ -192,7 +191,6 @@ suite("single_external_table", "p0,external,hive") {
sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_5"""
- sql """drop table if exists
${hive_catalog_name}.${hive_database}.${hive_table}"""
- sql """drop database if exists ${hive_catalog_name}.${hive_database}"""
+ sql """drop database if exists ${hive_catalog_name}.${hive_database}
force"""
sql """drop catalog if exists ${hive_catalog_name}"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
