This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new e9921d7a076 branch-2.1: [fix](external catalog) Fixed non-master FE
log replay during refresh (#46373)
e9921d7a076 is described below
commit e9921d7a07606944fd5d798d866cc8fdb39c4d3e
Author: zy-kkk <[email protected]>
AuthorDate: Fri Jan 3 22:03:30 2025 +0800
branch-2.1: [fix](external catalog) Fixed non-master FE log replay during
refresh (#46373)
cherry-pick #46369
---
.../java/org/apache/doris/datasource/ExternalCatalog.java | 2 +-
.../java/org/apache/doris/datasource/ExternalDatabase.java | 2 +-
.../external_table_p0/lower_case/test_conflict_name.groovy | 10 ++++++++--
.../lower_case/test_lower_case_meta_include.groovy | 10 ++++++++--
.../lower_case/test_lower_case_meta_show_and_select.groovy | 10 ++++++++--
...wer_case_meta_with_lower_table_conf_show_and_select.groovy | 11 ++++++++---
.../external_table_p0/lower_case/test_lower_case_mtmv.groovy | 10 ++++++++--
.../lower_case/test_meta_cache_select_without_refresh.groovy | 10 ++++++++--
.../lower_case/test_meta_names_mapping.groovy | 10 ++++++++--
.../lower_case/test_timing_refresh_catalog.groovy | 10 ++++++++--
.../suites/external_table_p0/lower_case/upgrade/load.groovy | 8 ++++++--
11 files changed, 72 insertions(+), 21 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 843cd9ef2fb..f9cd27baa81 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -718,7 +718,7 @@ public abstract class ExternalCatalog
public void replayInitCatalog(InitCatalogLog log) {
// If the remote name is missing during upgrade, all databases in the
Map will be reinitialized.
- if (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty())) {
dbNameToId = Maps.newConcurrentMap();
idToDb = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index e7eb3949eae..cbacf563c32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -191,7 +191,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
// If the remote name is missing during upgrade, all tables in the Map
will be reinitialized.
- if (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty()) {
+ if (log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty())) {
tableNameToId = Maps.newConcurrentMap();
idToTbl = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
index 6187d47b645..43d6a68c1ad 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_conflict_name.groovy
@@ -18,12 +18,16 @@
suite("test_conflict_name", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_conflict_name_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
sql """create database if not exists internal.external_conflict_name; """
@@ -91,4 +95,6 @@ suite("test_conflict_name",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_conflict_name; """
sql """drop database if exists internal.EXTERNAL_CONFLICT_NAME; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
index 854eb06a8e2..cd48678f25c 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_include.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_include_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping_db = """
{
"databases": [
@@ -155,4 +159,6 @@ suite("test_lower_case_meta_include",
"p0,external,doris,external_docker,externa
sql """drop catalog if exists test_lower_case_exclude """
sql """drop database if exists internal.external_INCLUDE; """
sql """drop database if exists internal.external_EXCLUDE; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
index d4efd141c2d..7b61216e949 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_show_and_select.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_without_conf_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
sql """create database if not exists internal.external_test_lower; """
@@ -251,4 +255,6 @@ suite("test_lower_case_meta_show_and_select",
"p0,external,doris,external_docker
sql """drop database if exists internal.external_test_lower; """
sql """drop database if exists internal.external_test_UPPER; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
index 4f6efc829c3..b310a5bd614 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_meta_with_lower_table_conf_show_and_select.groovy
@@ -18,14 +18,17 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_with_conf"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.external_test_lower_with_conf; """
- sql """drop database if exists internal.external_test_UPPER_with_conf; """
sql """create database if not exists
internal.external_test_lower_with_conf; """
sql """create table if not exists
internal.external_test_lower_with_conf.lower_with_conf
(id int, name varchar(20))
@@ -699,4 +702,6 @@
suite("test_lower_case_meta_with_lower_table_conf_show_and_select", "p0,external
sql """drop catalog if exists test_cache_true_lower_true_with_conf0 """
sql """drop database if exists internal.external_test_lower_with_conf; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
index 40322a22afe..f72b3c9e68d 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_lower_case_mtmv.groovy
@@ -18,12 +18,16 @@
suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_lower_case_mtmv_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV; """
sql """create database if not exists internal.EXTERNAL_LOWER_MTMV;"""
sql """create table if not exists internal.EXTERNAL_LOWER_MTMV.TABLE_TEST
@@ -61,4 +65,6 @@ suite("test_lower_case_mtmv",
"p0,external,doris,external_docker,external_docker
sql """drop catalog if exists test_lower_case_mtmv """
sql """drop database if exists internal.EXTERNAL_LOWER_MTMV """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
index e8351784212..4a58fcd65a1 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_cache_select_without_refresh.groovy
@@ -18,12 +18,16 @@
suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_cache_select_without_refresh_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """ drop database if exists
internal.external_lower_select_without_refresh; """
sql """create database if not exists
internal.external_lower_select_without_refresh;"""
@@ -89,4 +93,6 @@ suite("test_meta_cache_select_without_refresh",
"p0,external,doris,external_dock
sql """drop catalog if exists
test_meta_cache_lower_true_select_without_refresh """
sql """drop catalog if exists
test_meta_cache_lower_false_select_without_refresh """
sql """drop database if exists
internal.external_lower_select_without_refresh; """
+
+ try_sql """drop user ${jdbcUser}"""
}
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
index 1cf48b17c87..b58077295b3 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_meta_names_mapping.groovy
@@ -18,12 +18,16 @@
suite("test_meta_names_mapping", "p0,external,doris,meta_names_mapping") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_meta_names_mapping_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String validMetaNamesMapping = """
{
"databases": [
@@ -286,4 +290,6 @@ suite("test_meta_names_mapping",
"p0,external,doris,meta_names_mapping") {
sql """drop database if exists internal.external_meta_names_mapping; """
sql """drop database if exists internal.EXTERNAL_META_NAMES_MAPPING; """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
index 19d7e2db180..a95da3b5d67 100644
---
a/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
+++
b/regression-test/suites/external_table_p0/lower_case/test_timing_refresh_catalog.groovy
@@ -18,12 +18,16 @@
suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_timing_refresh_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
String mapping = """
{
"databases": [
@@ -158,4 +162,6 @@ suite("test_timing_refresh_catalog",
"p0,external,doris,external_docker,external
sql """drop catalog if exists test_timing_refresh_catalog1 """
sql """drop catalog if exists test_timing_refresh_catalog2 """
sql """drop database if exists internal.external_timing_refresh_catalog """
+
+ try_sql """drop user ${jdbcUser}"""
}
\ No newline at end of file
diff --git
a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
index 0ea89d4012d..ef03f97d0e2 100644
--- a/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
+++ b/regression-test/suites/external_table_p0/lower_case/upgrade/load.groovy
@@ -18,12 +18,16 @@
suite("test_upgrade_lower_case_catalog_prepare",
"p0,external,doris,external_docker,external_docker_doris") {
String jdbcUrl = context.config.jdbcUrl
- String jdbcUser = context.config.jdbcUser
- String jdbcPassword = context.config.jdbcPassword
+ String jdbcUser = "test_upgrade_lower_case_catalog_user"
+ String jdbcPassword = "C123_567p"
String s3_endpoint = getS3Endpoint()
String bucket = getS3BucketName()
String driver_url =
"https://${bucket}.${s3_endpoint}/regression/jdbc_driver/mysql-connector-j-8.3.0.jar";
+ try_sql """drop user ${jdbcUser}"""
+ sql """create user ${jdbcUser} identified by '${jdbcPassword}'"""
+ sql """grant all on *.*.* to ${jdbcUser}"""
+
sql """drop database if exists internal.upgrade_lower_case_catalog_lower;
"""
sql """drop database if exists internal.upgrade_lower_case_catalog_UPPER;
"""
sql """create database if not exists
internal.upgrade_lower_case_catalog_lower; """
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
