This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new a6fc852ce73 branch-2.1: [opt](catalog) use table in db object return
get db #46211 (#46229)
a6fc852ce73 is described below
commit a6fc852ce73a7f6bf75e41daac9398a54af48c24
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jan 1 08:00:49 2025 +0800
branch-2.1: [opt](catalog) use table in db object return get db #46211
(#46229)
Cherry-picked from #46211
Co-authored-by: zy-kkk <[email protected]>
Co-authored-by: morningman <[email protected]>
---
.../java/org/apache/doris/datasource/ExternalCatalog.java | 5 ++++-
.../java/org/apache/doris/datasource/ExternalDatabase.java | 14 ++++++++++++++
.../java/org/apache/doris/datasource/ExternalTable.java | 2 +-
.../apache/doris/datasource/jdbc/JdbcExternalTable.java | 2 +-
.../external_table_p0/hive/test_hive_use_meta_cache.out | 4 ++++
.../suites/external_table_p0/hive/test_hive_parquet.groovy | 3 ++-
.../external_table_p2/hudi/test_hudi_snapshot.groovy | 3 +--
7 files changed, 27 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index af99df48157..e515bcc7ed6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -108,7 +108,7 @@ public abstract class ExternalCatalog
public static final String CREATE_TIME = "create_time";
public static final boolean DEFAULT_USE_META_CACHE = true;
- public static final String FOUND_CONFLICTING = "Found conflicting";
+ public static final String FOUND_CONFLICTING = "Found conflicting";
public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES =
"only_test_lower_case_table_names";
// Properties that should not be shown in the `show create catalog` result
@@ -741,6 +741,7 @@ public abstract class ExternalCatalog
Preconditions.checkNotNull(db.get());
tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
tmpIdToDb.put(db.get().getId(), db.get());
+ LOG.info("Synchronized database (refresh): [Name: {}, ID: {}]",
db.get().getFullName(), db.get().getId());
}
for (int i = 0; i < log.getCreateCount(); i++) {
ExternalDatabase<? extends ExternalTable> db =
@@ -749,6 +750,8 @@ public abstract class ExternalCatalog
if (db != null) {
tmpDbNameToId.put(db.getFullName(), db.getId());
tmpIdToDb.put(db.getId(), db);
+ LOG.info("Synchronized database (create): [Name: {}, ID: {},
Remote Name: {}]",
+ db.getFullName(), db.getId(),
log.getRemoteDbNames().get(i));
}
}
dbNameToId = tmpDbNameToId;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 099d76e98e1..1759c3e30b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -211,6 +211,13 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
if (table.isPresent()) {
tmpTableNameToId.put(table.get().getName(),
table.get().getId());
tmpIdToTbl.put(table.get().getId(), table.get());
+
+ // Add logic to set the database if missing
+ if (table.get().getDb() == null) {
+ table.get().setDb(this);
+ }
+ LOG.info("Synchronized table (refresh): [Name: {}, ID: {}]",
table.get().getName(),
+ table.get().getId());
}
}
for (int i = 0; i < log.getCreateCount(); i++) {
@@ -219,6 +226,13 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
log.getCreateTableIds().get(i), catalog, this,
false);
tmpTableNameToId.put(table.getName(), table.getId());
tmpIdToTbl.put(table.getId(), table);
+
+ // Add logic to set the database if missing
+ if (table.getDb() == null) {
+ table.setDb(this);
+ }
+ LOG.info("Synchronized table (create): [Name: {}, ID: {}, Remote
Name: {}]",
+ table.getName(), table.getId(),
log.getRemoteTableNames().get(i));
}
tableNameToId = tmpTableNameToId;
idToTbl = tmpIdToTbl;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 1924ed1b9fc..2d607cc8ff8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -308,7 +308,7 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
@Override
public DatabaseIf getDatabase() {
- return catalog.getDbNullable(dbName);
+ return this.db;
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index 95c9820a013..b3a76abcf5a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -51,7 +51,7 @@ import java.util.stream.Collectors;
public class JdbcExternalTable extends ExternalTable {
private static final Logger LOG =
LogManager.getLogger(JdbcExternalTable.class);
- public static final String MYSQL_ROW_COUNT_SQL = "SELECT max(row_count) as
rows FROM ("
+ public static final String MYSQL_ROW_COUNT_SQL = "SELECT max(row_count) as
`rows` FROM ("
+ "(SELECT TABLE_ROWS AS row_count FROM INFORMATION_SCHEMA.TABLES "
+ "WHERE TABLE_SCHEMA = '${dbName}' AND TABLE_NAME = '${tblName}' "
+ "AND TABLE_TYPE = 'BASE TABLE') "
diff --git
a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
index 4d54c601c87..d8b269fdf0d 100644
--- a/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
+++ b/regression-test/data/external_table_p0/hive/test_hive_use_meta_cache.out
@@ -34,6 +34,8 @@ p1=part1
p1=part2
-- !sql10 --
+test_use_meta_cache_partitioned_tbl_hive
+test_use_meta_cache_tbl_hive
-- !sql11 --
@@ -123,6 +125,8 @@ p1=part1
p1=part2
-- !sql10 --
+test_use_meta_cache_partitioned_tbl_hive
+test_use_meta_cache_tbl_hive
-- !sql11 --
diff --git
a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
index d1e0922fe74..81d2954daf9 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_parquet.groovy
@@ -186,7 +186,8 @@ suite("test_hive_parquet",
"p0,external,hive,external_docker,external_docker_hiv
"type"="hms",
'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hms_port}'
);"""
- sql """use `${catalog_name}`.`default`"""
+ sql """switch ${catalog_name}"""
+ sql """use `default`"""
sql """set enable_fallback_to_original_planner=false;"""
diff --git
a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
index 0bfe7068814..3c650648dca 100644
--- a/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
+++ b/regression-test/suites/external_table_p2/hudi/test_hudi_snapshot.groovy
@@ -34,7 +34,6 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
sql """ use regression_hudi;"""
sql """ set enable_fallback_to_original_planner=false """
- // 创建groovy函数，接收table_name为参数
def test_hudi_snapshot_querys = { table_name ->
// Query users by event_time in descending order and limit output
qt_q01 """SELECT * FROM ${table_name} ORDER BY event_time DESC LIMIT
10;"""
@@ -49,7 +48,7 @@ suite("test_hudi_snapshot",
"p2,external,hudi,external_remote,external_remote_hu
qt_q04 """SELECT * FROM ${table_name} WHERE event_time BETWEEN
'2024-01-01 00:00:00' AND '2024-12-31 23:59:59' ORDER BY event_time LIMIT 10;"""
// Count users by age group and limit output
- qt_q05 """SELECT age, COUNT(*) AS user_count FROM ${table_name} GROUP
BY age ORDER BY user_count DESC LIMIT 5;"""
+ qt_q05 """SELECT age, COUNT(*) AS user_count FROM ${table_name} GROUP
BY age ORDER BY user_count, age DESC LIMIT 5;"""
// Query users with purchase records and limit output
qt_q06 """SELECT user_id, purchases FROM ${table_name} WHERE
array_size(purchases) > 0 ORDER BY user_id LIMIT 5;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
