This is an automated email from the ASF dual-hosted git repository. lihaopeng pushed a commit to branch tmp_tpc_preview4-mysk in repository https://gitbox.apache.org/repos/asf/doris.git
commit e390c4a03f831f69d702cdd0fff01c1a4635963e Author: morningman <[email protected]> AuthorDate: Wed Feb 4 16:47:34 2026 +0800 [feat](catalog) Support include_table_list and lower_case_database_names for external catalog #60580 --- .../org/apache/doris/catalog/JdbcResource.java | 16 +- .../java/org/apache/doris/catalog/Resource.java | 4 - .../apache/doris/datasource/ExternalCatalog.java | 147 ++++++- .../doris/RemoteDorisExternalCatalog.java | 3 +- .../doris/datasource/es/EsExternalCatalog.java | 3 +- .../doris/datasource/hive/HMSExternalCatalog.java | 3 +- .../datasource/iceberg/IcebergExternalCatalog.java | 3 +- .../doris/datasource/jdbc/JdbcExternalCatalog.java | 5 +- .../datasource/jdbc/client/JdbcClientConfig.java | 5 +- .../lakesoul/LakeSoulExternalCatalog.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 3 +- .../datasource/paimon/PaimonExternalCatalog.java | 3 +- .../doris/datasource/test/TestExternalCatalog.java | 3 +- .../TrinoConnectorExternalCatalog.java | 3 +- .../doris/datasource/ExternalCatalogTest.java | 139 +++++++ .../doris/datasource/IncludeTableListTest.java | 424 +++++++++++++++++++++ .../datasource/jdbc/JdbcExternalCatalogTest.java | 7 +- ...aseNameComparedLowercaseMetaCacheFalseTest.java | 134 +++++++ ...baseNameComparedLowercaseMetaCacheTrueTest.java | 134 +++++++ ...abaseNameStoredLowercaseMetaCacheFalseTest.java | 138 +++++++ ...tabaseNameStoredLowercaseMetaCacheTrueTest.java | 138 +++++++ 21 files changed, 1271 insertions(+), 46 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 839260120de..894cefc903e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -124,10 +124,10 @@ public class JdbcResource extends Resource { TYPE, CREATE_TIME, ONLY_SPECIFIED_DATABASE, - LOWER_CASE_META_NAMES, - META_NAMES_MAPPING, - INCLUDE_DATABASE_LIST, - EXCLUDE_DATABASE_LIST, + ExternalCatalog.LOWER_CASE_META_NAMES, + ExternalCatalog.META_NAMES_MAPPING, + ExternalCatalog.INCLUDE_DATABASE_LIST, + ExternalCatalog.EXCLUDE_DATABASE_LIST, CONNECTION_POOL_MIN_SIZE, CONNECTION_POOL_MAX_SIZE, CONNECTION_POOL_MAX_LIFE_TIME, @@ -143,10 +143,10 @@ public class JdbcResource extends Resource { static { OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ONLY_SPECIFIED_DATABASE, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(LOWER_CASE_META_NAMES, "false"); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(META_NAMES_MAPPING, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(INCLUDE_DATABASE_LIST, ""); - OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(EXCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.META_NAMES_MAPPING, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.INCLUDE_DATABASE_LIST, ""); + OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(ExternalCatalog.EXCLUDE_DATABASE_LIST, ""); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MIN_SIZE, "1"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_SIZE, "30"); OPTIONAL_PROPERTIES_DEFAULT_VALUE.put(CONNECTION_POOL_MAX_LIFE_TIME, "1800000"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java index 5db82e3875a..de0c5542b3e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Resource.java @@ -47,10 +47,6 @@ import java.util.stream.Collectors; public abstract class Resource implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(OdbcCatalogResource.class); public static final String REFERENCE_SPLIT = "@"; - public static final String INCLUDE_DATABASE_LIST = "include_database_list"; - public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; - public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; - public static final String META_NAMES_MAPPING = "meta_names_mapping"; public enum ResourceType { UNKNOWN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a9a68d5b893..56b3e822aa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -23,7 +23,6 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.InfoSchemaDb; import org.apache.doris.catalog.MysqlDb; -import org.apache.doris.catalog.Resource; import org.apache.doris.catalog.TableIf; import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.Config; @@ -113,7 +112,11 @@ public abstract class ExternalCatalog public static final boolean DEFAULT_USE_META_CACHE = true; public static final String FOUND_CONFLICTING = "Found conflicting"; + @Deprecated + // use LOWER_CASE_TABLE_NAMES instead public static final String ONLY_TEST_LOWER_CASE_TABLE_NAMES = "only_test_lower_case_table_names"; + public static final String LOWER_CASE_TABLE_NAMES = "lower_case_table_names"; + public static final String LOWER_CASE_DATABASE_NAMES = "lower_case_database_names"; // https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0 public static final String OOS_ROOT_POLICY = "oss.root_policy"; @@ -133,6 +136,13 @@ public abstract class ExternalCatalog public static final String TEST_CONNECTION = "test_connection"; public static final boolean DEFAULT_TEST_CONNECTION = false; + public static final String INCLUDE_DATABASE_LIST = "include_database_list"; + public static final String EXCLUDE_DATABASE_LIST = "exclude_database_list"; + public static final String LOWER_CASE_META_NAMES = "lower_case_meta_names"; + public static final String META_NAMES_MAPPING = "meta_names_mapping"; + // db1.tbl1,db2.tbl2,... + public static final String INCLUDE_TABLE_LIST = "include_table_list"; + // Unique id of this catalog, will be assigned after catalog is loaded. @SerializedName(value = "id") protected long id; @@ -168,6 +178,8 @@ public abstract class ExternalCatalog protected MetaCache<ExternalDatabase<? extends ExternalTable>> metaCache; protected ExecutionAuthenticator executionAuthenticator; protected ThreadPoolExecutor threadPoolWithPreAuth; + // Map lowercase database names to actual remote database names for case-insensitive lookup + private Map<String, String> lowerCaseToDatabaseName = Maps.newConcurrentMap(); private volatile Configuration cachedConf = null; private byte[] confLock = new byte[0]; @@ -267,9 +279,29 @@ public abstract class ExternalCatalog /** * @param dbName - * @return names of tables in specified database + * @return names of tables in specified database, filtered by include_table_list if configured + */ + public final List<String> listTableNames(SessionContext ctx, String dbName) { + makeSureInitialized(); + Map<String, List<String>> includeTableMap = getIncludeTableMap(); + if (includeTableMap.containsKey(dbName) && !includeTableMap.get(dbName).isEmpty()) { + if (LOG.isDebugEnabled()) { + LOG.debug("get table list from include map. catalog: {}, db: {}, tables: {}", + name, dbName, includeTableMap.get(dbName)); + } + return includeTableMap.get(dbName); + } + return listTableNamesFromRemote(ctx, dbName); + } + + /** + * Subclasses implement this method to list table names from the remote data source. + * + * @param ctx session context + * @param dbName database name + * @return names of tables in the specified database from the remote source */ - public abstract List<String> listTableNames(SessionContext ctx, String dbName); + protected abstract List<String> listTableNamesFromRemote(SessionContext ctx, String dbName); /** * check if the specified table exist. @@ -452,6 +484,7 @@ public abstract class ExternalCatalog Map<String, Boolean> includeDatabaseMap = getIncludeDatabaseMap(); Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap(); + lowerCaseToDatabaseName.clear(); List<Pair<String, String>> remoteToLocalPairs = Lists.newArrayList(); allDatabases = allDatabases.stream().filter(dbName -> { @@ -469,6 +502,13 @@ public abstract class ExternalCatalog for (String remoteDbName : allDatabases) { String localDbName = fromRemoteDatabaseName(remoteDbName); + // Populate lowercase mapping for case-insensitive lookups + lowerCaseToDatabaseName.put(remoteDbName.toLowerCase(), remoteDbName); + // Apply lower_case_database_names mode to local name + int dbNameMode = getLowerCaseDatabaseNames(); + if (dbNameMode == 1) { + localDbName = localDbName.toLowerCase(); + } remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); } @@ -524,6 +564,7 @@ public abstract class ExternalCatalog synchronized (this.confLock) { this.cachedConf = null; } + this.lowerCaseToDatabaseName.clear(); onClose(); onRefreshCache(invalidCache); } @@ -643,6 +684,12 @@ public abstract class ExternalCatalog realDbName = InfoSchemaDb.DATABASE_NAME; } else if (realDbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) { realDbName = MysqlDb.DATABASE_NAME; + } else { + // Apply case-insensitive lookup for non-system databases + String localDbName = getLocalDatabaseName(realDbName, false); + if (localDbName != null) { + realDbName = localDbName; + } } // must use full qualified name to generate id. @@ -750,7 +797,14 @@ public abstract class ExternalCatalog if (!isInitialized()) { return Optional.empty(); } - return metaCache.tryGetMetaObj(dbName); + + // Apply case-insensitive lookup with isReplay=true (no remote calls) + String localDbName = getLocalDatabaseName(dbName, true); + if (localDbName == null) { + localDbName = dbName; // Fallback to original name + } + + return metaCache.tryGetMetaObj(localDbName); } /** @@ -873,6 +927,9 @@ public abstract class ExternalCatalog if (tableAutoAnalyzePolicy == null) { tableAutoAnalyzePolicy = Maps.newHashMap(); } + if (this.lowerCaseToDatabaseName == null) { + this.lowerCaseToDatabaseName = Maps.newConcurrentMap(); + } } public void addDatabaseForTest(ExternalDatabase<? extends ExternalTable> db) { @@ -1044,11 +1101,38 @@ public abstract class ExternalCatalog } protected Map<String, Boolean> getIncludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.INCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.INCLUDE_DATABASE_LIST); } protected Map<String, Boolean> getExcludeDatabaseMap() { - return getSpecifiedDatabaseMap(Resource.EXCLUDE_DATABASE_LIST); + return getSpecifiedDatabaseMap(ExternalCatalog.EXCLUDE_DATABASE_LIST); + } + + protected Map<String, List<String>> getIncludeTableMap() { + Map<String, List<String>> includeTableMap = Maps.newHashMap(); + String tableList = catalogProperty.getOrDefault(ExternalCatalog.INCLUDE_TABLE_LIST, ""); + if (Strings.isNullOrEmpty(tableList)) { + return includeTableMap; + } + String[] parts = tableList.split(","); + for (String part : parts) { + String dbTbl = part.trim(); + String[] splits = dbTbl.split("\\."); + if (splits.length != 2) { + LOG.warn("debug invalid include table list: {}, ignore", part); + continue; + } + String db = splits[0]; + String tbl = splits[1]; + List<String> tbls = includeTableMap.get(db); + if (tbls == null) { + includeTableMap.put(db, Lists.newArrayList()); + tbls = includeTableMap.get(db); + } + tbls.add(tbl); + } + LOG.info("debug get include table map: {}", includeTableMap); + return includeTableMap; } private Map<String, Boolean> getSpecifiedDatabaseMap(String catalogPropertyKey) { @@ -1070,15 +1154,60 @@ public abstract class ExternalCatalog public String getLowerCaseMetaNames() { - return catalogProperty.getOrDefault(Resource.LOWER_CASE_META_NAMES, "false"); + return catalogProperty.getOrDefault(LOWER_CASE_META_NAMES, "false"); } public int getOnlyTestLowerCaseTableNames() { - return Integer.parseInt(catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0")); + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_TABLE_NAMES, + catalogProperty.getOrDefault(ONLY_TEST_LOWER_CASE_TABLE_NAMES, "0"))); + } + + /** + * Get the lower_case_database_names configuration value. + * Returns the mode for database name case handling: + * - 0: Case-sensitive (default) + * - 1: Database names are stored as lowercase + * - 2: Database name comparison is case-insensitive + */ + public int getLowerCaseDatabaseNames() { + return Integer.parseInt(catalogProperty.getOrDefault(LOWER_CASE_DATABASE_NAMES, "0")); } public String getMetaNamesMapping() { - return catalogProperty.getOrDefault(Resource.META_NAMES_MAPPING, ""); + return catalogProperty.getOrDefault(ExternalCatalog.META_NAMES_MAPPING, ""); + } + + /** + * Get the local database name based on the lower_case_database_names mode. + * Handles case-insensitive database lookup similar to ExternalDatabase.getLocalTableName(). + */ + @Nullable + private String getLocalDatabaseName(String dbName, boolean isReplay) { + String finalName = dbName; + int mode = getLowerCaseDatabaseNames(); + + if (mode == 1) { + // Mode 1: Store as lowercase + finalName = dbName.toLowerCase(); + } else if (mode == 2) { + // Mode 2: Case-insensitive comparison + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + if (finalName == null && !isReplay) { + // Refresh database list and try again + try { + getFilteredDatabaseNames(); + finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + } catch (Exception e) { + LOG.warn("Failed to refresh database list for catalog {}", getName(), e); + } + } + if (finalName == null && LOG.isDebugEnabled()) { + LOG.debug("Failed to get database name from: {}.{}, isReplay={}", + getName(), dbName, isReplay); + } + } + + return finalName; } public String bindBrokerName() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java index b63a2a03b1f..159726fe0a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/RemoteDorisExternalCatalog.java @@ -166,8 +166,7 @@ public class RemoteDorisExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return dorisRestClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java index 8e9c6b08d72..60371365c50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/EsExternalCatalog.java @@ -134,8 +134,7 @@ public class EsExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return esRestClient.listTable(enableIncludeHiddenIndex()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 8b5e93f113c..1589568b7df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -174,8 +174,7 @@ public class HMSExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return metadataOps.listTableNames(ClusterNamespace.getNameFromFullName(dbName)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index f1a655456e0..975a07258bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -122,8 +122,7 @@ public abstract class IcebergExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { // On the Doris side, the result of SHOW TABLES for Iceberg external tables includes both tables and views, // so the combined set of tables and views is used here. List<String> tableNames = metadataOps.listTableNames(dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index 6cf28f5da37..bcb6c2f4bf3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -105,7 +105,7 @@ public class JdbcExternalCatalog extends ExternalCatalog { } JdbcResource.checkBooleanProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, getOnlySpecifiedDatabase()); - JdbcResource.checkBooleanProperty(JdbcResource.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); + JdbcResource.checkBooleanProperty(ExternalCatalog.LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); JdbcResource.checkBooleanProperty(JdbcResource.CONNECTION_POOL_KEEP_ALIVE, String.valueOf(isConnectionPoolKeepAlive())); JdbcResource.checkBooleanProperty(JdbcResource.TEST_CONNECTION, String.valueOf(isTestConnection())); @@ -278,8 +278,7 @@ public class JdbcExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return jdbcClient.getTablesNameList(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java index 85f3bd8f256..81d43fccce5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/client/JdbcClientConfig.java @@ -19,6 +19,7 @@ package org.apache.doris.datasource.jdbc.client; import org.apache.doris.catalog.JdbcResource; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; @@ -46,8 +47,8 @@ public class JdbcClientConfig implements Cloneable { public JdbcClientConfig() { this.onlySpecifiedDatabase = JdbcResource.getDefaultPropertyValue(JdbcResource.ONLY_SPECIFIED_DATABASE); - this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(JdbcResource.LOWER_CASE_META_NAMES); - this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(JdbcResource.META_NAMES_MAPPING); + this.isLowerCaseMetaNames = JdbcResource.getDefaultPropertyValue(ExternalCatalog.LOWER_CASE_META_NAMES); + this.metaNamesMapping = JdbcResource.getDefaultPropertyValue(ExternalCatalog.META_NAMES_MAPPING); this.connectionPoolMinSize = Integer.parseInt( JdbcResource.getDefaultPropertyValue(JdbcResource.CONNECTION_POOL_MIN_SIZE)); this.connectionPoolMaxSize = Integer.parseInt( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java index 0d44e41dce9..465f2c78382 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java @@ -60,7 +60,7 @@ public class LakeSoulExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { // makeSureInitialized(); // List<TableInfo> tifs = lakesoulMetadataManager.getTableInfosByNamespace(dbName); // List<String> tableNames = Lists.newArrayList(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index 65af7f967fe..eec0a6eee2e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -283,8 +283,7 @@ public class MaxComputeExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return mcStructureHelper.listTableNames(getClient(), dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 76e093656bb..c712b650e1f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -104,8 +104,7 @@ public class PaimonExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { try { return executionAuthenticator.execute(() -> { List<String> tableNames = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java index 1fd8e3721f3..57a483f28c7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java @@ -85,8 +85,7 @@ public class TestExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { return mockedTableNames(dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java index 3627aff636d..e2db35707dd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorExternalCatalog.java @@ -184,8 +184,7 @@ public class TrinoConnectorExternalCatalog extends ExternalCatalog { } @Override - public List<String> listTableNames(SessionContext ctx, String dbName) { - makeSureInitialized(); + protected List<String> listTableNamesFromRemote(SessionContext ctx, String dbName) { QualifiedTablePrefix qualifiedTablePrefix = new QualifiedTablePrefix(trinoCatalogHandle.getCatalogName(), dbName); List<QualifiedObjectName> tables = trinoListTables(qualifiedTablePrefix); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java index e75925f5203..68ffe7221bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogTest.java @@ -215,6 +215,145 @@ public class ExternalCatalogTest extends TestWithFeService { Assertions.assertEquals(MysqlStateType.OK, rootCtx.getState().getStateType()); } + @Test + public void testGetIncludeTableMap() throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + + // Test 1: Empty include_table_list + String createStmt = "create catalog test_include_table_empty properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\"\n" + + ");"; + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + TestExternalCatalog ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_empty"); + Map<String, List<String>> includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertTrue(includeTableMap.isEmpty()); + + // Test 2: Single table + createStmt = "create catalog test_include_table_single properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_single"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + + // Test 3: Multiple tables in same database + createStmt = "create catalog test_include_table_same_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,db1.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_same_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(1, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertEquals(3, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl3")); + + // Test 4: Multiple tables in different databases + createStmt = "create catalog test_include_table_diff_db properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db2.tbl2,db3.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_diff_db"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(3, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertTrue(includeTableMap.containsKey("db3")); + Assertions.assertEquals(1, includeTableMap.get("db1").size()); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertEquals(1, includeTableMap.get("db3").size()); + Assertions.assertEquals("tbl1", includeTableMap.get("db1").get(0)); + Assertions.assertEquals("tbl2", includeTableMap.get("db2").get(0)); + Assertions.assertEquals("tbl3", includeTableMap.get("db3").get(0)); + + // Test 5: Invalid format (should be ignored) + createStmt = "create catalog test_include_table_invalid properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,invalid_format,db2.tbl2,too.many.dots\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_invalid"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertFalse(includeTableMap.containsKey("invalid_format")); + Assertions.assertFalse(includeTableMap.containsKey("too")); + + // Test 6: With whitespace (should be trimmed) + createStmt = "create catalog test_include_table_whitespace properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \" db1.tbl1 , db2.tbl2 \"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_whitespace"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + + // Test 7: Mixed valid and invalid with multiple tables in same db + createStmt = "create catalog test_include_table_mixed properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.RefreshCatalogTest$RefreshCatalogProvider\",\n" + + " \"include_table_list\" = \"db1.tbl1,db1.tbl2,invalid,db2.tbl3\"\n" + + ");"; + logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + ctl = (TestExternalCatalog) mgr.getCatalog("test_include_table_mixed"); + includeTableMap = ctl.getIncludeTableMap(); + Assertions.assertEquals(2, includeTableMap.size()); + Assertions.assertTrue(includeTableMap.containsKey("db1")); + Assertions.assertTrue(includeTableMap.containsKey("db2")); + Assertions.assertEquals(2, includeTableMap.get("db1").size()); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl1")); + Assertions.assertTrue(includeTableMap.get("db1").contains("tbl2")); + Assertions.assertEquals(1, includeTableMap.get("db2").size()); + Assertions.assertTrue(includeTableMap.get("db2").contains("tbl3")); + } + public static class RefreshCatalogProvider implements TestExternalCatalog.TestCatalogProvider { public static final Map<String, Map<String, List<Column>>> MOCKED_META; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java new file mode 100644 index 00000000000..2f8e5d5177c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/IncludeTableListTest.java @@ -0,0 +1,424 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class IncludeTableListTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + private void createCatalog(String catalogName, String providerClass, String includeTableList) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("create catalog ").append(catalogName).append(" properties(\n"); + sb.append(" \"type\" = \"test\",\n"); + sb.append(" \"catalog_provider.class\" = \"").append(providerClass).append("\""); + if (includeTableList != null) { + sb.append(",\n \"").append(ExternalCatalog.INCLUDE_TABLE_LIST) + .append("\" = \"").append(includeTableList).append("\""); + } + sb.append("\n);"); + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sb.toString()); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void dropCatalog(String catalogName) throws Exception { + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog " + catalogName); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + private void refreshCatalog(String catalogName) { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand(catalogName, null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + } + + private static final String PROVIDER_CLASS = + "org.apache.doris.datasource.IncludeTableListTest$IncludeTableListProvider"; + + // ==================== Basic filtering tests ==================== + + /** + * When include_table_list is not configured, all tables should be visible. + */ + @Test + public void testNoIncludeTableList() throws Exception { + String catalogName = "test_no_include"; + createCatalog(catalogName, PROVIDER_CLASS, null); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> tableNames = db1.getTableNamesWithLock(); + // All 3 tables in db1 should be visible + Assertions.assertEquals(3, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + + ExternalDatabase<?> db2 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set<String> db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + Assertions.assertTrue(db2TableNames.contains("tbl_a")); + Assertions.assertTrue(db2TableNames.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies a single table in one db, only that table should be visible + * in that db; other dbs should show all tables. + */ + @Test + public void testSingleTableInclude() throws Exception { + String catalogName = "test_single_include"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> tableNames = db1.getTableNamesWithLock(); + // Only tbl1 should be visible in db1 + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertFalse(tableNames.contains("tbl3")); + + // db2 should still show all tables (not in include_table_list) + ExternalDatabase<?> db2 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set<String> db2TableNames = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2TableNames.size()); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies multiple tables in same db. + */ + @Test + public void testMultipleTablesInSameDb() throws Exception { + String catalogName = "test_multi_same_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.tbl3"); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> tableNames = db1.getTableNamesWithLock(); + // Only tbl1 and tbl3 should be visible + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertFalse(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list specifies tables across multiple dbs. + */ + @Test + public void testMultipleTablesAcrossDbs() throws Exception { + String catalogName = "test_multi_cross_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db2.tbl_a"); + try { + refreshCatalog(catalogName); + + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl2")); + + ExternalDatabase<?> db2 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set<String> db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Error / edge case tests ==================== + + /** + * When include_table_list specifies a table that does NOT exist in the remote source, + * the table name should still appear in listTableNames but getTableNullable should return null. + */ + @Test + public void testNonExistentTableInIncludeList() throws Exception { + String catalogName = "test_nonexist_tbl"; + // "nonexistent_table" does not exist in the provider's db1 + createCatalog(catalogName, PROVIDER_CLASS, "db1.nonexistent_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + // The include list overrides remote listing, so it reports "nonexistent_table" + Set<String> tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, tableNames.size()); + Assertions.assertTrue(tableNames.contains("nonexistent_table")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Mix of existing and non-existing tables in include_table_list. + * Existing table should be accessible; non-existing table should return null. + */ + @Test + public void testMixExistentAndNonExistentTables() throws Exception { + String catalogName = "test_mix_exist"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,db1.no_such_table"); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("no_such_table")); + + // Existing table should be accessible + Assertions.assertNotNull(db1.getTableNullable("tbl1")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list refers to a non-existent database, that db entry + * in the include map is simply ignored (the db won't appear). + */ + @Test + public void testNonExistentDbInIncludeList() throws Exception { + String catalogName = "test_nonexist_db"; + // "no_such_db" does not exist in the provider + createCatalog(catalogName, PROVIDER_CLASS, "no_such_db.tbl1,db1.tbl1"); + try { + refreshCatalog(catalogName); + // db1 should still work with filtered table + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> db1Tables = db1.getTableNamesWithLock(); + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + // The non-existent db should return null + Assertions.assertNull( + env.getCatalogMgr().getCatalog(catalogName).getDbNullable("no_such_db")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list contains entries with invalid format (no dot separator), + * those entries are silently ignored. + */ + @Test + public void testInvalidFormatInIncludeList() throws Exception { + String catalogName = "test_invalid_fmt"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1,bad_format,db2.tbl_a,too.many.dots"); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> db1Tables = db1.getTableNamesWithLock(); + // Only "db1.tbl1" is a valid entry for db1 + Assertions.assertEquals(1, db1Tables.size()); + Assertions.assertTrue(db1Tables.contains("tbl1")); + + ExternalDatabase<?> db2 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set<String> db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(1, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list entries have extra whitespace, they should be trimmed properly. + */ + @Test + public void testWhitespaceInIncludeList() throws Exception { + String catalogName = "test_whitespace"; + createCatalog(catalogName, PROVIDER_CLASS, " db1.tbl1 , db1.tbl2 "); + try { + refreshCatalog(catalogName); + ExternalDatabase<?> db1 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db1"); + Assertions.assertNotNull(db1); + Set<String> tableNames = db1.getTableNamesWithLock(); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl1")); + Assertions.assertTrue(tableNames.contains("tbl2")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * When include_table_list is set but all entries are for one db, + * another db should show all its tables (unaffected). + */ + @Test + public void testUnaffectedDbShowsAllTables() throws Exception { + String catalogName = "test_unaffected_db"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl1"); + try { + refreshCatalog(catalogName); + // db2 is not mentioned in include_table_list, so all tables should be visible + ExternalDatabase<?> db2 = (ExternalDatabase<?>) env.getCatalogMgr() + .getCatalog(catalogName).getDbNullable("db2"); + Assertions.assertNotNull(db2); + Set<String> db2Tables = db2.getTableNamesWithLock(); + Assertions.assertEquals(2, db2Tables.size()); + Assertions.assertTrue(db2Tables.contains("tbl_a")); + Assertions.assertTrue(db2Tables.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + /** + * Test that listTableNames (the catalog-level API) returns the included list directly + * when include_table_list is configured for that db. + */ + @Test + public void testListTableNamesAPI() throws Exception { + String catalogName = "test_api_list"; + createCatalog(catalogName, PROVIDER_CLASS, "db1.tbl2,db1.tbl3"); + try { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog(catalogName); + List<String> tableNames = catalog.listTableNames(null, "db1"); + Assertions.assertEquals(2, tableNames.size()); + Assertions.assertTrue(tableNames.contains("tbl2")); + Assertions.assertTrue(tableNames.contains("tbl3")); + Assertions.assertFalse(tableNames.contains("tbl1")); + + // db2 not in include map → returns all remote tables + List<String> db2Names = catalog.listTableNames(null, "db2"); + Assertions.assertEquals(2, db2Names.size()); + Assertions.assertTrue(db2Names.contains("tbl_a")); + Assertions.assertTrue(db2Names.contains("tbl_b")); + } finally { + dropCatalog(catalogName); + } + } + + // ==================== Mock data provider ==================== + + public static class IncludeTableListProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map<String, Map<String, List<Column>>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + + // db1 with 3 tables + Map<String, List<Column>> db1Tables = Maps.newHashMap(); + db1Tables.put("tbl1", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("name", PrimitiveType.VARCHAR))); + db1Tables.put("tbl2", Lists.newArrayList( + new Column("id", PrimitiveType.INT), + new Column("value", PrimitiveType.BIGINT))); + db1Tables.put("tbl3", Lists.newArrayList( + new Column("key", PrimitiveType.VARCHAR), + new Column("data", PrimitiveType.STRING))); + MOCKED_META.put("db1", db1Tables); + + // db2 with 2 tables + Map<String, List<Column>> db2Tables = Maps.newHashMap(); + db2Tables.put("tbl_a", Lists.newArrayList( + new Column("col1", PrimitiveType.INT), + new Column("col2", PrimitiveType.FLOAT))); + db2Tables.put("tbl_b", Lists.newArrayList( + new Column("x", PrimitiveType.BIGINT), + new Column("y", PrimitiveType.DOUBLE))); + MOCKED_META.put("db2", db2Tables); + } + + @Override + public Map<String, Map<String, List<Column>>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java index 5a7379b2e84..c63aebfcb35 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.JdbcResource; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogFactory; +import org.apache.doris.datasource.ExternalCatalog; import com.google.common.collect.Maps; import org.junit.Assert; @@ -96,14 +97,14 @@ public class JdbcExternalCatalogTest { exception1.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "true"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "1"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "1"); Exception exception2 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals("errCode = 2, detailMessage = lower_case_meta_names must be true or false", exception2.getMessage()); jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.ONLY_SPECIFIED_DATABASE, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.LOWER_CASE_META_NAMES, "false"); - jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.INCLUDE_DATABASE_LIST, "db1,db2"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.LOWER_CASE_META_NAMES, "false"); + jdbcExternalCatalog.getCatalogProperty().addProperty(ExternalCatalog.INCLUDE_DATABASE_LIST, "db1,db2"); DdlException exceptione3 = Assert.assertThrows(DdlException.class, () -> jdbcExternalCatalog.checkProperties()); Assert.assertEquals( "errCode = 2, detailMessage = include_database_list and exclude_database_list cannot be set when only_specified_database is false", diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000..33eae7ff312 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheFalseTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List<String> dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map<String, Map<String, List<Column>>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map<String, List<Column>> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map<String, Map<String, List<Column>>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000..299bdb48cc6 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest.java @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 2 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameComparedLowercaseMetaCacheTrueTest$ExternalDatabaseNameComparedLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"2\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(2, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with lowercase, should retrieve original case + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("database1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("DATABASE1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DataBase2"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("DATABASE2", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List<String> dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("DATABASE1")); // Original case preserved + Assertions.assertTrue(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameComparedLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map<String, Map<String, List<Column>>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map<String, List<Column>> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with uppercase names that preserve case + MOCKED_META.put("DATABASE1", tables); + MOCKED_META.put("DATABASE2", tables); + } + + @Override + public Map<String, Map<String, List<Column>>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java new file mode 100644 index 00000000000..626f2494035 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheFalseTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List<String> dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map<String, Map<String, List<Column>>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map<String, List<Column>> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map<String, Map<String, List<Column>>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java new file mode 100644 index 00000000000..83268e5704e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lowercase/ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lowercase; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.DropCatalogCommand; +import org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest extends TestWithFeService { + private static Env env; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + // 1. create test catalog with lower_case_database_names = 1 + String createStmt = "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.lowercase.ExternalDatabaseNameStoredLowercaseMetaCacheTrueTest$ExternalDatabaseNameStoredLowercaseProvider\",\n" + + " \"" + ExternalCatalog.LOWER_CASE_DATABASE_NAMES + "\" = \"1\"\n" + + ");"; + + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(createStmt); + if (logicalPlan instanceof CreateCatalogCommand) { + ((CreateCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Override + protected void beforeCluster() { + FeConstants.runningUnitTest = true; + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle("drop catalog test1"); + if (logicalPlan instanceof DropCatalogCommand) { + ((DropCatalogCommand) logicalPlan).run(rootCtx, null); + } + } + + @Test + public void testGlobalVariable() { + ExternalCatalog catalog = (ExternalCatalog) env.getCatalogMgr().getCatalog("test1"); + Assertions.assertEquals(1, catalog.getLowerCaseDatabaseNames()); + } + + @Test + public void testGetDbWithOutList() { + RefreshCatalogCommand refreshCatalogCommand = new RefreshCatalogCommand("test1", null); + try { + refreshCatalogCommand.run(connectContext, null); + } catch (Exception e) { + // Do nothing + } + // Query with uppercase, should retrieve lowercase + ExternalDatabase db = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("DATABASE1"); + Assertions.assertNotNull(db); + Assertions.assertEquals("database1", db.getFullName()); + + // Query with mixed case + ExternalDatabase db2 = (ExternalDatabase) env.getCatalogMgr().getCatalog("test1") + .getDbNullable("Database1"); + Assertions.assertNotNull(db2); + Assertions.assertEquals("database1", db2.getFullName()); + } + + @Test + public void testDatabaseNameLowerCase() { + List<String> dbNames = env.getCatalogMgr().getCatalog("test1").getDbNames(); + Assertions.assertTrue(dbNames.contains("database1")); + Assertions.assertTrue(dbNames.contains("database2")); + Assertions.assertTrue(dbNames.contains("database3")); + Assertions.assertFalse(dbNames.contains("Database1")); + Assertions.assertFalse(dbNames.contains("DATABASE2")); + } + + public static class ExternalDatabaseNameStoredLowercaseProvider implements TestExternalCatalog.TestCatalogProvider { + public static final Map<String, Map<String, List<Column>>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map<String, List<Column>> tables = Maps.newHashMap(); + tables.put("table1", Lists.newArrayList(new Column("k1", PrimitiveType.INT))); + + // Test databases with mixed case in remote system + MOCKED_META.put("Database1", tables); + MOCKED_META.put("DATABASE2", tables); + MOCKED_META.put("database3", tables); + } + + @Override + public Map<String, Map<String, List<Column>>> getMetadata() { + return MOCKED_META; + } + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
