This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new bc5f8cd7c80 [fix](fe) Skip HMS database events that
include/exclude_database_list filters out (#70165)
bc5f8cd7c80 is described below
commit bc5f8cd7c800245b925faefef0a7c01467fef6a9
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Thu Nov 5 07:08:02 2026 +0000
[fix](fe) Skip HMS database events that include/exclude_database_list
filters out (#70165)
### What problem does this PR solve?
Issue Number: None
Related PR: #67417, #68196
Problem Summary:
**In short.** This ports from branch-4.x #67417 and #68196 the check
that keeps a metastore event from adding a database that
`include_database_list` or `exclude_database_list` keeps out of an HMS
catalog. On master, a CREATE DATABASE event for such a database puts it
into the catalog's database list on every FE that applies the event:
SHOW DATABASES lists it and queries can reach it until the list is
reloaded.
**Background.** A catalog lists its databases through
`ExternalCatalog#getFilteredDatabaseNames`, which drops a remote
database that `exclude_database_list` names or, when
`include_database_list` is set, one that it does not name. On an HMS
catalog with `hive.enable_hms_events_incremental_sync`,
`MetastoreEventSyncDriver` applies the metastore's events on every FE. A
CREATE DATABASE event, and the new name of an ALTER DATABASE rename,
reach `PluginDrivenExternalCatalog#registerDatabaseFromEvent`, which
adds the database to the cached database names. branch-4.1 and
branch-4.2 have checked the two lists for these events since #67417.
**The problem, and what it cost.** `registerDatabaseFromEvent` adds the
database without the check. A database that the catalog is configured to
hide then shows in SHOW DATABASES, `USE` and queries reach it
(privileges still apply), and table events register tables in it. It
goes away only when the cached names are reloaded: after
`external_cache_refresh_time_minutes` (10 by default), on a lookup of a
name that is not cached, on REFRESH CATALOG, or after a restart.
**How this PR fixes it.** The check moves out of
`getFilteredDatabaseNames` into `ExternalCatalog#isDatabaseListed`,
which the listing and `registerDatabaseFromEvent` share; an event for a
database the lists keep out returns before it touches any cache. Like
the listing, the check compares the remote name exactly. Not ported from
branch-4.x: skipping the table, partition and refresh events of such
databases, a performance item (on master they find no database and
return, after one reload of the names), and comparing names ignoring
case when `lower_case_database_names` is not 0, which would change the
listing as well.
**Results.**
| Event, with the database names cached | Before | After |
|---|---|---|
| CREATE DATABASE `archived`, `exclude_database_list = archived` |
`archived` listed | not listed |
| CREATE DATABASE `other`, `include_database_list = sales,sales2` |
`other` listed | not listed |
| CREATE DATABASE `sales2`, allowed by either list | listed | listed |
Original author: @924060929
Co-authored-by: 924060929 <[email protected]>
---
.../apache/doris/datasource/ExternalCatalog.java | 36 ++++---
.../plugin/PluginDrivenExternalCatalog.java | 5 +
...PluginDrivenExternalCatalogEventFilterTest.java | 118 +++++++++++++++++++++
3 files changed, 147 insertions(+), 12 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 7b64a46d62d..fa39a5e0594 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -601,18 +601,9 @@ public abstract class ExternalCatalog
Map<String, Boolean> excludeDatabaseMap = getExcludeDatabaseMap();
List<Pair<String, String>> remoteToLocalPairs = Lists.newArrayList();
- allDatabases = allDatabases.stream().filter(dbName -> {
- if (!dbName.equals(InfoSchemaDb.DATABASE_NAME) &&
!dbName.equals(MysqlDb.DATABASE_NAME)) {
- // Exclude database map take effect with higher priority over
include database map
- if (!excludeDatabaseMap.isEmpty() &&
excludeDatabaseMap.containsKey(dbName)) {
- return false;
- }
- if (!includeDatabaseMap.isEmpty() &&
!includeDatabaseMap.containsKey(dbName)) {
- return false;
- }
- }
- return true;
- }).collect(Collectors.toList());
+ allDatabases = allDatabases.stream()
+ .filter(dbName -> isDatabaseListed(dbName, includeDatabaseMap,
excludeDatabaseMap))
+ .collect(Collectors.toList());
for (String remoteDbName : allDatabases) {
remoteToLocalPairs.add(Pair.of(
@@ -1415,6 +1406,27 @@ public abstract class ExternalCatalog
throw new NotImplementedException("registerDatabaseFromEvent not
implemented");
}
+ /**
+ * Whether the database list of this catalog keeps the remote database:
information_schema and mysql always, the
+ * others as include_database_list and exclude_database_list allow. A
database that an event announces is added
+ * only when it is.
+ */
+ protected boolean isDatabaseListed(String remoteDbName) {
+ return isDatabaseListed(remoteDbName, getIncludeDatabaseMap(),
getExcludeDatabaseMap());
+ }
+
+ private static boolean isDatabaseListed(String remoteDbName, Map<String,
Boolean> includeDatabaseMap,
+ Map<String, Boolean> excludeDatabaseMap) {
+ if (remoteDbName.equals(InfoSchemaDb.DATABASE_NAME) ||
remoteDbName.equals(MysqlDb.DATABASE_NAME)) {
+ return true;
+ }
+ // Exclude database map take effect with higher priority over include
database map
+ if (!excludeDatabaseMap.isEmpty() &&
excludeDatabaseMap.containsKey(remoteDbName)) {
+ return false;
+ }
+ return includeDatabaseMap.isEmpty() ||
includeDatabaseMap.containsKey(remoteDbName);
+ }
+
protected Map<String, Boolean> getIncludeDatabaseMap() {
return getSpecifiedDatabaseMap(ExternalCatalog.INCLUDE_DATABASE_LIST);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
index d58087e1d43..95628d1eca6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java
@@ -692,6 +692,11 @@ public class PluginDrivenExternalCatalog extends
ExternalCatalog {
*/
@Override
public void registerDatabaseFromEvent(String remoteDbName, String
localDbName) {
+ // The database list leaves out what include_database_list and
exclude_database_list keep out; an event must
+ // not add it to the list either.
+ if (!isDatabaseListed(remoteDbName)) {
+ return;
+ }
long dbId = Util.genIdByName(getName(), localDbName);
try {
ExternalDatabase<? extends ExternalTable> db =
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogEventFilterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogEventFilterTest.java
new file mode 100644
index 00000000000..c9114f6ca7e
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogEventFilterTest.java
@@ -0,0 +1,118 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.plugin;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.connector.spi.Connector;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalMetaCacheMgr;
+import org.apache.doris.datasource.ExternalTable;
+import org.apache.doris.datasource.log.InitCatalogLog;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import org.mockito.MockedStatic;
+import org.mockito.Mockito;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A database that include_database_list or exclude_database_list keeps out of
a catalog's database list must not
+ * get into it through a metastore event (a CREATE DATABASE, or an ALTER
DATABASE that renames to it) either.
+ */
+public class PluginDrivenExternalCatalogEventFilterTest {
+
+ @Test
+ public void testEventDoesNotAddExcludedDatabase() {
+ assertEventKeepsOut(ExternalCatalog.EXCLUDE_DATABASE_LIST, "archived",
"archived", "sales2");
+ }
+
+ @Test
+ public void testEventDoesNotAddDatabaseOutsideIncludeList() {
+ assertEventKeepsOut(ExternalCatalog.INCLUDE_DATABASE_LIST,
"sales,sales2", "other", "sales2");
+ }
+
+ // With the database names cached, an event for keptOut leaves them as
they are, and one for allowed adds it.
+ private static void assertEventKeepsOut(String filterKey, String
filterValue, String keptOut, String allowed) {
+ Env env = Mockito.mock(Env.class);
+ ExternalMetaCacheMgr cacheMgr =
Mockito.mock(ExternalMetaCacheMgr.class);
+ Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr);
+
Mockito.when(cacheMgr.commonRefreshExecutor()).thenReturn(MoreExecutors.newDirectExecutorService());
+ try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
+ mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
+ FilteredCatalog catalog = new FilteredCatalog(filterKey,
filterValue);
+ catalog.setInitializedForTest(true);
+ Assertions.assertFalse(catalog.getDbNames().contains(keptOut));
+
+ catalog.registerDatabaseFromEvent(keptOut, keptOut);
+ Assertions.assertFalse(catalog.getDbNames().contains(keptOut));
+ Assertions.assertFalse(catalog.builtDatabases.contains(keptOut));
+
+ catalog.registerDatabaseFromEvent(allowed, allowed);
+ Assertions.assertTrue(catalog.getDbNames().contains(allowed));
+ }
+ }
+
+ private static class FilteredCatalog extends PluginDrivenExternalCatalog {
+ private final List<String> builtDatabases = new ArrayList<>();
+
+ FilteredCatalog(String filterKey, String filterValue) {
+ super(1L, "event_filter_catalog", null, properties(filterKey,
filterValue), "",
+ Mockito.mock(Connector.class));
+ }
+
+ @Override
+ protected void initLocalObjectsImpl() {
+ }
+
+ @Override
+ protected List<String> listDatabaseNames() {
+ return ImmutableList.of("sales", "archived", "other");
+ }
+
+ @Override
+ public String fromRemoteDatabaseName(String remoteDatabaseName) {
+ return remoteDatabaseName;
+ }
+
+ @Override
+ protected ExternalDatabase<? extends ExternalTable> buildDbForInit(
+ String remoteDbName, String localDbName, long dbId,
+ InitCatalogLog.Type logType, boolean checkExists) {
+ builtDatabases.add(localDbName);
+ ExternalDatabase<? extends ExternalTable> db =
Mockito.mock(ExternalDatabase.class);
+ Mockito.when(db.getId()).thenReturn(dbId);
+ Mockito.when(db.getRemoteName()).thenReturn(remoteDbName);
+ Mockito.when(db.getFullName()).thenReturn(localDbName);
+ return db;
+ }
+
+ private static Map<String, String> properties(String filterKey, String
filterValue) {
+ Map<String, String> properties = new HashMap<>();
+ properties.put("type", "hms");
+ properties.put(filterKey, filterValue);
+ return properties;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
