This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 149c376f01b branch-3.0: [fix](external catalog) Fix missing fields
when rebuilding metadata from image (#47726)
149c376f01b is described below
commit 149c376f01ba3cf758db9a4eaf1be2112699dba5
Author: zy-kkk <[email protected]>
AuthorDate: Tue Feb 11 09:39:40 2025 +0800
branch-3.0: [fix](external catalog) Fix missing fields when rebuilding
metadata from image (#47726)
cherry-pick #47603
---
.../apache/doris/datasource/ExternalCatalog.java | 23 ++++++++++++--
.../apache/doris/datasource/ExternalDatabase.java | 36 ++++++++++++++++++++--
.../apache/doris/datasource/InitCatalogLog.java | 7 ++++-
.../apache/doris/datasource/InitDatabaseLog.java | 7 ++++-
4 files changed, 66 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 05ad1c07cb0..7a81076773b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -390,7 +390,7 @@ public abstract class ExternalCatalog
db.setRemoteName(remoteDbName);
}
tmpIdToDb.put(dbId, db);
- initCatalogLog.addRefreshDb(dbId);
+ initCatalogLog.addRefreshDb(dbId, remoteDbName);
} else {
dbId = Env.getCurrentEnv().getNextId();
tmpDbNameToId.put(localDbName, dbId);
@@ -722,8 +722,12 @@ public abstract class ExternalCatalog
}
public void replayInitCatalog(InitCatalogLog log) {
- // If the remote name is missing during upgrade, all databases in the
Map will be reinitialized.
- if (log.getCreateCount() > 0 && (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty())) {
+ // If the remote name is missing during upgrade, or
+ // the refresh db's remote name is empty,
+ // all databases in the Map will be reinitialized.
+ if ((log.getCreateCount() > 0 && (log.getRemoteDbNames() == null ||
log.getRemoteDbNames().isEmpty()))
+ || (log.getRefreshCount() > 0
+ && (log.getRefreshRemoteDbNames() == null ||
log.getRefreshRemoteDbNames().isEmpty()))) {
dbNameToId = Maps.newConcurrentMap();
idToDb = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
@@ -743,6 +747,7 @@ public abstract class ExternalCatalog
log.getRefreshDbIds().get(i), name);
continue;
}
+ db.get().setRemoteName(log.getRefreshRemoteDbNames().get(i));
Preconditions.checkNotNull(db.get());
tmpDbNameToId.put(db.get().getFullName(), db.get().getId());
tmpIdToDb.put(db.get().getId(), db.get());
@@ -759,6 +764,18 @@ public abstract class ExternalCatalog
db.getFullName(), db.getId(),
log.getRemoteDbNames().get(i));
}
}
+ // Check whether the remoteName of db in tmpIdToDb is empty
+ for (ExternalDatabase<? extends ExternalTable> db :
tmpIdToDb.values()) {
+ if (Strings.isNullOrEmpty(db.getRemoteName())) {
+ LOG.info("Database [{}] remoteName is empty in catalog [{}],
mark as uninitialized",
+ db.getFullName(), name);
+ dbNameToId = Maps.newConcurrentMap();
+ idToDb = Maps.newConcurrentMap();
+ lastUpdateTime = log.getLastUpdateTime();
+ initialized = false;
+ return;
+ }
+ }
dbNameToId = tmpDbNameToId;
idToDb = tmpIdToDb;
lastUpdateTime = log.getLastUpdateTime();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index cbacf563c32..ea02a20a07b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -191,7 +191,9 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
// If the remote name is missing during upgrade, all tables in the Map
will be reinitialized.
- if (log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty())) {
+ if ((log.getCreateCount() > 0 && (log.getRemoteTableNames() == null ||
log.getRemoteTableNames().isEmpty()))
+ || (log.getRefreshCount() > 0
+ && (log.getRefreshRemoteTableNames() == null ||
log.getRefreshRemoteTableNames().isEmpty()))) {
tableNameToId = Maps.newConcurrentMap();
idToTbl = Maps.newConcurrentMap();
lastUpdateTime = log.getLastUpdateTime();
@@ -209,6 +211,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
// So we need add a validation here to avoid table(s) not found,
this is just a temporary solution
// because later we will remove all the logics about
InitCatalogLog/InitDatabaseLog.
if (table.isPresent()) {
+
table.get().setRemoteName(log.getRefreshRemoteTableNames().get(i));
tmpTableNameToId.put(table.get().getName(),
table.get().getId());
tmpIdToTbl.put(table.get().getId(), table.get());
@@ -234,6 +237,19 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
LOG.info("Synchronized table (create): [Name: {}, ID: {}, Remote
Name: {}]",
table.getName(), table.getId(),
log.getRemoteTableNames().get(i));
}
+ // Check whether the remoteName and db Tbl db in idToTbl is empty
+ for (T table : idToTbl.values()) {
+ if (Strings.isNullOrEmpty(table.getRemoteName())
+ || table.getDb() == null) {
+ LOG.info("Table [{}] remoteName or database is empty, mark as
uninitialized",
+ table.getName());
+ tableNameToId = Maps.newConcurrentMap();
+ idToTbl = Maps.newConcurrentMap();
+ lastUpdateTime = log.getLastUpdateTime();
+ initialized = false;
+ return;
+ }
+ }
tableNameToId = tmpTableNameToId;
idToTbl = tmpIdToTbl;
lastUpdateTime = log.getLastUpdateTime();
@@ -266,7 +282,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
table.setDb(this);
}
tmpIdToTbl.put(tblId, table);
- initDatabaseLog.addRefreshTable(tblId);
+ initDatabaseLog.addRefreshTable(tblId, remoteTableName);
} else {
tblId = Env.getCurrentEnv().getNextId();
tmpTableNameToId.put(localTableName, tblId);
@@ -620,14 +636,22 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
case "ExternalInfoSchemaTable":
ExternalInfoSchemaTable infoSchemaTable =
GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(obj),
ExternalInfoSchemaTable.class);
+ if (infoSchemaTable.getDb() == null) {
+ infoSchemaTable.setDb(this);
+ }
tmpIdToTbl.put(infoSchemaTable.getId(), (T)
infoSchemaTable);
tableNameToId.put(infoSchemaTable.getName(),
infoSchemaTable.getId());
+
lowerCaseToTableName.put(infoSchemaTable.getName().toLowerCase(),
infoSchemaTable.getName());
break;
case "ExternalMysqlTable":
ExternalMysqlTable mysqlTable =
GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(obj),
ExternalMysqlTable.class);
+ if (mysqlTable.getDb() == null) {
+ mysqlTable.setDb(this);
+ }
tmpIdToTbl.put(mysqlTable.getId(), (T) mysqlTable);
tableNameToId.put(mysqlTable.getName(),
mysqlTable.getId());
+
lowerCaseToTableName.put(mysqlTable.getName().toLowerCase(),
mysqlTable.getName());
break;
default:
break;
@@ -640,6 +664,14 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
((ExternalTable) obj).getName());
}
}
+ // Check whether the remoteName and db Tbl db in idToTbl is empty
+ for (T table : idToTbl.values()) {
+ if (Strings.isNullOrEmpty(table.getRemoteName())
+ || table.getDb() == null) {
+ initialized = false;
+ break;
+ }
+ }
idToTbl = tmpIdToTbl;
rwLock = new MonitoredReentrantReadWriteLock(true);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
index 023eecc2fa4..ac262764de6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java
@@ -58,6 +58,9 @@ public class InitCatalogLog implements Writable {
@SerializedName(value = "refreshDbIds")
private List<Long> refreshDbIds;
+ @SerializedName(value = "refreshRemoteDbNames")
+ private List<String> refreshRemoteDbNames;
+
@SerializedName(value = "createDbIds")
private List<Long> createDbIds;
@@ -78,15 +81,17 @@ public class InitCatalogLog implements Writable {
createCount = 0;
catalogId = 0;
refreshDbIds = Lists.newArrayList();
+ refreshRemoteDbNames = Lists.newArrayList();
createDbIds = Lists.newArrayList();
createDbNames = Lists.newArrayList();
remoteDbNames = Lists.newArrayList();
type = Type.UNKNOWN;
}
- public void addRefreshDb(long id) {
+ public void addRefreshDb(long id, String remoteName) {
refreshCount += 1;
refreshDbIds.add(id);
+ refreshRemoteDbNames.add(remoteName);
}
public void addCreateDb(long id, String name, String remoteName) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
index 44ee0a39c56..d1ea04a16b0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java
@@ -62,6 +62,9 @@ public class InitDatabaseLog implements Writable {
@SerializedName(value = "refreshTableIds")
private List<Long> refreshTableIds;
+ @SerializedName(value = "refreshRemoteTableNames")
+ private List<String> refreshRemoteTableNames;
+
@SerializedName(value = "createTableIds")
private List<Long> createTableIds;
@@ -83,15 +86,17 @@ public class InitDatabaseLog implements Writable {
catalogId = 0;
dbId = 0;
refreshTableIds = Lists.newArrayList();
+ refreshRemoteTableNames = Lists.newArrayList();
createTableIds = Lists.newArrayList();
createTableNames = Lists.newArrayList();
remoteTableNames = Lists.newArrayList();
type = Type.UNKNOWN;
}
- public void addRefreshTable(long id) {
+ public void addRefreshTable(long id, String remoteName) {
refreshCount += 1;
refreshTableIds.add(id);
+ refreshRemoteTableNames.add(remoteName);
}
public void addCreateTable(long id, String name, String remoteName) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
