This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0f6c69de53 [Fix](multi-catalog) Fix sync hms event failed when start
FE soon. (#19344)
0f6c69de53 is described below
commit 0f6c69de53e9e1f29769413ce97f5be1d43d6e66
Author: Xiangyu Wang <[email protected]>
AuthorDate: Thu May 11 01:00:55 2023 +0800
[Fix](multi-catalog) Fix sync hms event failed when start FE soon. (#19344)
* [Fix](multi-catalog) Fix sync hms event failed when start FE soon after.
* [Fix](multi-catalog) Fix sync hms event failed when start FE soon after.
---------
Co-authored-by: [email protected] <[email protected]>
---
.../org/apache/doris/catalog/RefreshManager.java | 2 +-
.../org/apache/doris/datasource/CatalogMgr.java | 79 ++++++++++++++++------
.../datasource/hive/event/AddPartitionEvent.java | 2 +-
.../datasource/hive/event/AlterPartitionEvent.java | 6 +-
.../datasource/hive/event/AlterTableEvent.java | 10 +--
.../datasource/hive/event/CreateDatabaseEvent.java | 2 +-
.../datasource/hive/event/CreateTableEvent.java | 11 +--
.../datasource/hive/event/DropDatabaseEvent.java | 2 +-
.../datasource/hive/event/DropPartitionEvent.java | 2 +-
.../datasource/hive/event/DropTableEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 2 +-
11 files changed, 77 insertions(+), 43 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 62a692c6ad..f108f0f301 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -69,7 +69,7 @@ public class RefreshManager {
refreshInternalCtlIcebergTable(stmt, env);
} else {
// Process external catalog table refresh
- env.getCatalogMgr().refreshExternalTable(dbName, tableName,
catalogName);
+ env.getCatalogMgr().refreshExternalTable(dbName, tableName,
catalogName, false);
}
LOG.info("Successfully refresh table: {} from db: {}", tableName,
dbName);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 40eea74fd6..566b1d455d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -581,7 +581,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
}
- public void refreshExternalTable(String dbName, String tableName, String
catalogName) throws DdlException {
+ public void refreshExternalTable(String dbName, String tableName, String
catalogName, boolean ignoreIfNotExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -591,12 +592,18 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in
db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist
in db " + dbName);
+ }
+ return;
}
if (table instanceof ExternalTable) {
((ExternalTable) table).unsetObjectCreated();
@@ -630,7 +637,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
.invalidateTableCache(catalog.getId(), db.getFullName(),
table.getName());
}
- public void dropExternalTable(String dbName, String tableName, String
catalogName) throws DdlException {
+ public void dropExternalTable(String dbName, String tableName, String
catalogName, boolean ignoreIfExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -640,12 +648,18 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in
db " + dbName);
+ if (!ignoreIfExists) {
+ throw new DdlException("Table " + tableName + " does not exist
in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
@@ -695,7 +709,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
return ((ExternalCatalog) catalog).tableExistInLocal(dbName,
tableName);
}
- public void createExternalTable(String dbName, String tableName, String
catalogName) throws DdlException {
+ public void createExternalTable(String dbName, String tableName, String
catalogName, boolean ignoreIfExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -705,12 +720,18 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table != null) {
- throw new DdlException("Table " + tableName + " has exist in db "
+ dbName);
+ if (!ignoreIfExists) {
+ throw new DdlException("Table " + tableName + " has exist in
db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
@@ -742,7 +763,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
}
- public void dropExternalDatabase(String dbName, String catalogName) throws
DdlException {
+ public void dropExternalDatabase(String dbName, String catalogName,
boolean ignoreIfNotExists) throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -752,7 +773,10 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -785,7 +809,7 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
}
- public void createExternalDatabase(String dbName, String catalogName)
throws DdlException {
+ public void createExternalDatabase(String dbName, String catalogName,
boolean ignoreIfExists) throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " +
catalogName);
@@ -795,7 +819,10 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db != null) {
- throw new DdlException("Database " + dbName + " has exist in
catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " has exist in
catalog " + catalog.getName());
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -822,7 +849,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
}
- public void addExternalPartitions(String catalogName, String dbName,
String tableName, List<String> partitionNames)
+ public void addExternalPartitions(String catalogName, String dbName,
String tableName, List<String> partitionNames,
+ boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -833,12 +861,18 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in
db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist
in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -872,7 +906,8 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
.addPartitionsCache(catalog.getId(), table,
log.getPartitionNames());
}
- public void dropExternalPartitions(String catalogName, String dbName,
String tableName, List<String> partitionNames)
+ public void dropExternalPartitions(String catalogName, String dbName,
String tableName, List<String> partitionNames,
+ boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -883,12 +918,18 @@ public class CatalogMgr implements Writable,
GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in
catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist
in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in
db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist
in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
index e5f8d1bb71..9ced60b322 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
@@ -79,7 +79,7 @@ public class AddPartitionEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(), partitionNames);
+ .addExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(), partitionNames, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
index 5209edccab..1e2eb6d06c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
@@ -79,9 +79,11 @@ public class AlterPartitionEvent extends MetastoreTableEvent
{
catalogName, dbName, tblName, partitionNameBefore,
partitionNameAfter);
if (isRename) {
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName, tblName,
Lists.newArrayList(partitionNameBefore));
+ .dropExternalPartitions(catalogName, dbName, tblName,
+ Lists.newArrayList(partitionNameBefore), true);
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName, tblName,
Lists.newArrayList(partitionNameAfter));
+ .addExternalPartitions(catalogName, dbName, tblName,
+ Lists.newArrayList(partitionNameAfter), true);
} else {
Env.getCurrentEnv().getCatalogMgr()
.refreshExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
index 9b731461cd..902ceacfa7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -73,9 +73,9 @@ public class AlterTableEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName);
+ .dropExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalTable(tableAfter.getDbName(),
tableAfter.getTableName(), catalogName);
+ .createExternalTable(tableAfter.getDbName(),
tableAfter.getTableName(), catalogName, true);
}
private void processRename() throws DdlException {
@@ -91,9 +91,9 @@ public class AlterTableEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName);
+ .dropExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalTable(tableAfter.getDbName(),
tableAfter.getTableName(), catalogName);
+ .createExternalTable(tableAfter.getDbName(),
tableAfter.getTableName(), catalogName, true);
}
@@ -118,7 +118,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
}
//The scope of refresh can be narrowed in the future
Env.getCurrentEnv().getCatalogMgr()
- .refreshExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName);
+ .refreshExternalTable(tableBefore.getDbName(),
tableBefore.getTableName(), catalogName, true);
} catch (Exception e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 48476bce57..2205356caf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -48,7 +48,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalDatabase(dbName, catalogName);
+ .createExternalDatabase(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
index 2d364c4c14..1dbfd08ccf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
@@ -59,16 +59,7 @@ public class CreateTableEvent extends MetastoreTableEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tblName);
- boolean hasExist = Env.getCurrentEnv().getCatalogMgr()
- .externalTableExistInLocal(dbName, hmsTbl.getTableName(),
catalogName);
- if (hasExist) {
- infoLog(
- "CreateExternalTable canceled,because table has exist,"
- +
"catalogName:[{}],dbName:[{}],tableName:[{}]",
- catalogName, dbName, tblName);
- return;
- }
- Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName,
hmsTbl.getTableName(), catalogName);
+ Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName,
hmsTbl.getTableName(), catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
index b51145a258..a11e893ef2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
@@ -48,7 +48,7 @@ public class DropDatabaseEvent extends MetastoreEvent {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalDatabase(dbName, catalogName);
+ .dropExternalDatabase(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
index 1dce403f55..59254fc4f2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
@@ -79,7 +79,7 @@ public class DropPartitionEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(), partitionNames);
+ .dropExternalPartitions(catalogName, dbName,
hmsTbl.getTableName(), partitionNames, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
index aa74c67512..d42de68cbb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
@@ -59,7 +59,7 @@ public class DropTableEvent extends MetastoreTableEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]",
catalogName, dbName, tableName);
- Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName,
tableName, catalogName);
+ Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName,
tableName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
index cf4ba1d5b0..2b53e86639 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -66,7 +66,7 @@ public class InsertEvent extends MetastoreTableEvent {
* the file cache of this table,
* but <a href="https://github.com/apache/doris/pull/17932";>this
PR</a> has fixed it.
*/
- Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName,
tblName, catalogName);
+ Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName,
tblName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
