This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3e41c91246f [feat](Nereids) support refresh database command (#44298)
3e41c91246f is described below
commit 3e41c91246f0ed615c546e6d7f6dfc8b3e698a87
Author: Petrichor <[email protected]>
AuthorDate: Mon Dec 2 10:58:37 2024 +0800
[feat](Nereids) support refresh database command (#44298)
Issue Number: close #https://github.com/apache/doris/issues/42579
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 2 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 22 ++-
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../commands/refresh/RefreshDatabaseCommand.java | 149 +++++++++++++++++++++
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../test_nereids_refresh_catalog.out | 24 +++-
.../test_nereids_refresh_catalog.groovy | 28 +++-
7 files changed, 218 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index d30ef144e52..fd5ae1bd574 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -437,6 +437,7 @@ channelDescription
supportedRefreshStatement
: REFRESH CATALOG name=identifier propertyClause?
#refreshCatalog
+ | REFRESH DATABASE name=multipartIdentifier propertyClause?
#refreshDatabase
;
supportedCleanStatement
@@ -445,7 +446,6 @@ supportedCleanStatement
unsupportedRefreshStatement
: REFRESH TABLE name=multipartIdentifier
#refreshTable
- | REFRESH DATABASE name=multipartIdentifier propertyClause?
#refreshDatabase
| REFRESH LDAP (ALL | (FOR user=identifierOrText))
#refreshLdap
;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 1422d807b71..70c4f61e403 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -188,6 +188,7 @@ import
org.apache.doris.nereids.DorisParser.RecoverDatabaseContext;
import org.apache.doris.nereids.DorisParser.RecoverPartitionContext;
import org.apache.doris.nereids.DorisParser.RecoverTableContext;
import org.apache.doris.nereids.DorisParser.RefreshCatalogContext;
+import org.apache.doris.nereids.DorisParser.RefreshDatabaseContext;
import org.apache.doris.nereids.DorisParser.RefreshMTMVContext;
import org.apache.doris.nereids.DorisParser.RefreshMethodContext;
import org.apache.doris.nereids.DorisParser.RefreshScheduleContext;
@@ -618,6 +619,7 @@ import
org.apache.doris.nereids.trees.plans.commands.load.LoadSeparator;
import org.apache.doris.nereids.trees.plans.commands.load.LoadSequenceClause;
import org.apache.doris.nereids.trees.plans.commands.load.LoadWhereClause;
import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
+import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
import org.apache.doris.nereids.trees.plans.logical.LogicalCTE;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
@@ -4351,6 +4353,25 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
}
@Override
+ public RefreshDatabaseCommand visitRefreshDatabase(RefreshDatabaseContext
ctx) {
+ Map<String, String> properties =
visitPropertyClause(ctx.propertyClause()) == null ? Maps.newHashMap()
+ : visitPropertyClause(ctx.propertyClause());
+ List<String> parts = visitMultipartIdentifier(ctx.name);
+ int size = parts.size();
+ if (size == 0) {
+ throw new ParseException("database name can't be empty");
+ }
+ String dbName = parts.get(size - 1);
+
+ // [db].
+ if (size == 1) {
+ return new RefreshDatabaseCommand(dbName, properties);
+ } else if (parts.size() == 2) { // [ctl,db].
+ return new RefreshDatabaseCommand(parts.get(0), dbName,
properties);
+ }
+ throw new ParseException("Only one dot can be in the name: " +
String.join(".", parts));
+ }
+
public LogicalPlan visitShowLastInsert(ShowLastInsertContext ctx) {
return new ShowLastInsertCommand();
}
@@ -4379,7 +4400,6 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
partitionId = Long.parseLong(ctx.partitionId.getText());
}
return new ShowPartitionIdCommand(partitionId);
-
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 177e6b4fcff..6a8fcadaf52 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -187,6 +187,7 @@ public enum PlanType {
SET_USER_PROPERTIES_COMMAND,
SET_DEFAULT_STORAGE_VAULT_COMMAND,
REFRESH_CATALOG_COMMAND,
+ REFRESH_DATABASE_COMMAND,
PREPARED_COMMAND,
EXECUTE_COMMAND,
DROP_SQL_BLOCK_RULE_COMMAND,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java
new file mode 100644
index 00000000000..010adbb05d1
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/refresh/RefreshDatabaseCommand.java
@@ -0,0 +1,149 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands.refresh;
+
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.InfoSchemaDb;
+import org.apache.doris.catalog.MysqlDb;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalObjectLog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.Command;
+import org.apache.doris.nereids.trees.plans.commands.ForwardWithSync;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Strings;
+
+import java.util.Map;
+
+/**
+ * Refresh database.
+ */
+public class RefreshDatabaseCommand extends Command implements ForwardWithSync
{
+ private static final String INVALID_CACHE = "invalid_cache";
+
+ private String catalogName;
+ private String dbName;
+ private Map<String, String> properties;
+ private boolean invalidCache = false;
+
+ public RefreshDatabaseCommand(String dbName, Map<String, String>
properties) {
+ super(PlanType.REFRESH_DATABASE_COMMAND);
+ this.dbName = dbName;
+ this.properties = properties;
+ }
+
+ public RefreshDatabaseCommand(String catalogName, String dbName,
Map<String, String> properties) {
+ super(PlanType.REFRESH_DATABASE_COMMAND);
+ this.catalogName = catalogName;
+ this.dbName = dbName;
+ this.properties = properties;
+ }
+
+ private void validate(ConnectContext ctx) throws AnalysisException {
+ if (Strings.isNullOrEmpty(catalogName)) {
+ catalogName = ConnectContext.get().getCurrentCatalog().getName();
+ }
+ if (Strings.isNullOrEmpty(dbName)) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME,
dbName);
+ }
+
+ // Don't allow dropping 'information_schema' database
+ if (dbName.equalsIgnoreCase(InfoSchemaDb.DATABASE_NAME)) {
+
+ ErrorReport.reportAnalysisException(
+ ErrorCode.ERR_DBACCESS_DENIED_ERROR,
ctx.getQualifiedUser(), dbName);
+ }
+ // Don't allow dropping 'mysql' database
+ if (dbName.equalsIgnoreCase(MysqlDb.DATABASE_NAME)) {
+ ErrorReport.reportAnalysisException(
+ ErrorCode.ERR_DBACCESS_DENIED_ERROR,
ctx.getQualifiedUser(), dbName);
+ }
+ // check access
+ if
(!Env.getCurrentEnv().getAccessManager().checkDbPriv(ConnectContext.get(),
catalogName,
+ dbName, PrivPredicate.SHOW)) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_DB_ACCESS_DENIED_ERROR,
+ PrivPredicate.SHOW.getPrivs().toString(), dbName);
+ }
+ String invalidConfig = properties == null ? null :
properties.get(INVALID_CACHE);
+ // Default is to invalid cache.
+ invalidCache = invalidConfig == null ||
invalidConfig.equalsIgnoreCase("true");
+ }
+
+ /**
+ * Refresh database
+ */
+ public void handleRefreshDb() throws DdlException {
+ Env env = Env.getCurrentEnv();
+ CatalogIf catalog = catalogName != null ?
env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
+ if (catalog == null) {
+ throw new DdlException("Catalog " + catalogName + " doesn't
exist.");
+ }
+ if (!(catalog instanceof ExternalCatalog)) {
+ throw new DdlException("Only support refresh database in external
catalog");
+ }
+ DatabaseIf db = catalog.getDbOrDdlException(dbName);
+ ((ExternalDatabase<?>) db).setUnInitialized(invalidCache);
+
+ ExternalObjectLog log = new ExternalObjectLog();
+ log.setCatalogId(catalog.getId());
+ log.setDbId(db.getId());
+ log.setInvalidCache(invalidCache);
+ Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log);
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
+ validate(ctx);
+ handleRefreshDb();
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitRefreshDatabaseCommand(this, context);
+ }
+
+ /**
+ * refresh database statement.
+ */
+ public String toSql() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("REFRESH DATABASE ");
+ if (catalogName != null) {
+ sb.append("`").append(catalogName).append("`.");
+ }
+ sb.append("`").append(dbName).append("`");
+ return sb.toString();
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.REFRESH;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index de5228f4981..fec52fd395f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -119,6 +119,7 @@ import
org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableComma
import
org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.load.CreateRoutineLoadCommand;
import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshCatalogCommand;
+import
org.apache.doris.nereids.trees.plans.commands.refresh.RefreshDatabaseCommand;
/** CommandVisitor. */
public interface CommandVisitor<R, C> {
@@ -353,6 +354,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(showViewCommand, context);
}
+ default R visitRefreshDatabaseCommand(RefreshDatabaseCommand
refreshDatabaseCommand, C context) {
+ return visitCommand(refreshDatabaseCommand, context);
+ }
+
default R visitShowBackendsCommand(ShowBackendsCommand
showBackendsCommand, C context) {
return visitCommand(showBackendsCommand, context);
}
diff --git
a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
index 89b47bbe4eb..c3f57d7ba81 100644
---
a/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
+++
b/regression-test/data/external_table_p0/nereids_commands/test_nereids_refresh_catalog.out
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !sql --
+-- !database --
DORIS
Doris
doris
@@ -19,7 +19,7 @@ show_test_do_not_modify
114 abf
115 abg
--- !sql --
+-- !preceding_create_external_database --
DORIS
Doris
doris
@@ -29,7 +29,7 @@ init_db
mysql
show_test_do_not_modify
--- !sql --
+-- !subsequent_create_external_database --
DORIS
Doris
doris
@@ -40,7 +40,21 @@ mysql
new_mysql_db
show_test_do_not_modify
--- !sql --
+-- !sql_show_tables --
+
+-- !preceding_refresh_database --
+
+-- !subsequent_refresh_database --
+new_mysql_table1
+
+-- !preceding_refresh_database --
+new_mysql_table1
+
+-- !subsequent_refresh_database --
+new_mysql_table1
+new_mysql_table2
+
+-- !preceding_drop_external_database --
DORIS
Doris
doris
@@ -51,7 +65,7 @@ mysql
new_mysql_db
show_test_do_not_modify
--- !sql --
+-- !subsequent_drop_external_database --
DORIS
Doris
doris
diff --git
a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
index f74f78a3709..87dacd41425 100644
---
a/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
+++
b/regression-test/suites/external_table_p0/nereids_commands/test_nereids_refresh_catalog.groovy
@@ -28,6 +28,8 @@ suite("test_nereids_refresh_catalog",
"p0,external,mysql,external_docker,externa
String mysql_port = context.config.otherConfigs.get("mysql_57_port");
String ex_tb0 = "ex_tb0";
String new_mysql_db = "new_mysql_db";
+ String new_mysql_table1 = "new_mysql_table1";
+ String new_mysql_table2 = "new_mysql_table2";
sql """drop catalog if exists ${catalog_name} """
@@ -43,27 +45,41 @@ suite("test_nereids_refresh_catalog",
"p0,external,mysql,external_docker,externa
"driver_class" = "com.mysql.cj.jdbc.Driver"
);"""
- sql """switch ${catalog_name}"""
sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists
${new_mysql_db}");"""
+ sql """switch ${catalog_name}"""
- qt_sql """show databases;"""
+ qt_database """show databases;"""
sql """ use ${ex_db_name}"""
qt_ex_tb0_where """select id from ${ex_tb0} where id = 111;"""
order_qt_ex_tb0 """ select id, name from ${ex_tb0} order by id; """
// create database in mysql
sql """CALL EXECUTE_STMT("${catalog_name}", "create database
${new_mysql_db} ;");"""
- qt_sql """show databases;"""
+ qt_preceding_create_external_database """show databases;"""
checkNereidsExecute("refresh catalog ${catalog_name} ;")
- qt_sql """show databases;"""
+ qt_subsequent_create_external_database """show databases;"""
checkNereidsExecute("refresh catalog ${catalog_name} properties
('invalid_cache'='true');")
+ sql """use ${new_mysql_db}"""
+ qt_sql_show_tables """show tables;"""
+
+ // create table in mysql external database
+ sql """CALL EXECUTE_STMT("${catalog_name}", "create table
${new_mysql_db}.${new_mysql_table1} (id int, name varchar(20));");"""
+
+ qt_preceding_refresh_database """show tables;"""
+ checkNereidsExecute("refresh database ${new_mysql_db} ;")
+ qt_subsequent_refresh_database """show tables;"""
+
+ sql """CALL EXECUTE_STMT("${catalog_name}", "create table
${new_mysql_db}.${new_mysql_table2} (id int, name varchar(20));");"""
+ qt_preceding_refresh_database """show tables;"""
+ checkNereidsExecute("refresh database ${catalog_name}.${new_mysql_db}
;")
+ qt_subsequent_refresh_database """show tables;"""
sql """CALL EXECUTE_STMT("${catalog_name}", "drop database if exists
${new_mysql_db} ;");"""
- qt_sql """show databases;"""
+ qt_preceding_drop_external_database """show databases;"""
checkNereidsExecute("refresh catalog ${catalog_name} properties
('invalid_cache'='true');")
- qt_sql """show databases;"""
+ qt_subsequent_drop_external_database """show databases;"""
sql """ drop catalog if exists ${catalog_name} ;"""
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
