This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 94b44be3064 [Enhancement] (nereids)implement adminCleanTrashCommand in
nereids (#45051)
94b44be3064 is described below
commit 94b44be3064dcb17d47190e6bb68ebadbc351ad5
Author: wudongliang <[email protected]>
AuthorDate: Wed Dec 11 10:25:19 2024 +0800
[Enhancement] (nereids)implement adminCleanTrashCommand in nereids (#45051)
Issue Number: close #42846
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 9 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 11 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../plans/commands/AdminCleanTrashCommand.java | 120 +++++++++++++++++++++
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../auth_call/test_database_management_auth.groovy | 2 +-
.../ddl/show_trash/test_nereids_trash.groovy | 4 +
7 files changed, 147 insertions(+), 5 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index fdad6baccdc..c833e7d4b62 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -183,9 +183,9 @@ supportedCreateStatement
| CREATE (EXTERNAL)? TABLE (IF NOT EXISTS)? name=multipartIdentifier
LIKE existedTable=multipartIdentifier
(WITH ROLLUP (rollupNames=identifierList)?)?
#createTableLike
- | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?
#createRole
+ | CREATE ROLE (IF NOT EXISTS)? name=identifier (COMMENT STRING_LITERAL)?
#createRole
| CREATE WORKLOAD GROUP (IF NOT EXISTS)?
- name=identifierOrText properties=propertyClause?
#createWorkloadGroup
+ name=identifierOrText properties=propertyClause?
#createWorkloadGroup
| CREATE ROW POLICY (IF NOT EXISTS)? name=identifier
ON table=multipartIdentifier
AS type=(RESTRICTIVE | PERMISSIVE)
@@ -496,6 +496,9 @@ supportedAdminStatement
| ADMIN COMPACT TABLE baseTableRef (WHERE TYPE EQ STRING_LITERAL)?
#adminCompactTable
| ADMIN CHECK tabletList properties=propertyClause?
#adminCheckTablets
| ADMIN SHOW TABLET STORAGE FORMAT VERBOSE?
#adminShowTabletStorageFormat
+ | ADMIN CLEAN TRASH
+ (ON LEFT_PAREN backends+=STRING_LITERAL
+ (COMMA backends+=STRING_LITERAL)* RIGHT_PAREN)?
#adminCleanTrash
;
supportedRecoverStatement
@@ -517,8 +520,6 @@ unsupportedAdminStatement
(COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?
#adminRebalanceDisk
| ADMIN CANCEL REBALANCE DISK (ON LEFT_PAREN backends+=STRING_LITERAL
(COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?
#adminCancelRebalanceDisk
- | ADMIN CLEAN TRASH (ON LEFT_PAREN backends+=STRING_LITERAL
- (COMMA backends+=STRING_LITERAL) RIGHT_PAREN)?
#adminCleanTrash
| ADMIN SET TABLE name=multipartIdentifier
PARTITION VERSION properties=propertyClause?
#adminSetPartitionVersion
| ADMIN COPY TABLET tabletId=INTEGER_VALUE properties=propertyClause?
#adminCopyTablet
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 95dbbb50471..a825f80f4e0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -477,6 +477,7 @@ import
org.apache.doris.nereids.trees.plans.algebra.Aggregate;
import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterMTMVCommand;
@@ -4579,6 +4580,16 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return new ShowTrashCommand();
}
+ @Override
+ public LogicalPlan visitAdminCleanTrash(DorisParser.AdminCleanTrashContext
ctx) {
+ if (ctx.ON() != null) {
+ List<String> backendsQuery = Lists.newArrayList();
+ ctx.backends.forEach(backend ->
backendsQuery.add(stripQuotes(backend.getText())));
+ return new AdminCleanTrashCommand(backendsQuery);
+ }
+ return new AdminCleanTrashCommand();
+ }
+
@Override
public LogicalPlan
visitAdminShowReplicaStatus(AdminShowReplicaStatusContext ctx) {
Expression where = null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 91ec3591003..b0e475a8161 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -174,6 +174,7 @@ public enum PlanType {
SHOW_CREATE_PROCEDURE_COMMAND,
CREATE_VIEW_COMMAND,
CLEAN_ALL_PROFILE_COMMAND,
+ CLEAN_TRASH_COMMAND,
CREATE_ROLE_COMMAND,
ALTER_ROLE_COMMAND,
ALTER_VIEW_COMMAND,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java
new file mode 100644
index 00000000000..60a293c5123
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/AdminCleanTrashCommand.java
@@ -0,0 +1,120 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.common.util.NetUtils;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.StmtExecutor;
+import org.apache.doris.system.Backend;
+import org.apache.doris.task.AgentBatchTask;
+import org.apache.doris.task.AgentTaskExecutor;
+import org.apache.doris.task.CleanTrashTask;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * admin clean trash
+ */
+public class AdminCleanTrashCommand extends Command implements NoForward {
+ private static final Logger LOG =
LogManager.getLogger(AdminCleanTrashCommand.class);
+ private List<String> backendsQuery;
+
+ public AdminCleanTrashCommand() {
+ super(PlanType.CLEAN_TRASH_COMMAND);
+ }
+
+ public AdminCleanTrashCommand(List<String> backendsQuery) {
+ super(PlanType.CLEAN_TRASH_COMMAND);
+ this.backendsQuery = backendsQuery;
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
+ if
(!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(),
PrivPredicate.ADMIN)) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
"ADMIN");
+ }
+ handleCleanTrash(backendsQuery);
+ }
+
+ private void handleCleanTrash(List<String> backendsQuery) throws
AnalysisException {
+ List<Backend> backends = getNeedCleanedBackends(backendsQuery);
+ cleanTrash(backends);
+ }
+
+ private List<Backend> getNeedCleanedBackends(List<String> backendsQuery)
throws AnalysisException {
+ List<Backend> needCleanedBackends = Lists.newArrayList();
+ ImmutableMap<Long, Backend> backendsInfo =
Env.getCurrentSystemInfo().getAllBackendsByAllCluster();
+ if (backendsQuery == null) {
+ needCleanedBackends.addAll(backendsInfo.values());
+ } else {
+ Map<String, Long> backendsID = new HashMap<>();
+ for (Backend backend : backendsInfo.values()) {
+ backendsID.put(
+
NetUtils.getHostPortInAccessibleFormat(backend.getHost(),
backend.getHeartbeatPort()),
+ backend.getId());
+ }
+ for (String backendQuery : backendsQuery) {
+ if (backendsID.containsKey(backendQuery)) {
+
needCleanedBackends.add(backendsInfo.get(backendsID.get(backendQuery)));
+ backendsID.remove(backendQuery);
+ }
+ }
+ }
+ return needCleanedBackends;
+ }
+
+ private void cleanTrash(List<Backend> backends) {
+ if (backends.isEmpty()) {
+ LOG.info("The matching be is empty, no be to clean trash.");
+ return;
+ }
+ AgentBatchTask batchTask = new AgentBatchTask();
+ for (Backend backend : backends) {
+ CleanTrashTask cleanTrashTask = new
CleanTrashTask(backend.getId());
+ batchTask.addTask(cleanTrashTask);
+ LOG.info("clean trash in be {}, beId {}", backend.getHost(),
backend.getId());
+ }
+ AgentTaskExecutor.submit(batchTask);
+ }
+
+ @Override
+ protected void checkSupportedInCloudMode(ConnectContext ctx) throws
DdlException {
+ LOG.info("AdminCleanTrashCommand not supported in cloud mode");
+ throw new DdlException("Unsupported operation");
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitAdminCleanTrashCommand(this, context);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index 5da741f303f..3634732077b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -19,6 +19,7 @@ package org.apache.doris.nereids.trees.plans.visitor;
import org.apache.doris.nereids.trees.plans.commands.AddConstraintCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminCheckTabletsCommand;
+import org.apache.doris.nereids.trees.plans.commands.AdminCleanTrashCommand;
import org.apache.doris.nereids.trees.plans.commands.AdminCompactTableCommand;
import
org.apache.doris.nereids.trees.plans.commands.AdminShowReplicaStatusCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterJobStatusCommand;
@@ -221,6 +222,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(adminCompactTableCommand, context);
}
+ default R visitAdminCleanTrashCommand(AdminCleanTrashCommand
adminCleanTrashCommand, C context) {
+ return visitCommand(adminCleanTrashCommand, context);
+ }
+
default R visitDropConstraintCommand(DropConstraintCommand
dropConstraintCommand, C context) {
return visitCommand(dropConstraintCommand, context);
}
diff --git
a/regression-test/suites/auth_call/test_database_management_auth.groovy
b/regression-test/suites/auth_call/test_database_management_auth.groovy
index b481cf8de75..ea10b034524 100644
--- a/regression-test/suites/auth_call/test_database_management_auth.groovy
+++ b/regression-test/suites/auth_call/test_database_management_auth.groovy
@@ -111,7 +111,7 @@ suite("test_database_management_auth","p0,auth_call") {
}
test {
sql """ADMIN CLEAN TRASH;"""
- exception "denied"
+ exception "${error_in_cloud}"
}
test {
sql """RECOVER DATABASE db_name;"""
diff --git
a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
index 5d795e6fd11..26a57c7e82d 100644
--- a/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
+++ b/regression-test/suites/nereids_p0/ddl/show_trash/test_nereids_trash.groovy
@@ -19,4 +19,8 @@ suite("show_trash_nereids") {
// can not use qt command since the output change based on cluster and
backend ip
checkNereidsExecute("""show trash;""")
checkNereidsExecute("""show trash on "127.0.0.1:9050";""")
+
+ checkNereidsExecute("""ADMIN CLEAN TRASH;""")
+ checkNereidsExecute("""ADMIN CLEAN TRASH ON ("127.0.0.1:9050");""")
+ checkNereidsExecute("""ADMIN CLEAN TRASH ON ("192.168.0.1:9050",
"192.168.0.2:9050", "192.168.0.3:9050");""")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
