This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 394af9c3cab [Enhancement] (nereids) implement backupCommand in
nereids (#50589)
394af9c3cab is described below
commit 394af9c3cabf9515fff8c466a09cff4175936a09
Author: yaoxiao <[email protected]>
AuthorDate: Thu Jun 12 10:50:11 2025 +0800
[Enhancement] (nereids) implement backupCommand in nereids (#50589)
Issue Number: #42830
---
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 6 +-
.../org/apache/doris/backup/BackupHandler.java | 205 +++++++++++++++
.../doris/nereids/parser/LogicalPlanBuilder.java | 25 ++
.../apache/doris/nereids/trees/TableSample.java | 4 +
.../apache/doris/nereids/trees/plans/PlanType.java | 1 +
.../trees/plans/commands/BackupCommand.java | 274 +++++++++++++++++++++
.../trees/plans/commands/info/TableRefInfo.java | 36 +++
.../trees/plans/visitor/CommandVisitor.java | 5 +
.../trees/plans/commands/BackupCommandTest.java | 125 ++++++++++
9 files changed, 678 insertions(+), 3 deletions(-)
diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
index 8780868b7ce..c56234dea7e 100644
--- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
+++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
@@ -458,15 +458,15 @@ supportedOtherStatement
| INSTALL PLUGIN FROM source=identifierOrText properties=propertyClause?
#installPlugin
| UNINSTALL PLUGIN name=identifierOrText
#uninstallPlugin
| LOCK TABLES (lockTable (COMMA lockTable)*)?
#lockTables
+ | BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier
+ ((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)*
RIGHT_PAREN)?
+ properties=propertyClause?
#backup
;
unsupportedOtherStatement
: WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH
((CLUSTER | COMPUTE GROUP) source=identifier |
(warmUpItem (AND warmUpItem)*)) FORCE?
#warmUpCluster
- | BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier
- ((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)*
RIGHT_PAREN)?
- properties=propertyClause?
#backup
| RESTORE SNAPSHOT label=multipartIdentifier FROM repo=identifier
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)*
RIGHT_PAREN)?
properties=propertyClause?
#restore
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 294d9b3bed5..a79f962d747 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -54,8 +54,11 @@ import org.apache.doris.fs.FileSystemFactory;
import org.apache.doris.fs.remote.AzureFileSystem;
import org.apache.doris.fs.remote.RemoteFileSystem;
import org.apache.doris.fs.remote.S3FileSystem;
+import org.apache.doris.nereids.trees.plans.commands.BackupCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelBackupCommand;
import org.apache.doris.nereids.trees.plans.commands.CreateRepositoryCommand;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
import org.apache.doris.persist.BarrierLog;
import org.apache.doris.task.DirMoveTask;
import org.apache.doris.task.DownloadTask;
@@ -78,6 +81,7 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
import java.util.Deque;
import java.util.HashMap;
import java.util.LinkedList;
@@ -381,6 +385,48 @@ public class BackupHandler extends MasterDaemon implements
Writable {
}
}
+ public void process(BackupCommand command) throws DdlException {
+ if (Config.isCloudMode()) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "BACKUP and RESTORE are not supported by the cloud mode
yet");
+ }
+
+ // check if repo exist
+ String repoName = command.getRepoName();
+ Repository repository = null;
+ if (!repoName.equals(Repository.KEEP_ON_LOCAL_REPO_NAME)) {
+ repository = repoMgr.getRepo(repoName);
+ if (repository == null) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "Repository " + repoName + " does not exist");
+ }
+ }
+
+ // check if db exist
+ String dbName = command.getDbName();
+ Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
+
+ // Try to get sequence lock.
+ // We expect at most one operation on a repo at same time.
+ // But this operation may take a few seconds with lock held.
+ // So we use tryLock() to give up this operation if we can not get
lock.
+ tryLock();
+ try {
+ // Check if there is backup or restore job running on this database
+ AbstractJob currentJob = getCurrentJob(db.getId());
+ if (currentJob != null && !currentJob.isDone()) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "Can only run one backup or restore job of a database
at same time "
+ + ", current running: label = " +
currentJob.getLabel() + " jobId = "
+ + currentJob.getJobId() + ", to run label = " +
command.getLabel());
+ }
+
+ backup(repository, db, command);
+ } finally {
+ seqlock.unlock();
+ }
+ }
+
// the entry method of submitting a backup or restore job
public void process(AbstractBackupStmt stmt) throws DdlException {
if (Config.isCloudMode()) {
@@ -440,6 +486,165 @@ public class BackupHandler extends MasterDaemon
implements Writable {
}
}
+ private void backup(Repository repository, Database db, BackupCommand
command) throws DdlException {
+ if (repository != null && repository.isReadOnly()) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Repository " + repository.getName()
+ + " is read only");
+ }
+
+ long commitSeq = 0;
+ Set<String> tableNames = Sets.newHashSet();
+
+ List<TableRefInfo> tableRefInfos = command.getTableRefInfos();
+
+ // Obtain the snapshot commit seq, any creating table binlog will be
visible.
+ db.readLock();
+ try {
+ BarrierLog log = new BarrierLog(db.getId(), db.getFullName());
+ commitSeq = env.getEditLog().logBarrier(log);
+
+ // Determine the tables to be backed up
+ if (tableRefInfos.isEmpty()) {
+ tableNames = db.getTableNames();
+ } else if (command.isExclude()) {
+ tableNames = db.getTableNames();
+ for (TableRefInfo tableRefInfo : tableRefInfos) {
+ if
(!tableNames.remove(tableRefInfo.getTableNameInfo().getTbl())) {
+ LOG.info("exclude table " +
tableRefInfo.getTableNameInfo().getTbl()
+ + " of backup stmt is not exists in db " +
db.getFullName());
+ }
+ }
+ }
+ } finally {
+ db.readUnlock();
+ }
+
+ List<TableRef> tblRefs = Lists.newArrayList();
+ if (!tableRefInfos.isEmpty() && !command.isExclude()) {
+ for (TableRefInfo tableRefInfo : tableRefInfos) {
+ tblRefs.add(tableRefInfo.translateToLegacyTableRef());
+ }
+ } else {
+ for (String tableName : tableNames) {
+ TableRefInfo tableRefInfo = new TableRefInfo(new
TableNameInfo(db.getFullName(), tableName),
+ null,
+ null,
+ null,
+ new ArrayList<>(),
+ null,
+ null,
+ new ArrayList<>());
+ tblRefs.add(tableRefInfo.translateToLegacyTableRef());
+ }
+ }
+
+ // Check if backup objects are valid
+ // This is just a pre-check to avoid most of invalid backup requests.
+ // Also calculate the signature for incremental backup check.
+ List<TableRef> tblRefsNotSupport = Lists.newArrayList();
+ for (TableRef tableRef : tblRefs) {
+ String tblName = tableRef.getName().getTbl();
+ Table tbl = db.getTableOrDdlException(tblName);
+
+ // filter the table types which are not supported by local backup.
+ if (repository == null && tbl.getType() != TableType.OLAP
+ && tbl.getType() != TableType.VIEW && tbl.getType() !=
TableType.MATERIALIZED_VIEW) {
+ tblRefsNotSupport.add(tableRef);
+ continue;
+ }
+
+ if (tbl.getType() == TableType.VIEW || tbl.getType() ==
TableType.ODBC
+ || tbl.getType() == TableType.MATERIALIZED_VIEW) {
+ continue;
+ }
+ if (tbl.getType() != TableType.OLAP) {
+ if (Config.ignore_backup_not_support_table_type) {
+ LOG.warn("Table '{}' is a {} table, can not backup and
ignore it."
+ + "Only OLAP(Doris)/ODBC/VIEW table can be backed
up",
+ tblName, tbl.getType().toString());
+ tblRefsNotSupport.add(tableRef);
+ continue;
+ } else {
+
ErrorReport.reportDdlException(ErrorCode.ERR_NOT_OLAP_TABLE, tblName);
+ }
+ }
+
+ if (tbl.isTemporary()) {
+ if (Config.ignore_backup_not_support_table_type ||
tblRefs.size() > 1) {
+ LOG.warn("Table '{}' is a temporary table, can not backup
and ignore it."
+ + "Only OLAP(Doris)/ODBC/VIEW table can be backed
up",
+ Util.getTempTableDisplayName(tblName));
+ tblRefsNotSupport.add(tableRef);
+ continue;
+ } else {
+ ErrorReport.reportDdlException("Table " +
Util.getTempTableDisplayName(tblName)
+ + " is a temporary table, do not support backup");
+ }
+ }
+
+ OlapTable olapTbl = (OlapTable) tbl;
+ tbl.readLock();
+ try {
+ if (!Config.ignore_backup_tmp_partitions &&
olapTbl.existTempPartitions()) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "Do not support backup table " + olapTbl.getName()
+ " with temp partitions");
+ }
+
+ PartitionNames partitionNames = tableRef.getPartitionNames();
+ if (partitionNames != null) {
+ if (!Config.ignore_backup_tmp_partitions &&
partitionNames.isTemp()) {
+
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "Do not support backup temp partitions in
table " + tableRef.getName());
+ }
+
+ for (String partName : partitionNames.getPartitionNames())
{
+ Partition partition = olapTbl.getPartition(partName);
+ if (partition == null) {
+
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
+ "Unknown partition " + partName + " in
table" + tblName);
+ }
+ }
+ }
+ } finally {
+ tbl.readUnlock();
+ }
+ }
+
+ tblRefs.removeAll(tblRefsNotSupport);
+
+ // Check if label already be used
+ long repoId = Repository.KEEP_ON_LOCAL_REPO_ID;
+ if (repository != null) {
+ List<String> existSnapshotNames = Lists.newArrayList();
+ Status st = repository.listSnapshots(existSnapshotNames);
+ if (!st.ok()) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
st.getErrMsg());
+ }
+ if (existSnapshotNames.contains(command.getLabel())) {
+ if (command.getBackupType() == BackupCommand.BackupType.FULL) {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Snapshot with name '"
+ + command.getLabel() + "' already exist in
repository");
+ } else {
+ ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Currently does not support "
+ + "incremental backup");
+ }
+ }
+ repoId = repository.getId();
+ }
+
+ // Create a backup job
+ BackupJob backupJob = new BackupJob(command.getLabel(), db.getId(),
+ ClusterNamespace.getNameFromFullName(db.getFullName()),
+ tblRefs, command.getTimeoutMs(),
command.translateToLagecyContent(), env, repoId, commitSeq);
+ // write log
+ env.getEditLog().logBackupJob(backupJob);
+
+ // must put to dbIdToBackupOrRestoreJob after edit log, otherwise the
state of job may be changed.
+ addBackupOrRestoreJob(db.getId(), backupJob);
+
+ LOG.info("finished to submit backup job: {}", backupJob);
+ }
+
private void backup(Repository repository, Database db, BackupStmt stmt)
throws DdlException {
if (repository != null && repository.isReadOnly()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Repository " + repository.getName()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 280aa909dd3..bdebfecaa38 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -592,6 +592,7 @@ import
org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
import
org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
import org.apache.doris.nereids.trees.plans.commands.AnalyzeDatabaseCommand;
import org.apache.doris.nereids.trees.plans.commands.AnalyzeTableCommand;
+import org.apache.doris.nereids.trees.plans.commands.BackupCommand;
import org.apache.doris.nereids.trees.plans.commands.CallCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelBackupCommand;
@@ -8177,6 +8178,30 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
return new RefreshDictionaryCommand(dbName, dictName);
}
+ @Override
+ public LogicalPlan visitBackup(DorisParser.BackupContext ctx) {
+ List<String> labelParts = visitMultipartIdentifier(ctx.label);
+ String snapshotName;
+ String dbName = null;
+ if (labelParts.size() == 1) {
+ snapshotName = labelParts.get(0);
+ } else if (labelParts.size() == 2) {
+ dbName = labelParts.get(0);
+ snapshotName = labelParts.get(1);
+ } else {
+ throw new ParseException("only support
[<db_name>.]<snapshot_name>", ctx.label);
+ }
+ LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, snapshotName);
+ String repoName = ctx.repo.getText();
+ boolean isExclude = ctx.EXCLUDE() != null;
+ List<TableRefInfo> tableRefInfos = new ArrayList<>();
+ for (BaseTableRefContext baseTableRefContext : ctx.baseTableRef()) {
+ tableRefInfos.add(visitBaseTableRefContext(baseTableRefContext));
+ }
+ Map<String, String> properties = visitPropertyClause(ctx.properties);
+ return new BackupCommand(labelNameInfo, repoName, tableRefInfos,
properties, isExclude);
+ }
+
@Override
public LogicalPlan visitShowRoutineLoad(DorisParser.ShowRoutineLoadContext
ctx) {
LabelNameInfo labelNameInfo = null;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TableSample.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TableSample.java
index da300cd442d..b0949460fb0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TableSample.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/TableSample.java
@@ -58,4 +58,8 @@ public class TableSample {
public int hashCode() {
return Objects.hash(sampleValue, isPercent, seek);
}
+
+ public org.apache.doris.analysis.TableSample
translateToLegacyTableSample() {
+ return new org.apache.doris.analysis.TableSample(this.isPercent,
this.sampleValue, this.seek);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
index 8762e4ecf6e..364f0aef223 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java
@@ -402,6 +402,7 @@ public enum PlanType {
CANCEL_BACKUP_AND_RESTORE_COMMAND,
CANCEL_BUILD_INDEX_COMMAND,
CREATE_RESOURCE_COMMAND,
+ BACKUP_COMMAND,
CREATE_DATA_SYNC_JOB_COMMAND,
CREATE_STAGE_COMMAND,
DROP_STAGE_COMMAND,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java
new file mode 100644
index 00000000000..7a275b1a716
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/BackupCommand.java
@@ -0,0 +1,274 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.analysis.BackupStmt;
+import org.apache.doris.analysis.StmtType;
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.ErrorReport;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.GlobalVariable;
+import org.apache.doris.qe.StmtExecutor;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * BackupCommand
+ */
+public class BackupCommand extends Command implements ForwardWithSync {
+ private static final Logger LOG =
LogManager.getLogger(BackupCommand.class);
+ private static final String PROP_TIMEOUT = "timeout";
+ private static final long MIN_TIMEOUT_MS = 600 * 1000L;
+ private static final String PROP_TYPE = "type";
+ private static final String PROP_CONTENT = "content";
+
+ /**
+ * BackupType
+ */
+ public enum BackupType {
+ INCREMENTAL, FULL
+ }
+
+ /**
+ * BackupContent
+ */
+ public enum BackupContent {
+ METADATA_ONLY, ALL
+ }
+
+ private BackupType type = BackupType.FULL;
+ private BackupContent content = BackupContent.ALL;
+
+ private final LabelNameInfo labelNameInfo;
+ private final String repoName;
+ private final List<TableRefInfo> tableRefInfos;
+ private final Map<String, String> properties;
+ private final boolean isExclude;
+
+ private long timeoutMs;
+
+ /**
+ * BackupCommand
+ */
+ public BackupCommand(LabelNameInfo labelNameInfo,
+ String repoName,
+ List<TableRefInfo> tableRefInfos,
+ Map<String, String> properties,
+ boolean isExclude) {
+ super(PlanType.BACKUP_COMMAND);
+ Objects.requireNonNull(labelNameInfo, "labelNameInfo is null");
+ Objects.requireNonNull(repoName, "repoName is null");
+ Objects.requireNonNull(tableRefInfos, "tableRefInfos is null");
+ Objects.requireNonNull(properties, "properties is null");
+ this.labelNameInfo = labelNameInfo;
+ this.repoName = repoName;
+ this.tableRefInfos = tableRefInfos;
+ this.properties = properties;
+ this.isExclude = isExclude;
+ }
+
+ @Override
+ public void run(ConnectContext ctx, StmtExecutor executor) throws
Exception {
+ validate(ctx);
+ ctx.getEnv().getBackupHandler().process(this);
+ }
+
+ /**
+ * validate
+ */
+ public void validate(ConnectContext ctx) throws AnalysisException {
+ labelNameInfo.validate(ctx);
+
+ // user need database level privilege(not table level),
+ if (!Env.getCurrentEnv().getAccessManager()
+ .checkDbPriv(ConnectContext.get(),
InternalCatalog.INTERNAL_CATALOG_NAME,
+ labelNameInfo.getDb(), PrivPredicate.LOAD)) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR,
"LOAD");
+ }
+
+ analyzeTableRefInfo();
+ analyzeProperties();
+ }
+
+ private void analyzeTableRefInfo() throws AnalysisException {
+ if (tableRefInfos.isEmpty()) {
+ return;
+ }
+ checkTableRefWithoutDatabase();
+ updateTableRefInfos();
+ // tbl refs can not set alias in backup
+ for (TableRefInfo tableRefInfo : tableRefInfos) {
+ if (tableRefInfo.hasAlias()) {
+ throw new AnalysisException("Can not set alias for table in
Backup Command: " + tableRefInfo);
+ }
+ }
+ }
+
+ private void checkTableRefWithoutDatabase() throws AnalysisException {
+ for (TableRefInfo tableRef : tableRefInfos) {
+ if (!Strings.isNullOrEmpty(tableRef.getTableNameInfo().getDb())) {
+ throw new AnalysisException("Cannot specify database name on
backup objects: "
+ + tableRef.getTableNameInfo().getTbl() + ". Specify
database name before label");
+ }
+ // set db name because we can not persist empty string when
writing bdbje log
+ tableRef.getTableNameInfo().setDb(labelNameInfo.getDb());
+ }
+ }
+
+ private void updateTableRefInfos() throws AnalysisException {
+ Map<String, TableRefInfo> tblPartsMap;
+ if (GlobalVariable.lowerCaseTableNames == 0) {
+ // comparisons case sensitive
+ tblPartsMap = Maps.newTreeMap();
+ } else {
+ tblPartsMap = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER);
+ }
+
+ for (TableRefInfo tableRefInfo : tableRefInfos) {
+ String tableName = tableRefInfo.getTableNameInfo().getTbl();
+ if (!tblPartsMap.containsKey(tableName)) {
+ tblPartsMap.put(tableName, tableRefInfo);
+ } else {
+ throw new AnalysisException("Duplicated table: " + tableName);
+ }
+ }
+
+ // update table ref
+ tableRefInfos.clear();
+ tableRefInfos.addAll(tblPartsMap.values());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("table refs after normalization: {}",
Joiner.on(",").join(tableRefInfos));
+ }
+ }
+
+ private void analyzeProperties() throws AnalysisException {
+ // timeout
+ if (properties.containsKey("timeout")) {
+ try {
+ timeoutMs = Long.valueOf(properties.get(PROP_TIMEOUT));
+ } catch (NumberFormatException e) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid timeout format: " +
properties.get(PROP_TIMEOUT));
+ }
+
+ if (timeoutMs * 1000 < MIN_TIMEOUT_MS) {
+
ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR, "timeout must
be at least 10 min");
+ }
+
+ timeoutMs = timeoutMs * 1000;
+ properties.remove(PROP_TIMEOUT);
+ } else {
+ timeoutMs = Config.backup_job_default_timeout_ms;
+ }
+
+ Map<String, String> copiedProperties = Maps.newHashMap(properties);
+ // type
+ String typeProp = copiedProperties.get(PROP_TYPE);
+ if (typeProp != null) {
+ try {
+ type = BackupType.valueOf(typeProp.toUpperCase());
+ } catch (Exception e) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid backup job type: " + typeProp);
+ }
+ copiedProperties.remove(PROP_TYPE);
+ }
+ // content
+ String contentProp = copiedProperties.get(PROP_CONTENT);
+ if (contentProp != null) {
+ try {
+ content = BackupContent.valueOf(contentProp.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Invalid backup job content:" + contentProp);
+ }
+ copiedProperties.remove(PROP_CONTENT);
+ }
+
+ if (!copiedProperties.isEmpty()) {
+ ErrorReport.reportAnalysisException(ErrorCode.ERR_COMMON_ERROR,
+ "Unknown backup job properties: " +
copiedProperties.keySet());
+ }
+ }
+
+ public List<TableRefInfo> getTableRefInfos() {
+ return tableRefInfos;
+ }
+
+ public long getTimeoutMs() {
+ return timeoutMs;
+ }
+
+ public boolean isExclude() {
+ return isExclude;
+ }
+
+ public String getRepoName() {
+ return repoName;
+ }
+
+ public BackupType getBackupType() {
+ return type;
+ }
+
+ public BackupStmt.BackupContent translateToLagecyContent() {
+ return BackupStmt.BackupContent.valueOf(content.name());
+ }
+
+ public String getLabel() {
+ return labelNameInfo.getLabel();
+ }
+
+ public String getDbName() {
+ return labelNameInfo.getDb();
+ }
+
+ @Override
+ public <R, C> R accept(PlanVisitor<R, C> visitor, C context) {
+ return visitor.visitBackupCommand(this, context);
+ }
+
+ @Override
+ public StmtType stmtType() {
+ return StmtType.BACKUP;
+ }
+
+ @Override
+ protected void checkSupportedInCloudMode(ConnectContext ctx) throws
DdlException {
+ LOG.info("BackupCommand not supported in cloud mode");
+ throw new DdlException("denied");
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
index 1a221b08f9e..0d3f142d86d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/TableRefInfo.java
@@ -18,12 +18,16 @@
package org.apache.doris.nereids.trees.plans.commands.info;
+import org.apache.doris.analysis.PartitionNames;
+import org.apache.doris.analysis.TableName;
+import org.apache.doris.analysis.TableRef;
import org.apache.doris.analysis.TableScanParams;
import org.apache.doris.analysis.TableSnapshot;
import org.apache.doris.common.UserException;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.qe.ConnectContext;
+import java.util.ArrayList;
import java.util.List;
/**
@@ -75,4 +79,36 @@ public class TableRefInfo {
partitionNamesInfo.validate();
}
}
+
+ public String getTableAlias() {
+ return tableAlias;
+ }
+
+ public boolean hasAlias() {
+ return tableAlias != null;
+ }
+
+ /**
+ * translateToLegacyTableRef
+ */
+ public TableRef translateToLegacyTableRef() {
+ TableName tableName = new TableName(tableNameInfo.getCtl(),
tableNameInfo.getDb(), tableNameInfo.getTbl());
+ String alias = tableAlias;
+ PartitionNames partitionNames =
+ partitionNamesInfo != null ?
partitionNamesInfo.translateToLegacyPartitionNames() : null;
+ ArrayList<Long> sampleTabletIds = tabletIdList != null ? new
ArrayList<>(tabletIdList) : new ArrayList<>();
+ org.apache.doris.analysis.TableSample legacyTableSample =
+ tableSample != null ?
tableSample.translateToLegacyTableSample() : null;
+ ArrayList<String> commonHints = relationHints != null ? new
ArrayList<>(relationHints) : new ArrayList<>();
+ TableSnapshot tableSnapshot = tableSnapShot;
+ TableScanParams tableScanParams = scanParams;
+ return new TableRef(tableName,
+ alias,
+ partitionNames,
+ sampleTabletIds,
+ legacyTableSample,
+ commonHints,
+ tableSnapshot,
+ tableScanParams);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
index ae852b9164d..a59b09ca4a5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/CommandVisitor.java
@@ -47,6 +47,7 @@ import
org.apache.doris.nereids.trees.plans.commands.AlterUserCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterViewCommand;
import org.apache.doris.nereids.trees.plans.commands.AlterWorkloadGroupCommand;
import
org.apache.doris.nereids.trees.plans.commands.AlterWorkloadPolicyCommand;
+import org.apache.doris.nereids.trees.plans.commands.BackupCommand;
import org.apache.doris.nereids.trees.plans.commands.CallCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelAlterTableCommand;
import org.apache.doris.nereids.trees.plans.commands.CancelBackupCommand;
@@ -1240,6 +1241,10 @@ public interface CommandVisitor<R, C> {
return visitCommand(createResourceCommand, context);
}
+ default R visitBackupCommand(BackupCommand backupCommand, C context) {
+ return visitCommand(backupCommand, context);
+ }
+
default R visitRefreshLdapCommand(RefreshLdapCommand command, C context) {
return visitCommand(command, context);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/BackupCommandTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/BackupCommandTest.java
new file mode 100644
index 00000000000..7510e2468e1
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/BackupCommandTest.java
@@ -0,0 +1,125 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.commands;
+
+import org.apache.doris.catalog.Env;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.datasource.InternalCatalog;
+import org.apache.doris.mysql.privilege.AccessControllerManager;
+import org.apache.doris.mysql.privilege.PrivPredicate;
+import org.apache.doris.nereids.trees.plans.commands.info.LabelNameInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.PartitionNamesInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.TableNameInfo;
+import org.apache.doris.nereids.trees.plans.commands.info.TableRefInfo;
+import org.apache.doris.qe.ConnectContext;
+
+import mockit.Expectations;
+import mockit.Mocked;
+import org.apache.commons.collections.map.HashedMap;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+public class BackupCommandTest {
+ private static final String internalCtl =
InternalCatalog.INTERNAL_CATALOG_NAME;
+ @Mocked
+ private Env env;
+ @Mocked
+ private AccessControllerManager accessControllerManager;
+ @Mocked
+ private ConnectContext connectContext;
+
+ private String dbName = "test_db";
+
+ private void runBefore() {
+ new Expectations() {
+ {
+ Env.getCurrentEnv();
+ minTimes = 0;
+ result = env;
+
+ env.getAccessManager();
+ minTimes = 0;
+ result = accessControllerManager;
+
+ ConnectContext.get();
+ minTimes = 0;
+ result = connectContext;
+
+ connectContext.isSkipAuth();
+ minTimes = 0;
+ result = true;
+
+ accessControllerManager.checkDbPriv(connectContext,
internalCtl, dbName, PrivPredicate.LOAD);
+ minTimes = 0;
+ result = true;
+ }
+ };
+ }
+
+ @Test
+ public void testValidateNormal() {
+ runBefore();
+ LabelNameInfo labelNameInfo = new LabelNameInfo(dbName, "label0");
+ String repoName = "testRepo";
+
+ TableNameInfo tableNameInfo = new TableNameInfo(internalCtl, null,
"test_tbl");
+ List<String> partitionNames = new ArrayList<>();
+ partitionNames.add("p1");
+ partitionNames.add("p2");
+ PartitionNamesInfo partitionNamesInfo = new PartitionNamesInfo(false,
partitionNames);
+ TableRefInfo tableRefInfo1 = new TableRefInfo(tableNameInfo, null,
null, partitionNamesInfo, null, null, null, null);
+
+ TableNameInfo tableNameInfo2 = new TableNameInfo(internalCtl, null,
"test_tbl2");
+ TableRefInfo tableRefInfo2 = new TableRefInfo(tableNameInfo2, null,
null, partitionNamesInfo, null, null, null, null);
+
+ List<TableRefInfo> tableRefInfos = new ArrayList<>();
+ tableRefInfos.add(tableRefInfo1);
+ tableRefInfos.add(tableRefInfo2);
+
+ Map<String, String> properties = new HashedMap();
+ properties.put("timeout", "86400");
+ properties.put("type", BackupCommand.BackupType.FULL.name());
+ properties.put("content", BackupCommand.BackupContent.ALL.name());
+
+ boolean isExclude = false;
+
+ BackupCommand command = new BackupCommand(labelNameInfo, repoName,
tableRefInfos, properties, isExclude);
+ Assertions.assertDoesNotThrow(() -> command.validate(connectContext));
+
+ List<TableRefInfo> tableRefInfos2 = new ArrayList<>();
+ BackupCommand command2 = new BackupCommand(labelNameInfo, repoName,
tableRefInfos2, properties, isExclude);
+ Assertions.assertDoesNotThrow(() -> command2.validate(connectContext));
+
+ TableNameInfo tableNameInfo3 = new TableNameInfo(internalCtl, null,
"test_tbl");
+ TableRefInfo tableRefInfo3 = new TableRefInfo(tableNameInfo3, null,
null, partitionNamesInfo, null, null, null, null);
+ List<TableRefInfo> tableRefInfos3 = new ArrayList<>();
+ tableRefInfos3.add(tableRefInfo1);
+ tableRefInfos.add(tableRefInfo3);
+ BackupCommand command3 = new BackupCommand(labelNameInfo, repoName,
tableRefInfos3, properties, isExclude);
+ Assertions.assertThrows(AnalysisException.class, () ->
command3.validate(connectContext));
+
+ Map<String, String> properties2 = new HashedMap();
+ properties.put("key1", "value1");
+ BackupCommand command4 = new BackupCommand(labelNameInfo, repoName,
tableRefInfos, properties2, isExclude);
+ Assertions.assertThrows(AnalysisException.class, () ->
command4.validate(connectContext));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
