This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 5442d9a2a85fd0ea79fcc44600c355f11308e850 Author: xy720 <[email protected]> AuthorDate: Mon Aug 28 14:31:54 2023 +0800 [Feature](fe) Add admin set partition version statement (#23086) This commit add a statement to modify partition visible version. --- .../ADMIN-SET-PARTITION-VERSION.md | 67 ++++++++++++++++ docs/sidebars.json | 1 + .../ADMIN-SET-PARTITION-VERSION.md | 67 ++++++++++++++++ fe/fe-core/src/main/cup/sql_parser.cup | 4 + .../analysis/AdminSetPartitionVersionStmt.java | 93 ++++++++++++++++++++++ .../java/org/apache/doris/catalog/DatabaseIf.java | 9 +++ .../main/java/org/apache/doris/catalog/Env.java | 51 ++++++++++++ .../apache/doris/common/util/PropertyAnalyzer.java | 26 ++++++ .../org/apache/doris/journal/JournalEntity.java | 6 ++ .../java/org/apache/doris/persist/EditLog.java | 9 +++ .../org/apache/doris/persist/OperationType.java | 1 + .../persist/SetPartitionVersionOperationLog.java | 63 +++++++++++++++ .../main/java/org/apache/doris/qe/DdlExecutor.java | 3 + .../org/apache/doris/catalog/AdminStmtTest.java | 63 +++++++++++++++ .../version_p0/test_set_partition_version.groovy | 71 +++++++++++++++++ 15 files changed, 534 insertions(+) diff --git a/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md b/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md new file mode 100644 index 0000000000..14e63036bd --- /dev/null +++ b/docs/en/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md @@ -0,0 +1,67 @@ +--- +{ + "title": "ADMIN-SET-PARTITION-VERSION", + "language": "en" +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## ADMIN-SET-PARTITION-VERSION + +### Name + +ADMIN SET PARTITION VERSION + +### Description + +This statement is used to set the version of the specified partition. + +In certain cases, the version of the partition in the metadata may not be consistent with the version of the actual replica. This command can manually set the version of the partition in the metadata. + +grammar: + +```sql +ADMIN SET TABLE table_name PARTITION VERSION + PROPERTIES ("key" = "value", ...); +``` + +The following properties are currently supported: + +1. "partition_id": Required. Specify a Partition Id. +2. "visible_version": Required. Specify Version. + +> Note: +> +> It is necessary to first confirm the version of the actual replica on the Be before set the version of the partition. This command is generally only used for emergency troubleshooting, please proceed with caution. + +### Example + +1. Set the version of partition 1769152 to 100. + +```sql +ADMIN SET TABLE tbl1 PARTITION VERSION PROPERTIES("partition_id" = "1769152", "visible_version" = "100"); +``` + +### Keywords + + ADMIN, SET, PARTITION, VERSION + +### Best Practice diff --git a/docs/sidebars.json b/docs/sidebars.json index 04bc5f1c1b..d3316a330c 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -764,6 +764,7 @@ "sql-manual/sql-reference/Database-Administration-Statements/INSTALL-PLUGIN", "sql-manual/sql-reference/Database-Administration-Statements/UNINSTALL-PLUGIN", "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-REPLICA-STATUS", + "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION", "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-TABLE-STATUS", "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-DISTRIBUTION", "sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SHOW-REPLICA-STATUS", diff --git a/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md b/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md new file mode 100644 index 0000000000..3ce8d43cd6 --- /dev/null +++ b/docs/zh-CN/docs/sql-manual/sql-reference/Database-Administration-Statements/ADMIN-SET-PARTITION-VERSION.md @@ -0,0 +1,67 @@ +--- +{ + "title": "ADMIN-SET-PARTITION-VERSION", + "language": "zh-CN" +} +--- + +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +## ADMIN-SET-PARTITION-VERSION + +### Name + +ADMIN SET PARTITION VERSION + +### Description + +该语句用于手动改变指定分区的可见版本。 + +在某些特殊情况下，元数据中分区的版本有可能和实际副本的版本不一致，该命令可手动改变元数据中分区的版本。 + +语法： + +```sql +ADMIN SET TABLE table_name PARTITION VERSION + PROPERTIES ("key" = "value", ...); +``` + +目前支持如下属性： + +1. "partition_id"：必需。指定一个 Partition Id. +2. "visible_version"：必需。指定 Version. + +> 注意： +> +> 设置分区的版本需要先确认Be机器上实际副本的版本，此命令一般只用于紧急故障修复，请谨慎操作。 + +### Example + +1. 设置 partition 1769152 在 FE 元数据上的版本为 100。 + +```sql +ADMIN SET TABLE tbl1 PARTITION VERSION PROPERTIES("partition_id" = "1769152", "visible_version" = "100"); +``` + +### Keywords + + ADMIN, SET, PARTITION, VERSION + +### Best Practice diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup index daf303f5a2..fd59652fa8 100644 --- a/fe/fe-core/src/main/cup/sql_parser.cup +++ b/fe/fe-core/src/main/cup/sql_parser.cup @@ -7123,6 +7123,10 @@ admin_stmt ::= {: RESULT = new AdminCleanTrashStmt(null); :} + | KW_ADMIN KW_SET KW_TABLE table_name:name KW_PARTITION KW_VERSION opt_properties:properties + {: + RESULT = new AdminSetPartitionVersionStmt(name, properties); + :} | KW_ADMIN KW_DIAGNOSE KW_TABLET INTEGER_LITERAL:tabletId {: RESULT = new AdminDiagnoseTabletStmt(tabletId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetPartitionVersionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetPartitionVersionStmt.java new file mode 100644 index 0000000000..a1edeb7a2c --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/AdminSetPartitionVersionStmt.java @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.analysis; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.ErrorCode; +import org.apache.doris.common.ErrorReport; +import org.apache.doris.common.UserException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.common.util.Util; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; + +import java.util.Map; + +// Modify version of specified partition. Only used in emergency. +/* + * admin set table db.tbl partition version properties ("key" = "val", ..); + * "partition_id" = "20010", + * "visible_version" = "101" + */ +public class AdminSetPartitionVersionStmt extends DdlStmt { + private long partitionId = -1; + private long visibleVersion = -1; + private final TableName tableName; + private final Map<String, String> properties; + + public AdminSetPartitionVersionStmt(TableName tableName, Map<String, String> properties) { + this.tableName = tableName; + this.properties = properties; + } + + public String getDatabase() { + return tableName.getDb(); + } + + public String getTable() { + return tableName.getTbl(); + } + + public Long getPartitionId() { + return partitionId; + } + + public Long getVisibleVersion() { + return visibleVersion; + } + + @Override + public void analyze(Analyzer analyzer) throws AnalysisException, UserException { + super.analyze(analyzer); + + // check auth + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ConnectContext.get(), PrivPredicate.ADMIN)) { + ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN"); + } + + tableName.analyze(analyzer); + Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName()); + + checkProperties(); + } + + private void checkProperties() throws AnalysisException { + partitionId = PropertyAnalyzer.analyzePartitionId(properties); + if (partitionId == -1) { + throw new AnalysisException("Should specify 'partition_id' property."); + } + visibleVersion = PropertyAnalyzer.analyzeVisibleVersion(properties); + if (visibleVersion == -1) { + throw new AnalysisException("Should specify 'visible_version' property."); + } + if (properties != null && !properties.isEmpty()) { + throw new AnalysisException("Unknown properties: " + properties.keySet()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java index f9c5d94601..8c28337641 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java @@ -206,6 +206,15 @@ public interface DatabaseIf<T extends TableIf> { return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); } + default T getTableOrDdlException(long tableId, TableIf.TableType tableType) throws DdlException { + T table = getTableOrDdlException(tableId); + if (table.getType() != tableType) { + throw new DdlException( + "table type is not " + tableType + ", tableId=" + tableId + ", type=" + table.getType()); + } + return table; + } + default T getTableOrAnalysisException(String tableName) throws AnalysisException { return getTableOrException(tableName, t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 7a55cb7ea2..f679f679c8 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -30,6 +30,7 @@ import org.apache.doris.analysis.AdminCheckTabletsStmt.CheckType; import org.apache.doris.analysis.AdminCleanTrashStmt; import org.apache.doris.analysis.AdminCompactTableStmt; import org.apache.doris.analysis.AdminSetConfigStmt; +import org.apache.doris.analysis.AdminSetPartitionVersionStmt; import org.apache.doris.analysis.AdminSetReplicaStatusStmt; import org.apache.doris.analysis.AdminSetTableStatusStmt; import org.apache.doris.analysis.AlterDatabasePropertyStmt; @@ -191,6 +192,7 @@ import org.apache.doris.persist.RecoverInfo; import org.apache.doris.persist.RefreshExternalTableInfo; import org.apache.doris.persist.ReplacePartitionOperationLog; import org.apache.doris.persist.ReplicaPersistInfo; +import org.apache.doris.persist.SetPartitionVersionOperationLog; import org.apache.doris.persist.SetReplicaStatusOperationLog; import org.apache.doris.persist.SetTableStatusOperationLog; import org.apache.doris.persist.Storage; @@ -5349,6 +5351,55 @@ public class Env { } } + public void setPartitionVersion(AdminSetPartitionVersionStmt stmt) throws DdlException { + String database = stmt.getDatabase(); + String table = stmt.getTable(); + long partitionId = stmt.getPartitionId(); + long visibleVersion = stmt.getVisibleVersion(); + int setSuccess = setPartitionVersionInternal(database, table, partitionId, visibleVersion, false); + if (setSuccess == -1) { + throw new DdlException("Failed to set partition visible version to " + visibleVersion + ". " + "Partition " + + partitionId + " not exists. Database " + database + ", Table " + table + "."); + } + } + + public void replaySetPartitionVersion(SetPartitionVersionOperationLog log) throws DdlException { + int setSuccess = setPartitionVersionInternal(log.getDatabase(), log.getTable(), + log.getPartitionId(), log.getVisibleVersion(), true); + if (setSuccess == -1) { + LOG.warn("Failed to set partition visible version to {}. " + + "Database {}, Table {}, Partition {} not exists.", log.getDatabase(), log.getTable(), + log.getVisibleVersion(), log.getPartitionId()); + } + } + + public int setPartitionVersionInternal(String database, String table, long partitionId, + long visibleVersion, boolean isReplay) throws DdlException { + int result = -1; + Database db = getInternalCatalog().getDbOrDdlException(database); + OlapTable olapTable = db.getOlapTableOrDdlException(table); + olapTable.writeLockOrDdlException(); + try { + Partition partition = olapTable.getPartition(partitionId); + if (partition != null) { + Long oldVersion = partition.getVisibleVersion(); + partition.updateVisibleVersion(visibleVersion); + partition.setNextVersion(visibleVersion + 1); + result = 0; + if (!isReplay) { + SetPartitionVersionOperationLog log = new SetPartitionVersionOperationLog( + database, table, partitionId, visibleVersion); + getEditLog().logSetPartitionVersion(log); + } + LOG.info("set partition {} visible version from {} to {}. Database {}, Table {}, is replay:" + + " {}.", partitionId, oldVersion, visibleVersion, database, table, isReplay); + } + } finally { + olapTable.writeUnlock(); + } + return result; + } + public static boolean isStoredTableNamesLowerCase() { return GlobalVariable.lowerCaseTableNames == 1; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index f916d13336..431f357b78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -70,6 +70,8 @@ public class PropertyAnalyzer { public static final String PROPERTIES_VERSION_INFO = "version_info"; // for restore public static final String PROPERTIES_SCHEMA_VERSION = "schema_version"; + public static final String PROPERTIES_PARTITION_ID = "partition_id"; + public static final String PROPERTIES_VISIBLE_VERSION = "visible_version"; public static final String PROPERTIES_BF_COLUMNS = "bloom_filter_columns"; public static final String PROPERTIES_BF_FPP = "bloom_filter_fpp"; @@ -428,6 +430,30 @@ public class PropertyAnalyzer { return schemaVersion; } + private static Long getPropertyLong(Map<String, String> properties, String propertyId) throws AnalysisException { + long id = -1; + if (properties != null && properties.containsKey(propertyId)) { + String propertyIdStr = properties.get(propertyId); + try { + id = Long.parseLong(propertyIdStr); + } catch (Exception e) { + throw new AnalysisException("Invalid property long id: " + propertyIdStr); + } + + properties.remove(propertyId); + } + + return id; + } + + public static Long analyzePartitionId(Map<String, String> properties) throws AnalysisException { + return getPropertyLong(properties, PROPERTIES_PARTITION_ID); + } + + public static Long analyzeVisibleVersion(Map<String, String> properties) throws AnalysisException { + return getPropertyLong(properties, PROPERTIES_VISIBLE_VERSION); + } + public static Set<String> analyzeBloomFilterColumns(Map<String, String> properties, List<Column> columns, KeysType keysType) throws AnalysisException { Set<String> bfColumns = null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java index c87255e3dd..8feb3022fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/journal/JournalEntity.java @@ -105,6 +105,7 @@ import org.apache.doris.persist.ReplacePartitionOperationLog; import org.apache.doris.persist.ReplaceTableOperationLog; import org.apache.doris.persist.ReplicaPersistInfo; import org.apache.doris.persist.RoutineLoadOperation; +import org.apache.doris.persist.SetPartitionVersionOperationLog; import org.apache.doris.persist.SetReplicaStatusOperationLog; import org.apache.doris.persist.SetTableStatusOperationLog; import org.apache.doris.persist.TableAddOrDropColumnsInfo; @@ -600,6 +601,11 @@ public class JournalEntity implements Writable { isRead = true; break; } + case OperationType.OP_SET_PARTITION_VERSION: { + data = SetPartitionVersionOperationLog.read(in); + isRead = true; + break; + } case OperationType.OP_DYNAMIC_PARTITION: case OperationType.OP_MODIFY_IN_MEMORY: case OperationType.OP_MODIFY_REPLICATION_NUM: diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 79c36fe6a2..b1bac82f21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -805,6 +805,11 @@ public class EditLog { env.getAlterInstance().replayModifyComment(operation); break; } + case OperationType.OP_SET_PARTITION_VERSION: { + SetPartitionVersionOperationLog log = (SetPartitionVersionOperationLog) journal.getData(); + env.replaySetPartitionVersion(log); + break; + } case OperationType.OP_ALTER_ROUTINE_LOAD_JOB: { AlterRoutineLoadJobOperationLog log = (AlterRoutineLoadJobOperationLog) journal.getData(); env.getRoutineLoadManager().replayAlterRoutineLoadJob(log); @@ -1681,6 +1686,10 @@ public class EditLog { logEdit(OperationType.OP_ALTER_ROUTINE_LOAD_JOB, log); } + public void logSetPartitionVersion(SetPartitionVersionOperationLog log) { + logEdit(OperationType.OP_SET_PARTITION_VERSION, log); + } + public void logGlobalVariableV2(GlobalVarPersistInfo info) { logEdit(OperationType.OP_GLOBAL_VARIABLE_V2, info); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java index 53616af30e..85f74fea42 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/OperationType.java @@ -105,6 +105,7 @@ public class OperationType { public static final short OP_BACKEND_TABLETS_INFO = 46; public static final short OP_SET_REPLICA_STATUS = 47; public static final short OP_BACKEND_REPLICAS_INFO = 48; + public static final short OP_SET_PARTITION_VERSION = 49; public static final short OP_ADD_BACKEND = 50; public static final short OP_DROP_BACKEND = 51; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/SetPartitionVersionOperationLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/SetPartitionVersionOperationLog.java new file mode 100644 index 0000000000..ae5b3a4c38 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/SetPartitionVersionOperationLog.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.persist; + +import org.apache.doris.common.io.Text; +import org.apache.doris.common.io.Writable; +import org.apache.doris.persist.gson.GsonUtils; + +import com.google.gson.annotations.SerializedName; +import lombok.Getter; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +@Getter +public class SetPartitionVersionOperationLog implements Writable { + + @SerializedName(value = "database") + private String database; + + @SerializedName(value = "table") + private String table; + + @SerializedName(value = "partitionId") + private long partitionId; + + @SerializedName(value = "visibleVersion") + private long visibleVersion; + + public SetPartitionVersionOperationLog(String database, String table, long partitionId, long visibleVersion) { + this.database = database; + this.table = table; + this.partitionId = partitionId; + this.visibleVersion = visibleVersion; + } + + public static SetPartitionVersionOperationLog read(DataInput in) throws IOException { + String json = Text.readString(in); + return GsonUtils.GSON.fromJson(json, SetPartitionVersionOperationLog.class); + } + + @Override + public void write(DataOutput out) throws IOException { + String json = GsonUtils.GSON.toJson(this); + Text.writeString(out, json); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java index 6800898530..b291cdbed5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/DdlExecutor.java @@ -25,6 +25,7 @@ import org.apache.doris.analysis.AdminCompactTableStmt; import org.apache.doris.analysis.AdminRebalanceDiskStmt; import org.apache.doris.analysis.AdminRepairTableStmt; import org.apache.doris.analysis.AdminSetConfigStmt; +import org.apache.doris.analysis.AdminSetPartitionVersionStmt; import org.apache.doris.analysis.AdminSetReplicaStatusStmt; import org.apache.doris.analysis.AdminSetTableStatusStmt; import org.apache.doris.analysis.AlterCatalogNameStmt; @@ -257,6 +258,8 @@ public class DdlExecutor { env.checkTablets((AdminCheckTabletsStmt) ddlStmt); } else if (ddlStmt instanceof AdminSetReplicaStatusStmt) { env.setReplicaStatus((AdminSetReplicaStatusStmt) ddlStmt); + } else if (ddlStmt instanceof AdminSetPartitionVersionStmt) { + env.setPartitionVersion((AdminSetPartitionVersionStmt) ddlStmt); } else if (ddlStmt instanceof CreateResourceStmt) { env.getResourceMgr().createResource((CreateResourceStmt) ddlStmt); } else if (ddlStmt instanceof DropResourceStmt) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/AdminStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/AdminStmtTest.java index c43e08f08f..5d99435060 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/AdminStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/AdminStmtTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.MaterializedIndex.IndexExtState; import org.apache.doris.catalog.Replica.ReplicaStatus; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Pair; +import org.apache.doris.persist.SetPartitionVersionOperationLog; import org.apache.doris.persist.SetReplicaStatusOperationLog; import org.apache.doris.utframe.TestWithFeService; @@ -50,6 +51,15 @@ public class AdminStmtTest extends TestWithFeService { + "PROPERTIES (\n" + " \"replication_num\" = \"1\"\n" + ");"); + createTable("CREATE TABLE test.tbl2 (\n" + + " `id` int(11) NULL COMMENT \"\",\n" + + " `name` varchar(20) NULL\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`id`, `name`)\n" + + "DISTRIBUTED BY HASH(`id`) BUCKETS 3\n" + + "PROPERTIES (\n" + + " \"replication_num\" = \"1\"\n" + + ");"); } @Test @@ -120,4 +130,57 @@ public class AdminStmtTest extends TestWithFeService { } } + @Test + public void testAdminSetPartitionVersion() throws Exception { + Database db = Env.getCurrentInternalCatalog().getDbNullable("default_cluster:test"); + Assertions.assertNotNull(db); + OlapTable tbl = (OlapTable) db.getTableNullable("tbl2"); + Assertions.assertNotNull(tbl); + Partition partition = tbl.getPartitions().iterator().next(); + long partitionId = partition.getId(); + long oldVersion = partition.getVisibleVersion(); + // origin version is 1 + Assertions.assertEquals(1, oldVersion); + // set partition version to 100 + long newVersion = 100; + String adminStmt = "admin set table test.tbl2 partition version properties ('partition_id' = '" + + partitionId + "', " + "'visible_version' = '" + newVersion + "');"; + Assertions.assertNotNull(getSqlStmtExecutor(adminStmt)); + Assertions.assertEquals(newVersion, partition.getVisibleVersion()); + adminStmt = "admin set table test.tbl2 partition version properties ('partition_id' = '" + + partitionId + "', " + "'visible_version' = '" + oldVersion + "');"; + Assertions.assertNotNull(getSqlStmtExecutor(adminStmt)); + Assertions.assertEquals(oldVersion, partition.getVisibleVersion()); + } + + @Test + public void testSetPartitionVersionOperationLog() throws IOException, AnalysisException { + String fileName = "./SetPartitionVersionOperationLog"; + Path path = Paths.get(fileName); + try { + // 1. Write objects to file + Files.createFile(path); + DataOutputStream out = new DataOutputStream(Files.newOutputStream(path)); + + SetPartitionVersionOperationLog log = new SetPartitionVersionOperationLog( + "test", "tbl2", 10002, 100); + log.write(out); + out.flush(); + out.close(); + + // 2. Read objects from file + DataInputStream in = new DataInputStream(Files.newInputStream(path)); + + SetPartitionVersionOperationLog readLog = SetPartitionVersionOperationLog.read(in); + Assertions.assertEquals(log.getDatabase(), readLog.getDatabase()); + Assertions.assertEquals(log.getTable(), readLog.getTable()); + Assertions.assertEquals(log.getPartitionId(), readLog.getPartitionId()); + Assertions.assertEquals(log.getVisibleVersion(), readLog.getVisibleVersion()); + + in.close(); + } finally { + Files.deleteIfExists(path); + } + } + } diff --git a/regression-test/suites/version_p0/test_set_partition_version.groovy b/regression-test/suites/version_p0/test_set_partition_version.groovy new file mode 100644 index 0000000000..b461d01d80 --- /dev/null +++ b/regression-test/suites/version_p0/test_set_partition_version.groovy @@ -0,0 +1,71 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_set_partition_version") { + def tableName1 = "test_set_partition_version" + sql """ DROP TABLE IF EXISTS ${tableName1} """ + sql """ + CREATE TABLE ${tableName1} ( + `id` int NOT NULL, + `version` int NOT NULL COMMENT '插入次数' + ) ENGINE=OLAP + DUPLICATE KEY(`id`) + COMMENT 'OLAP' + DISTRIBUTED BY HASH(`id`) BUCKETS 10 + PROPERTIES + ( + "replication_allocation" = "tag.location.default: 1", + "disable_auto_compaction" = "true" + ); + """ + + def res = sql """ show partitions from ${tableName1}; """ + def partitionId = res[0][0].toString() + + // load 1 time, partition visible version should be 2 + sql """ insert into ${tableName1} values (1, 2); """ + res = sql """ show partitions from ${tableName1}; """ + assertEquals(res[0][2].toString(), "2") + + // load 2 time, partition visible version should be 3 + sql """ insert into ${tableName1} values (2, 3); """ + res = sql """ show partitions from ${tableName1}; """ + assertEquals(res[0][2].toString(), "3") + + // set partition visible version to 2 + sql """ ADMIN SET TABLE ${tableName1} PARTITION VERSION PROPERTIES ("partition_id" = "${partitionId}", "visible_version" = "2"); """ + res = sql """ show partitions from ${tableName1}; """ + assertEquals(res[0][2].toString(), "2") + + // check if table can query, and return row size should be 1 + res = sql """ select * from ${tableName1}; """ + assertEquals(res.size(), 1) + + // set partition visible version to 3 + sql """ ADMIN SET TABLE ${tableName1} PARTITION VERSION PROPERTIES ("partition_id" = "${partitionId}", "visible_version" = "3"); """ + res = sql """ show partitions from ${tableName1}; """ + assertEquals(res[0][2].toString(), "3") + + // check if table can query, and return row size should be 2 + res = sql """ select * from ${tableName1}; """ + assertEquals(res.size(), 2) + + // load 3 time, partition visible version should be 4 + sql """ insert into ${tableName1} values (3, 4); """ + res = sql """ show partitions from ${tableName1}; """ + assertEquals(res[0][2].toString(), "4") +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
