This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 4c303f2e1f [feature](create-table) support setting replication num for
creating table opertaion globally (#21848) (#22124)
4c303f2e1f is described below
commit 4c303f2e1f9f54ade8c6e7a690e48bf9540b5670
Author: Mingyu Chen <[email protected]>
AuthorDate: Sun Jul 23 22:32:07 2023 +0800
[feature](create-table) support setting replication num for creating table
opertaion globally (#21848) (#22124)
---
.../main/java/org/apache/doris/common/Config.java | 12 +++++++++
.../org/apache/doris/analysis/CreateTableStmt.java | 31 ++++++++++++++++++++++
.../org/apache/doris/catalog/CreateTableTest.java | 30 +++++++++++++++++++++
3 files changed, 73 insertions(+)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index dbf73be1ba..e49f6c2dd0 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2027,4 +2027,16 @@ public class Config extends ConfigBase {
"Hive行数估算分区采样数",
"Sample size for hive row count estimation."})
public static int hive_stats_partition_sample_size = 3000;
+
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "用于强制设定内表的副本数，如果改参数大于零，则用户在建表时指定的副本数将被忽略，而使用本参数设置的值。"
+ +
"同时，建表语句中指定的副本标签等参数会被忽略。该参数不影响包括创建分区、修改表属性的操作。该参数建议仅用于测试环境",
+ "Used to force the number of replicas of the internal table. If
the config is greater than zero, "
+ + "the number of replicas specified by the user when
creating the table will be ignored, "
+ + "and the value set by this parameter will be used. At
the same time, the replica tags "
+ + "and other parameters specified in the create table
statement will be ignored. "
+ + "This config does not effect the operations including
creating partitions "
+ + "and modifying table properties. "
+ + "This config is recommended to be used only in the test
environment"})
+ public static int force_olap_table_replication_num = 0;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index c332d6e34a..64409d3b4a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -25,6 +25,7 @@ import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.Index;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
+import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -529,6 +530,8 @@ public class CreateTableStmt extends DdlStmt {
}
if (engineName.equals("olap")) {
+ // before analyzing partition, handle the replication allocation
info
+ properties = rewriteReplicaAllocationProperties(properties);
// analyze partition
if (partitionDesc != null) {
if (partitionDesc instanceof ListPartitionDesc ||
partitionDesc instanceof RangePartitionDesc
@@ -619,6 +622,34 @@ public class CreateTableStmt extends DdlStmt {
}
}
+ private Map<String, String> rewriteReplicaAllocationProperties(Map<String,
String> properties) {
+ if (Config.force_olap_table_replication_num <= 0) {
+ return properties;
+ }
+ // if force_olap_table_replication_num is set, use this value to
rewrite the replication_num or
+ // replication_allocation properties
+ Map<String, String> newProperties = properties;
+ if (newProperties == null) {
+ newProperties = Maps.newHashMap();
+ }
+ boolean rewrite = false;
+ if
(newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM)) {
+ newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
+ String.valueOf(Config.force_olap_table_replication_num));
+ rewrite = true;
+ }
+ if
(newProperties.containsKey(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION))
{
+
newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_ALLOCATION,
+ new ReplicaAllocation((short)
Config.force_olap_table_replication_num).toCreateStmt());
+ rewrite = true;
+ }
+ if (!rewrite) {
+ newProperties.put(PropertyAnalyzer.PROPERTIES_REPLICATION_NUM,
+ String.valueOf(Config.force_olap_table_replication_num));
+ }
+ return newProperties;
+ }
+
private void analyzeEngineName() throws AnalysisException {
if (Strings.isNullOrEmpty(engineName)) {
engineName = "olap";
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
index f8878ad193..7d45752426 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.common.AnalysisException;
@@ -25,6 +26,7 @@ import org.apache.doris.common.ConfigBase;
import org.apache.doris.common.ConfigException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ExceptionChecker;
+import org.apache.doris.common.UserException;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.utframe.UtFrameUtils;
@@ -67,6 +69,11 @@ public class CreateTableTest {
Env.getCurrentEnv().createTable(createTableStmt);
}
+ private static void alterTable(String sql) throws Exception {
+ AlterTableStmt alterTableStmt = (AlterTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Env.getCurrentEnv().alterTable(alterTableStmt);
+ }
+
@Test
public void testDuplicateCreateTable() throws Exception {
// test
@@ -725,4 +732,27 @@ public class CreateTableTest {
+ "distributed by hash(k1) buckets 1
properties('replication_num' = '1','in_memory'='true');");
});
}
+
+ @Test
+ public void testCreateTableWithForceReplica() throws DdlException {
+ Config.force_olap_table_replication_num = 1;
+ // no need to specify replication_num, the table can still be created.
+ ExceptionChecker.expectThrowsNoException(() -> {
+ createTable("create table test.test_replica\n" + "(k1 int, k2 int)
partition by range(k1)\n" + "(\n"
+ + "partition p1 values less than(\"10\"),\n" + "partition
p2 values less than(\"20\")\n" + ")\n"
+ + "distributed by hash(k2) buckets 1;");
+ });
+
+ // can still set replication_num manually.
+ ExceptionChecker.expectThrowsWithMsg(UserException.class, "Failed to
find enough host with tag",
+ () -> {
+ alterTable("alter table test.test_replica modify partition
p1 set ('replication_num' = '3')");
+ });
+
+ Database db =
Env.getCurrentInternalCatalog().getDbOrDdlException("default_cluster:test");
+ OlapTable tb = (OlapTable) db.getTableOrDdlException("test_replica");
+ Partition p1 = tb.getPartition("p1");
+ Assert.assertEquals(1,
tb.getPartitionInfo().getReplicaAllocation(p1.getId()).getTotalReplicaNum());
+ Assert.assertEquals(1,
tb.getTableProperty().getReplicaAllocation().getTotalReplicaNum());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
