This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit e5359ba03ce03cd4e122e4304833364397d7e862 Author: zhengyu <[email protected]> AuthorDate: Mon Jul 31 19:26:21 2023 +0800 [fix](show-stmt) fix show create table missing storage_medium info (#21757) --- .../org/apache/doris/catalog/DataProperty.java | 16 ++++++ .../main/java/org/apache/doris/catalog/Env.java | 4 ++ .../java/org/apache/doris/catalog/OlapTable.java | 22 ++++++++ .../apache/doris/datasource/InternalCatalog.java | 5 +- .../analysis/CreateTableAsSelectStmtTest.java | 60 ++++++++++++++-------- .../org/apache/doris/catalog/CreateTableTest.java | 23 +++++++++ 6 files changed, 109 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java index 7028e5e449..23b4508c94 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataProperty.java @@ -55,6 +55,22 @@ public class DataProperty implements Writable, GsonPostProcessable { // for persist } + public String getStorageMediumString() { + int val = storageMedium.getValue(); + switch (val) { + case 0: + return "hdd"; + case 1: + return "ssd"; + case 2: + return "s3"; + case 3: + return "remote_cache"; + default: + return "unknown"; + } + } + public DataProperty(TStorageMedium medium) { this.storageMedium = medium; this.cooldownTimeMs = MAX_COOLDOWN_TIME_MS; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index f1bb519290..dc5598b710 100755 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3155,6 +3155,10 @@ public class Env { sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_ENABLE_SINGLE_REPLICA_COMPACTION).append("\" = \""); sb.append(olapTable.enableSingleReplicaCompaction()).append("\""); + // storage medium + sb.append(",\n\"").append(PropertyAnalyzer.PROPERTIES_STORAGE_MEDIUM).append("\" = \""); + sb.append(olapTable.getStorageMedium()).append("\""); + // enable duplicate without keys by default if (olapTable.isDuplicateWithoutKey()) { sb.append(",\n\"") diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index ff5e3a8570..977e7f9e45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -151,6 +151,8 @@ public class OlapTable extends Table { private TableProperty tableProperty; + private String storageMedium; + public OlapTable() { // for persist super(TableType.OLAP); @@ -1299,6 +1301,13 @@ public class OlapTable extends Table { } tempPartitions.write(out); + + if (storageMedium == null || storageMedium.length() == 0) { + out.writeBoolean(false); + } else { + out.writeBoolean(true); + Text.writeString(out, storageMedium); + } } @Override @@ -1395,6 +1404,10 @@ public class OlapTable extends Table { } tempPartitions.unsetPartitionInfo(); + if (in.readBoolean()) { + storageMedium = Text.readString(in); + } + // In the present, the fullSchema could be rebuilt by schema change while the properties is changed by MV. // After that, some properties of fullSchema and nameToColumn may be not same as properties of base columns. // So, here we need to rebuild the fullSchema to ensure the correctness of the properties. @@ -1816,6 +1829,15 @@ public class OlapTable extends Table { return false; } + + public void setStorageMedium(String medium) { + storageMedium = medium; + } + + public String getStorageMedium() { + return storageMedium; + } + public void setStoreRowColumn(boolean storeRowColumn) { TableProperty tableProperty = getOrCreatTableProperty(); tableProperty.modifyTableProperties(PropertyAnalyzer.PROPERTIES_STORE_ROW_COLUMN, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java index 6c1079688d..28bac7a540 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java @@ -2148,6 +2148,7 @@ public class InternalCatalog implements CatalogIf<Database> { try { dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM)); + olapTable.setStorageMedium(dataProperty.getStorageMediumString()); } catch (AnalysisException e) { throw new DdlException(e.getMessage()); } @@ -2313,10 +2314,12 @@ public class InternalCatalog implements CatalogIf<Database> { } else if (partitionInfo.getType() == PartitionType.RANGE || partitionInfo.getType() == PartitionType.LIST) { try { + DataProperty dataProperty = null; // just for remove entries in stmt.getProperties(), // and then check if there still has unknown properties - PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), + dataProperty = PropertyAnalyzer.analyzeDataProperty(stmt.getProperties(), new DataProperty(DataProperty.DEFAULT_STORAGE_MEDIUM)); + olapTable.setStorageMedium(dataProperty.getStorageMediumString()); if (partitionInfo.getType() == PartitionType.RANGE) { DynamicPartitionUtil.checkAndSetDynamicPartitionProperty(olapTable, properties, db); } else if (partitionInfo.getType() == PartitionType.LIST) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java index 2a5ff740d5..348c8571f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/CreateTableAsSelectStmtTest.java @@ -96,7 +96,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showCreateTableByName("select_decimal_table").getResultRows().get(0).get(1)); String selectFromDecimal1 = @@ -117,7 +118,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } else { @@ -134,7 +136,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showCreateTableByName("select_decimal_table_1").getResultRows().get(0).get(1)); } @@ -169,7 +172,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -193,7 +197,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); @@ -219,7 +224,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -242,7 +248,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); String selectAlias2 = "create table `test`.`select_alias_2` PROPERTIES(\"replication_num\" = \"1\") " + "as select userId as alias_name, username from `test`.`varchar_table`"; @@ -261,7 +268,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet2.getResultRows().get(0).get(1)); } @@ -287,7 +295,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromJoin1 = "create table `test`.`select_join1` PROPERTIES(\"replication_num\" = \"1\") " @@ -310,7 +319,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -337,7 +347,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -361,7 +372,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -384,7 +396,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); String selectFromCteAndUnion = "create table `test`.`select_cte_union` PROPERTIES(\"replication_num\" = \"1\")" @@ -403,7 +416,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet1.getResultRows().get(0).get(1)); } @@ -429,7 +443,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -454,7 +469,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -478,7 +494,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -503,7 +520,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", showResultSet.getResultRows().get(0).get(1)); } @@ -552,7 +570,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", createTableStmts.get(0)); } else { @@ -569,7 +588,8 @@ public class CreateTableAsSelectStmtTest extends TestWithFeService { + "\"storage_format\" = \"V2\",\n" + "\"light_schema_change\" = \"true\",\n" + "\"disable_auto_compaction\" = \"false\",\n" - + "\"enable_single_replica_compaction\" = \"false\"\n" + + "\"enable_single_replica_compaction\" = \"false\",\n" + + "\"storage_medium\" = \"hdd\"\n" + ");", createTableStmts.get(0)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java index 7d45752426..4e52ac8e51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateTableTest.java @@ -146,6 +146,16 @@ public class CreateTableTest { .expectThrowsNoException(() -> createTable("create table test.tb7(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');")); + ConfigBase.setMutableConfig("disable_storage_medium_check", "true"); + ExceptionChecker + .expectThrowsNoException(() -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n" + + "PARTITION BY RANGE(`key1`) (\n" + + " PARTITION `p1` VALUES LESS THAN (\"10\"),\n" + + " PARTITION `p2` VALUES LESS THAN (\"20\"),\n" + + " PARTITION `p3` VALUES LESS THAN (\"30\"))\n" + + "distributed by hash(key1)\n" + + "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');")); + ExceptionChecker .expectThrowsNoException(() -> createTable("create table test.compression1(key1 int, key2 varchar(10)) \n" + "distributed by hash(key1) buckets 1 \n" @@ -301,6 +311,19 @@ public class CreateTableTest { () -> createTable("create table test.tb7(key1 int, key2 varchar(10)) distributed by hash(key1) \n" + "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');")); + ConfigBase.setMutableConfig("disable_storage_medium_check", "false"); + ExceptionChecker + .expectThrowsWithMsg(DdlException.class, "Failed to find enough backend, please check the replication num,replication tag and storage medium.\n" + + "Create failed replications:\n" + + "replication tag: {\"location\" : \"default\"}, replication num: 1, storage medium: SSD", + () -> createTable("create table test.tb7_1(key1 int, key2 varchar(10))\n" + + "PARTITION BY RANGE(`key1`) (\n" + + " PARTITION `p1` VALUES LESS THAN (\"10\"),\n" + + " PARTITION `p2` VALUES LESS THAN (\"20\"),\n" + + " PARTITION `p3` VALUES LESS THAN (\"30\"))\n" + + "distributed by hash(key1)\n" + + "buckets 1 properties('replication_num' = '1', 'storage_medium' = 'ssd');")); + ExceptionChecker .expectThrowsWithMsg(DdlException.class, "sequence column only support UNIQUE_KEYS", () -> createTable("create table test.atbl8\n" + "(k1 varchar(40), k2 int, v1 int sum)\n" --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
