This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch variant
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/variant by this push:
new f801f0f2a8 [improvement](desc table) add session variable to control
extend variant column (#23147)
f801f0f2a8 is described below
commit f801f0f2a8f84796197dc3e9aecc5e14076715b8
Author: Chenyang Sun <[email protected]>
AuthorDate: Fri Aug 18 17:03:24 2023 +0800
[improvement](desc table) add session variable to control extend variant
column (#23147)
session variable to control whether show extened columns of variant
---
docs/en/docs/advanced/variables.md | 4 +
docs/zh-CN/docs/advanced/variables.md | 4 +
.../apache/doris/common/proc/IndexInfoProcDir.java | 4 +-
.../common/util/FetchRemoteTabletSchemaUtil.java | 5 +-
.../java/org/apache/doris/qe/SessionVariable.java | 21 +++
regression-test/data/variant_p0/desc.out | 174 ++++++++++++---------
regression-test/suites/variant_p0/desc.groovy | 20 +++
7 files changed, 154 insertions(+), 78 deletions(-)
diff --git a/docs/en/docs/advanced/variables.md
b/docs/en/docs/advanced/variables.md
index df9983c52b..d0bcde468c 100644
--- a/docs/en/docs/advanced/variables.md
+++ b/docs/en/docs/advanced/variables.md
@@ -677,6 +677,10 @@ Translated with www.DeepL.com/Translator (free version)
Because the maximum length of the char or varchar column in the schema of
the table is inconsistent with the schema in the underlying parquet or orc
file. At this time, if the option is turned on, it will be truncated according
to the maximum length in the schema of the table.
+* `describe_extend_variant_column`
+
+ Controls whether to extend variant column in desc table_name. The default
value is false.
+
***
#### Supplementary instructions on statement execution timeout control
diff --git a/docs/zh-CN/docs/advanced/variables.md
b/docs/zh-CN/docs/advanced/variables.md
index 4f545c35d5..94b1303169 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -664,6 +664,10 @@ try (Connection conn =
DriverManager.getConnection("jdbc:mysql://127.0.0.1:9030/
因为外表会存在表的 schema 中 char 或者 varchar 列的最大长度和底层 parquet 或者 orc 文件中的 schema
不一致的情况。此时开启改选项，会按照表的 schema 中的最大长度进行截断。
+* `describe_extend_variant_column`
+
+ 是否展示 variant 的拆解列。默认为 false。
+
***
#### 关于语句执行超时控制的补充说明
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
index 24882a3e97..de91b59dce 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
@@ -24,6 +24,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.qe.SessionVariable;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -128,7 +129,8 @@ public class IndexInfoProcDir implements ProcDirInterface {
throw new AnalysisException("Index " + idxId + " does not
exist");
}
bfColumns = olapTable.getCopiedBfColumns();
- if (olapTable.hasVariantColumns()) {
+ if (olapTable.hasVariantColumns()
+ &&
SessionVariable.enableDescribeExtendVariantColumn()) {
return new RemoteIndexSchemaProcDir(table, schema,
bfColumns);
}
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java
index 7145ac1acf..d4f3aba15f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/util/FetchRemoteTabletSchemaUtil.java
@@ -177,7 +177,10 @@ public class FetchRemoteTabletSchemaUtil {
boolean isNullable = column.getIsNullable();
String defaultValue = column.getDefaultValue().toString("UTF-8");
if (defaultValue.equals("")) {
- defaultValue = "NULL";
+ defaultValue = null;
+ }
+ if (isKey) {
+ aggType = null;
}
do {
if (type.isArrayType()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 5b69a1a3ce..bb86bc5d76 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -376,6 +376,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ROUND_PRECISE_DECIMALV2_VALUE =
"round_precise_decimalv2_value";
+ public static final String DESCRIBE_EXTEND_VARIANT_COLUMN =
"describe_extend_variant_column";
+
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@@ -703,6 +705,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = USE_RF_DEFAULT)
public boolean useRuntimeFilterDefaultSize = false;
+ @VariableMgr.VarAttr(name = DESCRIBE_EXTEND_VARIANT_COLUMN, needForward =
true)
+ public boolean enableDescribeExtendVariantColumn = false;
+
public int getBeNumber() {
return beNumber;
}
@@ -2489,5 +2494,21 @@ public class SessionVariable implements Serializable,
Writable {
}
return connectContext.getSessionVariable().enableAggState;
}
+
+ public boolean getEnableDescribeExtendVariantColumn() {
+ return enableDescribeExtendVariantColumn;
+ }
+
+ public void setEnableDescribeExtendVariantColumn(boolean
enableDescribeExtendVariantColumn) {
+ this.enableDescribeExtendVariantColumn =
enableDescribeExtendVariantColumn;
+ }
+
+ public static boolean enableDescribeExtendVariantColumn() {
+ ConnectContext connectContext = ConnectContext.get();
+ if (connectContext == null) {
+ return false;
+ }
+ return
connectContext.getSessionVariable().enableDescribeExtendVariantColumn;
+ }
}
diff --git a/regression-test/data/variant_p0/desc.out
b/regression-test/data/variant_p0/desc.out
index 91a6f95609..569e33b7af 100644
--- a/regression-test/data/variant_p0/desc.out
+++ b/regression-test/data/variant_p0/desc.out
@@ -1,101 +1,123 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql_1 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_2 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.ddd.aaa TINYINT Yes false NULL NONE
-v.ddd.mxmxm JSON Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.ddd.aaa TINYINT Yes false \N NONE
+v.ddd.mxmxm JSON Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_3 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.b JSON Yes false NULL NONE
-v.c.c SMALLINT Yes false NULL NONE
-v.c.e DOUBLE Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_5 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
-- !sql_6_1 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.ddd.aaa TINYINT Yes false NULL NONE
-v.ddd.mxmxm JSON Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.ddd.aaa TINYINT Yes false \N NONE
+v.ddd.mxmxm JSON Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_6_2 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_6_3 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.b JSON Yes false NULL NONE
-v.c.c SMALLINT Yes false NULL NONE
-v.c.e DOUBLE Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
-- !sql_6 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.b JSON Yes false NULL NONE
-v.c.c SMALLINT Yes false NULL NONE
-v.c.e DOUBLE Yes false NULL NONE
-v.ddd.aaa TINYINT Yes false NULL NONE
-v.ddd.mxmxm JSON Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+v.ddd.aaa TINYINT Yes false \N NONE
+v.ddd.mxmxm JSON Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_7 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.b JSON Yes false NULL NONE
-v.c.c SMALLINT Yes false NULL NONE
-v.c.e DOUBLE Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_7_1 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.xxxx TEXT Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_7_2 --
-k BIGINT Yes true NULL NONE
-v VARIANT Yes false NULL NONE
-v.a SMALLINT Yes false NULL NONE
-v.b JSON Yes false NULL NONE
-v.c.c SMALLINT Yes false NULL NONE
-v.c.e DOUBLE Yes false NULL NONE
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+
+-- !sql_7_3 --
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+v.xxxx TEXT Yes false \N NONE
-- !sql_8 --
-k BIGINT Yes true NULL NONE
-v1 VARIANT Yes false NULL NONE
-v1.a SMALLINT Yes false NULL NONE
-v1.b JSON Yes false NULL NONE
-v1.c.c SMALLINT Yes false NULL NONE
-v1.c.e DOUBLE Yes false NULL NONE
-v1.oooo.xxxx.xxx TINYINT Yes false NULL NONE
-v2 VARIANT Yes false NULL NONE
-v2.a SMALLINT Yes false NULL NONE
-v2.xxxx TEXT Yes false NULL NONE
-v3 VARIANT Yes false NULL NONE
-v3.a SMALLINT Yes false NULL NONE
-v3.b JSON Yes false NULL NONE
-v3.c.c SMALLINT Yes false NULL NONE
-v3.c.e DOUBLE Yes false NULL NONE
+k BIGINT Yes true \N
+v1 VARIANT Yes false \N NONE
+v1.a SMALLINT Yes false \N NONE
+v1.b JSON Yes false \N NONE
+v1.c.c SMALLINT Yes false \N NONE
+v1.c.e DOUBLE Yes false \N NONE
+v1.oooo.xxxx.xxx TINYINT Yes false \N NONE
+v2 VARIANT Yes false \N NONE
+v2.a SMALLINT Yes false \N NONE
+v2.xxxx TEXT Yes false \N NONE
+v3 VARIANT Yes false \N NONE
+v3.a SMALLINT Yes false \N NONE
+v3.b JSON Yes false \N NONE
+v3.c.c SMALLINT Yes false \N NONE
+v3.c.e DOUBLE Yes false \N NONE
+
+-- !sql_9 --
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+
+-- !sql_9_1 --
+k BIGINT Yes true \N
+v VARIANT Yes false \N NONE
+v.a SMALLINT Yes false \N NONE
+v.b JSON Yes false \N NONE
+v.c.c SMALLINT Yes false \N NONE
+v.c.e DOUBLE Yes false \N NONE
+v.oooo.xxxx.xxx TINYINT Yes false \N NONE
diff --git a/regression-test/suites/variant_p0/desc.groovy
b/regression-test/suites/variant_p0/desc.groovy
index 1058390f6e..c5918c5662 100644
--- a/regression-test/suites/variant_p0/desc.groovy
+++ b/regression-test/suites/variant_p0/desc.groovy
@@ -93,6 +93,7 @@ suite("regression_test_variant_desc", "variant_type_desc"){
def table_name = "sparse_columns"
create_table table_name
set_be_config.call("ratio_of_defaults_as_sparse_column", "0.95")
+ sql """set describe_extend_variant_column = true"""
sql """insert into sparse_columns select 0, '{"a": 11245, "b" : [123,
{"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}}' as json_str
union all select 0, '{"a": 1123}' as json_str union all select 0,
'{"a" : 1234, "xxxx" : "kaana"}' as json_str from numbers("number" = "4096")
limit 4096 ;"""
qt_sql_1 """desc ${table_name}"""
@@ -149,6 +150,7 @@ suite("regression_test_variant_desc", "variant_type_desc"){
qt_sql_7 """desc ${table_name}"""
qt_sql_7_1 """desc ${table_name} partition p2"""
qt_sql_7_2 """desc ${table_name} partition p3"""
+ qt_sql_7_3 """desc ${table_name} partition (p2, p3)"""
sql "truncate table ${table_name}"
// more variant
@@ -167,6 +169,24 @@ suite("regression_test_variant_desc", "variant_type_desc"){
sql """ insert into ${table_name} values (0, '{"a": 1123, "b" : [123,
{"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" :
{"akakaka" : null, "xxxx" : {"xxx" : 123}}}', '{"a": 11245, "xxxx" : "kaana"}',
'{"a": 11245, "b" : [123, {"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" :
7.111}}')"""
qt_sql_8 """desc ${table_name}"""
sql "truncate table ${table_name}"
+
+ // describe_extend_variant_column = false
+ sql """set describe_extend_variant_column = false"""
+ table_name = "no_extend_variant_column"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${table_name} (
+ k bigint,
+ v variant
+ )
+ DUPLICATE KEY(`k`)
+ DISTRIBUTED BY HASH(k) BUCKETS 5
+ properties("replication_num" = "1", "disable_auto_compaction" =
"false");
+ """
+ sql """ insert into ${table_name} values (0, '{"a": 1123, "b" : [123,
{"xx" : 1}], "c" : {"c" : 456, "d" : null, "e" : 7.111}, "zzz" : null, "oooo" :
{"akakaka" : null, "xxxx" : {"xxx" : 123}}}')"""
+ qt_sql_9 """desc ${table_name}"""
+ sql """set describe_extend_variant_column = true"""
+ qt_sql_9_1 """desc ${table_name}"""
+ sql "truncate table ${table_name}"
} finally {
// reset flags
set_be_config.call("ratio_of_defaults_as_sparse_column", "0.95")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
