This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2c25e0a681 [test](load) add more s3 load regression test cases
(#24906)
2c25e0a681 is described below
commit 2c25e0a6817a63d6bac29b2de59bf2b49ddae44f
Author: Siyang Tang <[email protected]>
AuthorDate: Thu Sep 28 22:01:36 2023 +0800
[test](load) add more s3 load regression test cases (#24906)
---
.../data/load_p0/broker_load/test_s3_load.out | 372 +++++++++++++++++++++
regression-test/data/load_p0/tvf/test_s3_tvf.out | 87 +++++
.../load_p0/broker_load/ddl/mow_tbl_basic.sql | 1 +
.../suites/load_p0/broker_load/test_s3_load.groovy | 109 +++++-
.../suites/load_p0/tvf/test_s3_tvf.groovy | 95 +++++-
5 files changed, 660 insertions(+), 4 deletions(-)
diff --git a/regression-test/data/load_p0/broker_load/test_s3_load.out
b/regression-test/data/load_p0/broker_load/test_s3_load.out
index 6bc8c5ca75..022c08cf8f 100644
--- a/regression-test/data/load_p0/broker_load/test_s3_load.out
+++ b/regression-test/data/load_p0/broker_load/test_s3_load.out
@@ -1,22 +1,394 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+18
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+60
+
+-- !select --
+34
+
+-- !select --
+34
+
+-- !select --
+34
+
+-- !select --
+54
+
+-- !select --
+17
+
+-- !select --
+17
+
+-- !select --
+80
+
-- !select --
48
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+72
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+100
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+90
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+120
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+108
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+140
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+126
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+160
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+144
+
+-- !select --
+18
+
+-- !select --
+18
+
-- !select --
180
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+162
+
+-- !select --
+18
+
+-- !select --
+18
+
-- !select --
200
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+180
+
+-- !select --
+18
+
+-- !select --
+18
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+220
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+198
+
-- !select --
18
+-- !select --
+18
+
+-- !select --
+240
+
-- !select --
48
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+216
+
-- !select --
18
+-- !select --
+18
+
+-- !select --
+260
+
-- !select --
48
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+280
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+48
+
+-- !select --
+47
+
+-- !select --
+47
+
+-- !select --
+48
+
+-- !select --
+216
+
+-- !select --
+280
+
+-- !select --
+18
+
+-- !select --
+47
+
+-- !select --
+18
+
+-- !select --
+47
+
diff --git a/regression-test/data/load_p0/tvf/test_s3_tvf.out
b/regression-test/data/load_p0/tvf/test_s3_tvf.out
index 71ab221db1..88b962cfd6 100644
--- a/regression-test/data/load_p0/tvf/test_s3_tvf.out
+++ b/regression-test/data/load_p0/tvf/test_s3_tvf.out
@@ -191,3 +191,90 @@
-- !select --
20
+-- !select --
+20
+
+-- !select --
+162
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+141
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+182
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+159
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+202
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+177
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+222
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+195
+
+-- !select --
+16
+
+-- !select --
+16
+
diff --git a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
index efc1ad2cab..8002f235f4 100644
--- a/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
+++ b/regression-test/suites/load_p0/broker_load/ddl/mow_tbl_basic.sql
@@ -61,6 +61,7 @@ PARTITION BY RANGE(k01)
)
DISTRIBUTED BY HASH(k00) BUCKETS 32
PROPERTIES (
+ "function_column.sequence_col" = "k12",
"bloom_filter_columns"="k05",
"replication_num" = "1",
"enable_unique_key_merge_on_write" = "true"
diff --git a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
index 69d193a19b..6312cc3feb 100644
--- a/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
+++ b/regression-test/suites/load_p0/broker_load/test_s3_load.groovy
@@ -41,6 +41,7 @@ suite("test_s3_load", "load_p0") {
]
def uniqTables = [
+ "mow_tbl_basic",
"uniq_tbl_basic"
]
@@ -371,6 +372,80 @@ suite("test_s3_load", "load_p0") {
"", "", "", "", "ORDER BY k01"))
}
+ /*========================================================== json
==========================================================*/
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ "${table}", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\",
\"fuzzy_parse\" = \"true\")"))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.json",
+ "agg_tbl_basic", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "PROPERTIES(\"strip_outer_array\" = \"true\", \"fuzzy_parse\" =
\"true\")"))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.json",
+ "${table}", "", "", "FORMAT AS \"json\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", "PROPERTIES(\"strip_outer_array\" = \"true\",
\"fuzzy_parse\" = \"true\")"))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ "${table}", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", "PROPERTIES(\"read_json_by_line\" =
\"true\")"))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_by_line.json",
+ "agg_tbl_basic", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", "PROPERTIES(\"read_json_by_line\" = \"true\")"))
+
+ for (String table : arrayTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data_by_line.json",
+ "${table}", "", "", "FORMAT AS \"JSON\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+ "", "", "", "", "PROPERTIES(\"read_json_by_line\" =
\"true\")"))
+ }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ "${table}", "", "", "FORMAT AS \"parquet\"",
"(K00,K01,K02,K03,K04,K05,K06,K07,K08,K09,K10,K11,K12,K13,K14,K15,K16,K17,K18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.parq",
+ "agg_tbl_basic", "", "", "FORMAT AS \"PARQUET\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+// for (String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// "${table}", "", "", "FORMAT AS \"parquet\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "", ""))
+// }
+
+ for (String table : basicTables) {
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ "${table}", "", "", "FORMAT AS \"orc\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "", "", ""))
+ }
+
+ attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data.orc",
+ "agg_tbl_basic", "", "", "FORMAT AS \"ORC\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18)",
+ "", "", "SET
(k19=to_bitmap(k04),k20=HLL_HASH(k04),k21=TO_QUANTILE_STATE(k04,1.0),kd19=to_bitmap(k05),kd20=HLL_HASH(k05),kd21=TO_QUANTILE_STATE(k05,1.0))",
"", ""))
+
+// for (String table : arrayTables) {
+// attributesList.add(new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_array_data.parq",
+// "${table}", "", "", "FORMAT AS \"parquet\"",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17)",
+// "", "", "", "", ""))
+// }
+
+ for(String table : uniqTables) {
+ def attributes = new
LoadAttributes("s3://doris-build-1308700295/regression/load/data/basic_data_delete.csv",
+ "${table}", "LINES TERMINATED BY \"\n\"", "COLUMNS TERMINATED
BY \"|\"", "",
"(k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,__DEL__)",
+ "", "", "", "", "DELETE ON __DEL__=true")
+ attributes.dataDesc.mergeType = "MERGE"
+ attributesList.add(attributes)
+ }
+
def ak = getS3AK()
def sk = getS3SK()
@@ -387,6 +462,7 @@ suite("test_s3_load", "load_p0") {
def sql_str = """
LOAD LABEL $label (
+ $attributes.dataDesc.mergeType
DATA INFILE("$attributes.dataDesc.path")
INTO TABLE $attributes.dataDesc.tableName
$attributes.dataDesc.columnTermClause
@@ -411,10 +487,7 @@ suite("test_s3_load", "load_p0") {
logger.info("submit sql: ${sql_str}");
sql """${sql_str}"""
logger.info("Submit load with lable: $label, table:
$attributes.dataDesc.tableName, path: $attributes.dataDesc.path")
- ++i
- }
- for (LoadAttributes attributes : attributesList) {
def max_try_milli_secs = 600000
while (max_try_milli_secs > 0) {
String[][] result = sql """ show load where
label="$attributes.label" order by createtime desc limit 1; """
@@ -439,14 +512,44 @@ suite("test_s3_load", "load_p0") {
assertTrue(false, "load Timeout: $attributes.label")
}
}
+ qt_select """ select count(*) from $attributes.dataDesc.tableName """
+ ++i
}
+// for (LoadAttributes attributes : attributesList) {
+// def max_try_milli_secs = 600000
+// while (max_try_milli_secs > 0) {
+// String[][] result = sql """ show load where
label="$attributes.label" order by createtime desc limit 1; """
+// if (result[0][2].equals("FINISHED")) {
+// if (attributes.isExceptFailed) {
+// assertTrue(false, "load should be failed but was
success: $result")
+// }
+// logger.info("Load FINISHED " + attributes.label + ":
$result")
+// break
+// }
+// if (result[0][2].equals("CANCELLED")) {
+// if (attributes.isExceptFailed) {
+// logger.info("Load FINISHED " + attributes.label)
+// break
+// }
+// assertTrue(false, "load failed: $result")
+// break
+// }
+// Thread.sleep(1000)
+// max_try_milli_secs -= 1000
+// if (max_try_milli_secs <= 0) {
+// assertTrue(false, "load Timeout: $attributes.label")
+// }
+// }
+// }
+
for(String tbl : tables) {
qt_select """ select count(*) from ${tbl} """
}
}
class DataDesc {
+ public String mergeType = ""
public String path
public String tableName
public String lineTermClause
diff --git a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
index 77d32b9c42..097c551ea2 100644
--- a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
+++ b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
@@ -41,7 +41,8 @@ suite("test_s3_tvf", "load_p0") {
]
def uniqTable = [
- "uniq_tbl_basic_tvf"
+ "uniq_tbl_basic_tvf",
+ "mow_tbl_basic_tvf"
]
def attributeList = [
@@ -273,6 +274,98 @@ suite("test_s3_tvf", "load_p0") {
.addProperty("column_separator", "|"))
}
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("format", "parquet")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.parq")
+ .addProperty("format", "parquet")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.parq")
+ .addProperty("format", "parquet")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("format", "orc")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.orc")
+ .addProperty("format", "orc")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.orc")
+ .addProperty("format", "orc")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "false")
+ .addProperty("strip_outer_array", "true")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18","k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18",
"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,
to_bitmap(k05) as k19, HLL_HASH(k05) as k20, TO_QUANTILE_STATE(k04, 1.0) as
k21, to_bitmap(k05) as kd19, HLL_HASH(k05) as kd20, TO_QUANTILE_STATE(k04, 1.0)
as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table,
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17",
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17", "",
"")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_by_line.json")
+ .addProperty("format", "json")
+ .addProperty("read_json_by_line", "true")
+ .addProperty("strip_outer_array", "false")
+ .addProperty("column_separator", "|"))
+ }
+
def ak = getS3AK()
def sk = getS3SK()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
