This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dae0dc1652 [test](load) add some S3 TVF load regression tests (#24719)
dae0dc1652 is described below
commit dae0dc1652f142801c905b946bf865ede3c9ded0
Author: Siyang Tang <[email protected]>
AuthorDate: Tue Sep 26 12:21:42 2023 +0800
[test](load) add some S3 TVF load regression tests (#24719)
---
regression-test/data/load_p0/tvf/test_s3_tvf.out | 193 ++++++++++
.../suites/load_p0/tvf/ddl/agg_tbl_basic_tvf.sql | 59 ++++
.../load_p0/tvf/ddl/agg_tbl_basic_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/dup_tbl_array_tvf.sql | 42 +++
.../load_p0/tvf/ddl/dup_tbl_array_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/dup_tbl_basic_tvf.sql | 65 ++++
.../load_p0/tvf/ddl/dup_tbl_basic_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/mow_tbl_array_tvf.sql | 44 +++
.../load_p0/tvf/ddl/mow_tbl_array_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql | 67 ++++
.../load_p0/tvf/ddl/mow_tbl_basic_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/uniq_tbl_array_tvf.sql | 43 +++
.../load_p0/tvf/ddl/uniq_tbl_array_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql | 58 +++
.../load_p0/tvf/ddl/uniq_tbl_basic_tvf_drop.sql | 1 +
.../suites/load_p0/tvf/test_s3_tvf.groovy | 393 +++++++++++++++++++++
16 files changed, 971 insertions(+)
diff --git a/regression-test/data/load_p0/tvf/test_s3_tvf.out
b/regression-test/data/load_p0/tvf/test_s3_tvf.out
new file mode 100644
index 0000000000..71ab221db1
--- /dev/null
+++ b/regression-test/data/load_p0/tvf/test_s3_tvf.out
@@ -0,0 +1,193 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+18
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+40
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+36
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+60
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+54
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+60
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+54
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+70
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+64
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+90
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+82
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+110
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+100
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+130
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+118
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+142
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+20
+
+-- !select --
+123
+
+-- !select --
+16
+
+-- !select --
+16
+
+-- !select --
+20
+
diff --git a/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf.sql
new file mode 100644
index 0000000000..2a7b109a94
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf.sql
@@ -0,0 +1,59 @@
+CREATE TABLE agg_tbl_basic_tvf
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN REPLACE NULL,
+ k03 TINYINT SUM NULL,
+ k04 SMALLINT SUM NULL,
+ k05 INT SUM NULL,
+ k06 BIGINT SUM NULL,
+ k07 LARGEINT SUM NULL,
+ k08 FLOAT SUM NULL,
+ k09 DOUBLE SUM NULL,
+ k10 DECIMAL(9,1) SUM NULL,
+ k11 DECIMALV3(9,1) SUM NULL,
+ k12 DATETIME REPLACE NULL,
+ k13 DATEV2 REPLACE NULL,
+ k14 DATETIMEV2 REPLACE NULL,
+ k15 CHAR(300) REPLACE NULL,
+ k16 VARCHAR(300) REPLACE NULL,
+ k17 STRING REPLACE NULL,
+ k18 JSON REPLACE NULL,
+ k19 BITMAP BITMAP_UNION NULL,
+ k20 HLL HLL_UNION NULL,
+ k21 QUANTILE_STATE QUANTILE_UNION NULL,
+ kd01 BOOLEAN REPLACE NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT SUM NOT NULL DEFAULT "1",
+ kd03 SMALLINT SUM NOT NULL DEFAULT "2",
+ kd04 INT SUM NOT NULL DEFAULT "3",
+ kd05 BIGINT SUM NOT NULL DEFAULT "4",
+ kd06 LARGEINT SUM NOT NULL DEFAULT "5",
+ kd07 FLOAT SUM NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE SUM NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL SUM NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 SUM NOT NULL DEFAULT "999999999",
+ kd11 DATE REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME REPLACE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 REPLACE NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 REPLACE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING REPLACE NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON REPLACE NULL,
+ kd19 BITMAP BITMAP_UNION NULL,
+ kd20 HLL HLL_UNION NULL,
+ kd21 QUANTILE_STATE QUANTILE_UNION NULL,
+
+ INDEX idx_bitmap_k104 (`k01`) USING BITMAP
+)
+ AGGREGATE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf_drop.sql
new file mode 100644
index 0000000000..bedd9ea634
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/agg_tbl_basic_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS agg_tbl_basic_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf.sql
new file mode 100644
index 0000000000..4ddbba4063
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf.sql
@@ -0,0 +1,42 @@
+CREATE TABLE dup_tbl_array_tvf
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+)
+ DUPLICATE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf_drop.sql
new file mode 100644
index 0000000000..9b5ecb1229
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_array_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS dup_tbl_array_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf.sql
new file mode 100644
index 0000000000..573807576c
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf.sql
@@ -0,0 +1,65 @@
+CREATE TABLE dup_tbl_basic_tvf
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+
+)
+ DUPLICATE KEY(k00)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf_drop.sql
new file mode 100644
index 0000000000..a9f67658af
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/dup_tbl_basic_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS dup_tbl_basic_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf.sql
new file mode 100644
index 0000000000..65b99950ab
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf.sql
@@ -0,0 +1,44 @@
+CREATE TABLE mow_tbl_array_tvf
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]"
+)
+ UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
diff --git a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf_drop.sql
new file mode 100644
index 0000000000..a4ffbdd312
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_array_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS mow_tbl_array_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql
new file mode 100644
index 0000000000..f17bbf6a83
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf.sql
@@ -0,0 +1,67 @@
+CREATE TABLE mow_tbl_basic_tvf
+(
+ k00 INT NOT NULL,
+ k01 DATE NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_inverted_k104 (`k05`) USING INVERTED,
+ INDEX idx_inverted_k110 (`k11`) USING INVERTED,
+ INDEX idx_inverted_k113 (`k13`) USING INVERTED,
+ INDEX idx_inverted_k114 (`k14`) USING INVERTED,
+ INDEX idx_inverted_k117 (`k17`) USING INVERTED PROPERTIES("parser" =
"english"),
+ INDEX idx_bitmap_k104 (`k05`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`k11`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP,
+ INDEX idx_ngrambf_k115 (`k15`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k116 (`k16`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256"),
+ INDEX idx_ngrambf_k117 (`k17`) USING NGRAM_BF PROPERTIES("gram_size"="3",
"bf_size"="256")
+)
+ UNIQUE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "bloom_filter_columns"="k05",
+ "replication_num" = "1",
+ "enable_unique_key_merge_on_write" = "true"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf_drop.sql
new file mode 100644
index 0000000000..d9f4884e35
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/mow_tbl_basic_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS mow_tbl_basic_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf.sql
new file mode 100644
index 0000000000..b38a1b0a3d
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf.sql
@@ -0,0 +1,43 @@
+CREATE TABLE uniq_tbl_array_tvf
+(
+ k00 INT NOT NULL,
+ k01 array<BOOLEAN> NULL,
+ k02 array<TINYINT> NULL,
+ k03 array<SMALLINT> NULL,
+ k04 array<INT> NULL,
+ k05 array<BIGINT> NULL,
+ k06 array<LARGEINT> NULL,
+ k07 array<FLOAT> NULL,
+ k08 array<DOUBLE> NULL,
+ k09 array<DECIMAL> NULL,
+ k10 array<DECIMALV3> NULL,
+ k11 array<DATE> NULL,
+ k12 array<DATETIME> NULL,
+ k13 array<DATEV2> NULL,
+ k14 array<DATETIMEV2> NULL,
+ k15 array<CHAR> NULL,
+ k16 array<VARCHAR> NULL,
+ k17 array<STRING> NULL,
+ kd01 array<BOOLEAN> NOT NULL DEFAULT "[]",
+ kd02 array<TINYINT> NOT NULL DEFAULT "[]",
+ kd03 array<SMALLINT> NOT NULL DEFAULT "[]",
+ kd04 array<INT> NOT NULL DEFAULT "[]",
+ kd05 array<BIGINT> NOT NULL DEFAULT "[]",
+ kd06 array<LARGEINT> NOT NULL DEFAULT "[]",
+ kd07 array<FLOAT> NOT NULL DEFAULT "[]",
+ kd08 array<DOUBLE> NOT NULL DEFAULT "[]",
+ kd09 array<DECIMAL> NOT NULL DEFAULT "[]",
+ kd10 array<DECIMALV3> NOT NULL DEFAULT "[]",
+ kd11 array<DATE> NOT NULL DEFAULT "[]",
+ kd12 array<DATETIME> NOT NULL DEFAULT "[]",
+ kd13 array<DATEV2> NOT NULL DEFAULT "[]",
+ kd14 array<DATETIMEV2> NOT NULL DEFAULT "[]",
+ kd15 array<CHAR> NOT NULL DEFAULT "[]",
+ kd16 array<VARCHAR> NOT NULL DEFAULT "[]",
+ kd17 array<STRING> NOT NULL DEFAULT "[]",
+)
+ UNIQUE KEY(k00)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf_drop.sql
new file mode 100644
index 0000000000..8a11460215
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_array_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS uniq_tbl_array_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql
b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql
new file mode 100644
index 0000000000..b255495130
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf.sql
@@ -0,0 +1,58 @@
+CREATE TABLE uniq_tbl_basic_tvf
+(
+ k00 INT NOT NULL,
+ k01 DATE NOT NULL,
+ k02 BOOLEAN NULL,
+ k03 TINYINT NULL,
+ k04 SMALLINT NULL,
+ k05 INT NULL,
+ k06 BIGINT NULL,
+ k07 LARGEINT NULL,
+ k08 FLOAT NULL,
+ k09 DOUBLE NULL,
+ k10 DECIMAL(9,1) NULL,
+ k11 DECIMALV3(9,1) NULL,
+ k12 DATETIME NULL,
+ k13 DATEV2 NULL,
+ k14 DATETIMEV2 NULL,
+ k15 CHAR NULL,
+ k16 VARCHAR NULL,
+ k17 STRING NULL,
+ k18 JSON NULL,
+ kd01 BOOLEAN NOT NULL DEFAULT "TRUE",
+ kd02 TINYINT NOT NULL DEFAULT "1",
+ kd03 SMALLINT NOT NULL DEFAULT "2",
+ kd04 INT NOT NULL DEFAULT "3",
+ kd05 BIGINT NOT NULL DEFAULT "4",
+ kd06 LARGEINT NOT NULL DEFAULT "5",
+ kd07 FLOAT NOT NULL DEFAULT "6.0",
+ kd08 DOUBLE NOT NULL DEFAULT "7.0",
+ kd09 DECIMAL NOT NULL DEFAULT "888888888",
+ kd10 DECIMALV3 NOT NULL DEFAULT "999999999",
+ kd11 DATE NOT NULL DEFAULT "2023-08-24",
+ kd12 DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd13 DATEV2 NOT NULL DEFAULT "2023-08-24",
+ kd14 DATETIMEV2 NOT NULL DEFAULT CURRENT_TIMESTAMP,
+ kd15 CHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd16 VARCHAR(300) NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd17 STRING NOT NULL DEFAULT "我能吞下玻璃而不伤身体",
+ kd18 JSON NULL,
+
+ INDEX idx_bitmap_k104 (`k02`) USING BITMAP,
+ INDEX idx_bitmap_k110 (`kd01`) USING BITMAP,
+ INDEX idx_bitmap_k113 (`k13`) USING BITMAP,
+ INDEX idx_bitmap_k114 (`k14`) USING BITMAP,
+ INDEX idx_bitmap_k117 (`k17`) USING BITMAP
+)
+UNIQUE KEY(k00,k01)
+PARTITION BY RANGE(k01)
+(
+ PARTITION p1 VALUES [('2023-08-01'), ('2023-08-11')),
+ PARTITION p2 VALUES [('2023-08-11'), ('2023-08-21')),
+ PARTITION p3 VALUES [('2023-08-21'), ('2023-09-01'))
+)
+DISTRIBUTED BY HASH(k00) BUCKETS 32
+PROPERTIES (
+ "function_column.sequence_col" = "k12",
+ "replication_num" = "1"
+);
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf_drop.sql
b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf_drop.sql
new file mode 100644
index 0000000000..0e64655f12
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/ddl/uniq_tbl_basic_tvf_drop.sql
@@ -0,0 +1 @@
+DROP TABLE IF EXISTS uniq_tbl_basic_tvf
\ No newline at end of file
diff --git a/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
new file mode 100644
index 0000000000..77d32b9c42
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/test_s3_tvf.groovy
@@ -0,0 +1,393 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_s3_tvf", "load_p0") {
+
+ def tables = [
+ "agg_tbl_basic_tvf",
+ "dup_tbl_array_tvf",
+ "dup_tbl_basic_tvf",
+ "mow_tbl_array_tvf",
+ "mow_tbl_basic_tvf",
+ "uniq_tbl_array_tvf",
+ "uniq_tbl_basic_tvf"
+ ]
+
+ //deal with agg tables in separate
+ def basicTables = [
+ "dup_tbl_basic_tvf",
+ "mow_tbl_basic_tvf",
+ "uniq_tbl_basic_tvf",
+ ]
+
+ def arrayTables = [
+ "dup_tbl_array_tvf",
+ "uniq_tbl_array_tvf",
+ "mow_tbl_array_tvf"
+ ]
+
+ def uniqTable = [
+ "uniq_tbl_basic_tvf"
+ ]
+
+ def attributeList = [
+
+ ]
+
+ // errors: no capital prop, no column mappings, array type loading stuck
+
+ /* normal */
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17", "K18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["K00", "K01", "K02", "K03",
"K04", "K05", "K06", "K07", "K08", "K09", "K10", "K11", "K12", "K13", "K14",
"K15", "K16", "K17"], "", "").addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+// for(String table : basicTables) {
+// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17", "k18"], "", "")
+// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295/regression/load/data/basic_data.csv";)
+// .addProperty("format", "csv")
+// .addProperty("column_separator", "|")
+// .addProperty("use_path_style", "true"))
+// }
+//
+// attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+//
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv";)
+// .addProperty("format", "csv")
+// .addProperty("column_separator", "|")
+// .addProperty("use_path_style", "true"))
+//
+// for(String table : arrayTables) {
+// attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02",
"k03", "k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13",
"k14", "k15", "k16", "k17"], "", "")
+// .addProperty("uri",
"https://cos.ap-beijing.myqcloud.com/doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv";)
+// .addProperty("format", "csv")
+// .addProperty("column_separator", "|")
+// .addProperty("use_path_style", "true"))
+// }
+
+ /* error */
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "", true)
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "", true)
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "", true)
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ /* skip lines */
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("skip_lines", "10"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("skip_lines", "10"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data_with_errors.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("skip_lines", "10"))
+ }
+
+ /* compress type */
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator",
"|").addProperty("compress_type", "GZ"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.gz";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "GZ"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.gz";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "GZ"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator",
"|").addProperty("compress_type", "BZ2"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.bz2";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "BZ2"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.bz2";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "BZ2"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator",
"|").addProperty("compress_type", "LZ4FRAME"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv.lz4";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "LZ4FRAME"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "", "")
+ .addProperty("uri",
"https://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv.lz4";)
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|")
+ .addProperty("compress_type", "LZ4FRAME"))
+ }
+
+ for(String table : basicTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "WHERE c1 > 50", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ attributeList.add(new TvfAttribute("agg_tbl_basic_tvf", "c1 as k00,c2 as
k01,c3 as k02,c4 as k03,c5 as k04,c6 as k05,c7 as k06,c8 as k07,c9 as k08,c10
as k09,c11 as k10,c12 as k11,c13 as k12,c14 as k13,c15 as k14,c16 as k15,c17 as
k16,c18 as k17,c19 as k18, to_bitmap(c6) as k19, HLL_HASH(c6) as k20,
TO_QUANTILE_STATE(c5, 1.0) as k21, to_bitmap(c6) as kd19, HLL_HASH(c6) as kd20,
TO_QUANTILE_STATE(c5, 1.0) as kd21",
+
"k00,k01,k02,k03,k04,k05,k06,k07,k08,k09,k10,k11,k12,k13,k14,k15,k16,k17,k18,k19,k20,k21,kd19,kd20,kd21"
,"WHERE c1 > 50", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+
+ for(String table : arrayTables) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17"], "WHERE c1 > 50", "")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_array_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ for(String table : uniqTable) {
+ attributeList.add(new TvfAttribute(table, ["k00", "k01", "k02", "k03",
"k04", "k05", "k06", "k07", "k08", "k09", "k10", "k11", "k12", "k13", "k14",
"k15", "k16", "k17", "k18"], "", "ORDER BY c1")
+ .addProperty("uri",
"s3://doris-build-1308700295.cos.ap-beijing.myqcloud.com/regression/load/data/basic_data.csv")
+ .addProperty("format", "csv")
+ .addProperty("column_separator", "|"))
+ }
+
+ def ak = getS3AK()
+ def sk = getS3SK()
+
+ for(String tbl : tables) {
+ sql new File("""${ context.file.parent }/ddl/${tbl}_drop.sql""").text
+ sql new File("""${ context.file.parent }/ddl/${tbl}.sql""").text
+ }
+
+ def i = 0
+ for (TvfAttribute attribute : attributeList) {
+
+ def prop = attribute.getPropertiesStr()
+ def insertList = attribute.getInsertList()
+ def selectList = attribute.getSelectList()
+
+ def sqlStr = """
+ INSERT INTO ${attribute.tableName} (${insertList})
+ SELECT ${selectList}
+ FROM S3 (
+ "s3.access_key" = "$ak",
+ "s3.secret_key" = "$sk",
+ "s3.region" = "ap-beijing",
+ ${prop}
+ ) ${attribute.whereClause}
+ ${attribute.orderByClause}
+ """
+
+ logger.info("submit sql: ${sqlStr}");
+ try {
+ sql """${sqlStr}"""
+ } catch (Exception ex) {
+ assertTrue(attribute.expectFiled)
+ logger.info("error: ", ex)
+ }
+
+ qt_select """ select count(*) from $attribute.tableName """
+ ++i
+ }
+
+
+}
+
+class TvfAttribute {
+ public String tableName
+ public String[] columns = new ArrayList<>()
+ public Map<String, String> properties = new HashMap<>()
+ public String whereClause
+ public String orderByClause
+ public boolean expectFiled
+ public String selectList
+ public String insertList
+
+ TvfAttribute(String tableName, List<String> columns, String whereClause,
String orderByClause, boolean expectFiled = false) {
+ this.tableName = tableName
+ this.columns = columns
+ this.whereClause = whereClause
+ this.orderByClause = orderByClause
+ this.expectFiled = expectFiled
+ this.insertList = ""
+ this.selectList = ""
+ }
+
+ TvfAttribute(String tableName, String selectList, String insertList,
String whereClause, String orderByClause, boolean expectFiled = false) {
+ this.tableName = tableName
+ this.columns = columns
+ this.whereClause = whereClause
+ this.orderByClause = orderByClause
+ this.expectFiled = expectFiled
+ this.insertList = insertList
+ this.selectList = selectList
+ }
+
+ String getSelectList() {
+ if (selectList.length() != 0) {
+ return selectList
+ }
+ if ("csv".equalsIgnoreCase(properties.get("format"))) {
+ String res = ""
+ def i = 1
+ for(String column : columns) {
+ res += ("c$i as $column")
+ res += ","
+ ++i
+ }
+ return res.substring(0, res.size() - 1)
+ }
+ return getInsertList()
+ }
+
+ String getInsertList() {
+ if (insertList.length() != 0) {
+ return insertList
+ }
+ String res = ""
+ for(String column : columns) {
+ res += column
+ res += ","
+ }
+ return res.substring(0, res.size() - 1)
+ }
+
+ String getPropertiesStr() {
+ if (properties.isEmpty()) {
+ return "*"
+ }
+ String prop = ""
+ properties.forEach (k, v) -> {
+ prop += "\"${k}\" = \"${v}\","
+ }
+ prop = prop.substring(0, prop.size() - 1)
+ return prop
+ }
+
+ TvfAttribute addProperty(String k, String v) {
+ properties.put(k, v)
+ return this
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
