This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e8aab052bd6 [fix](Outfile) add more regression-test cases for `Select
into outfile` (#45230)
e8aab052bd6 is described below
commit e8aab052bd6ba299d8f585e8b5d23096a924d7b8
Author: Tiewei Fang <[email protected]>
AuthorDate: Wed Dec 11 11:19:47 2024 +0800
[fix](Outfile) add more regression-test cases for `Select into outfile`
(#45230)
Problem Summary:
1. add more regression-test cases for `Select into outfile`
2. Support to export the NULL type data in Doris to Orc file format.
---
.../org/apache/doris/analysis/OutFileClause.java | 6 ++
.../export_p0/outfile/test_outfile_constant.out | 76 ++++++++++++++++++
.../export_p0/outfile/test_outfile_null_type.out | 12 +++
...ll_type.groovy => test_outfile_constant.groovy} | 92 ++++++++++++----------
.../outfile/test_outfile_null_type.groovy | 21 +++--
5 files changed, 155 insertions(+), 52 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
index 026e4da29b5..d05a52ec905 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java
@@ -289,6 +289,9 @@ public class OutFileClause {
private String dorisTypeToOrcTypeMap(Type dorisType) throws
AnalysisException {
String orcType = "";
switch (dorisType.getPrimitiveType()) {
+ case NULL_TYPE:
+ orcType = "tinyint";
+ break;
case BOOLEAN:
case TINYINT:
case SMALLINT:
@@ -413,6 +416,9 @@ public class OutFileClause {
Pair<String, String> schema = this.orcSchemas.get(i);
Type resultType = resultExprs.get(i).getType();
switch (resultType.getPrimitiveType()) {
+ case NULL_TYPE:
+ checkOrcType(schema.second, "tinyint", true,
resultType.getPrimitiveType().toString());
+ break;
case BOOLEAN:
case TINYINT:
case SMALLINT:
diff --git a/regression-test/data/export_p0/outfile/test_outfile_constant.out
b/regression-test/data/export_p0/outfile/test_outfile_constant.out
new file mode 100644
index 00000000000..855a66a129f
--- /dev/null
+++ b/regression-test/data/export_p0/outfile/test_outfile_constant.out
@@ -0,0 +1,76 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select_export --
+1 ftw-1 19
+2 ftw-2 20
+3 ftw-3 21
+4 ftw-4 22
+5 ftw-5 23
+6 ftw-6 24
+7 ftw-7 25
+8 ftw-8 26
+9 ftw-9 27
+10 \N \N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
+-- !s3_select --
+1
+
+-- !s3_select --
+3.1415926
+
+-- !s3_select --
+2.718281828459045
+
+-- !s3_select --
+3.141592653589793
+
+-- !s3_select --
+2003
+
+-- !s3_select --
+2024-12-09
+
+-- !s3_select --
+\N
+
diff --git a/regression-test/data/export_p0/outfile/test_outfile_null_type.out
b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
index 1777bba72c1..a39131b61e2 100644
--- a/regression-test/data/export_p0/outfile/test_outfile_null_type.out
+++ b/regression-test/data/export_p0/outfile/test_outfile_null_type.out
@@ -23,6 +23,18 @@
8 ftw-8 26 \N
9 ftw-9 27 \N
+-- !select_load_orc --
+1 ftw-1 19 \N
+10 \N \N \N
+2 ftw-2 20 \N
+3 ftw-3 21 \N
+4 ftw-4 22 \N
+5 ftw-5 23 \N
+6 ftw-6 24 \N
+7 ftw-7 25 \N
+8 ftw-8 26 \N
+9 ftw-9 27 \N
+
-- !select_load_csv --
1 ftw-1 19 \N
10 \N \N \N
diff --git
a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
b/regression-test/suites/export_p0/outfile/test_outfile_constant.groovy
similarity index 56%
copy from regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
copy to regression-test/suites/export_p0/outfile/test_outfile_constant.groovy
index 492b6770195..3df29499237 100644
--- a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
+++ b/regression-test/suites/export_p0/outfile/test_outfile_constant.groovy
@@ -15,20 +15,20 @@
// specific language governing permissions and limitations
// under the License.
-suite("test_outfile_null_type", "p0") {
+suite("test_outfile_constant", "p0") {
String ak = getS3AK()
String sk = getS3SK()
String s3_endpoint = getS3Endpoint()
String region = getS3Region()
String bucket = context.config.otherConfigs.get("s3BucketName");
- def export_table_name = "test_outfile_null_type"
- def outFilePath = "${bucket}/outfile/null_type/exp_"
+ def export_table_name = "test_outfile_constant"
+ def outFilePath = "${bucket}/outfile/constant_type/exp_"
- def outfile_to_S3 = { format ->
+ def outfile_to_S3 = { select_stmt, format ->
// select ... into outfile ...
def res = sql """
- SELECT *, NULL AS null_col FROM ${export_table_name} t
+ ${select_stmt}
INTO OUTFILE "s3://${outFilePath}"
FORMAT AS ${format}
PROPERTIES (
@@ -42,6 +42,17 @@ suite("test_outfile_null_type", "p0") {
return res[0][3]
}
+ def s3_tvf = { outfile_url, format ->
+ order_qt_s3_select """ SELECT * FROM S3 (
+ "uri" = "http://${bucket}.${s3_endpoint}${outfile_url.substring(5
+ bucket.length(), outfile_url.length() - 1)}0.${format}",
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "${format}",
+ "region" = "${region}"
+ );
+ """
+ }
+
sql """ DROP TABLE IF EXISTS ${export_table_name} """
sql """
CREATE TABLE `${export_table_name}` (
@@ -72,40 +83,39 @@ suite("test_outfile_null_type", "p0") {
logger.info("insert result: " + insert_res.toString())
qt_select_export """ SELECT * FROM ${export_table_name} t ORDER BY id; """
- // parquet file format
- def format = "parquet"
- def outfile_url = outfile_to_S3("${format}")
- order_qt_select_load_parquet """ SELECT * FROM S3 (
- "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.${format}",
- "ACCESS_KEY"= "${ak}",
- "SECRET_KEY" = "${sk}",
- "format" = "${format}",
- "region" = "${region}"
- );
- """
-
- // TODO(ftw): orc supports to export data of NULL type.
- // orc file foramt
- // format = "orc"
- // outfile_url = outfile_to_S3("${format}")
- // qt_select_load_orc """ SELECT * FROM S3 (
- // "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.${format}",
- // "ACCESS_KEY"= "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "format" = "${format}",
- // "region" = "${region}"
- // );
- // """
-
- // csv file foramt
- format = "csv"
- outfile_url = outfile_to_S3("${format}")
- order_qt_select_load_csv """ SELECT * FROM S3 (
- "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.${format}",
- "ACCESS_KEY"= "${ak}",
- "SECRET_KEY" = "${sk}",
- "format" = "${format}",
- "region" = "${region}"
- );
- """
+
+ def test_outfile_constant = { format ->
+ // select 1
+ def outfile_url = outfile_to_S3("Select 1", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select 3.1415926
+ outfile_url = outfile_to_S3("Select 3.1415926", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select e()
+ outfile_url = outfile_to_S3("Select e()", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select Pi()
+ outfile_url = outfile_to_S3("Select Pi()", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select 2024-12-09;
+ outfile_url = outfile_to_S3("Select 2024-12-09", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select "2024-12-09";
+ outfile_url = outfile_to_S3("""Select "2024-12-09" """, "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+
+ // select null
+ outfile_url = outfile_to_S3("Select null", "${format}")
+ s3_tvf("${outfile_url}", "${format}")
+ }
+
+ test_outfile_constant("parquet");
+ test_outfile_constant("orc");
+ test_outfile_constant("csv");
+
}
\ No newline at end of file
diff --git
a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
index 492b6770195..fcd75f63c5b 100644
--- a/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
+++ b/regression-test/suites/export_p0/outfile/test_outfile_null_type.groovy
@@ -84,18 +84,17 @@ suite("test_outfile_null_type", "p0") {
);
"""
- // TODO(ftw): orc supports to export data of NULL type.
// orc file foramt
- // format = "orc"
- // outfile_url = outfile_to_S3("${format}")
- // qt_select_load_orc """ SELECT * FROM S3 (
- // "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.${format}",
- // "ACCESS_KEY"= "${ak}",
- // "SECRET_KEY" = "${sk}",
- // "format" = "${format}",
- // "region" = "${region}"
- // );
- // """
+ format = "orc"
+ outfile_url = outfile_to_S3("${format}")
+ order_qt_select_load_orc """ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}${outfile_url.substring(5 + bucket.length(),
outfile_url.length() - 1)}0.${format}",
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "${format}",
+ "region" = "${region}"
+ );
+ """
// csv file foramt
format = "csv"
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
