This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 90afc28cfc6c04f4ad3fa2af9acb045e75cf5558 Author: amory <[email protected]> AuthorDate: Tue Aug 22 15:25:10 2023 +0800 [FIX](regress-test) fix local_tvf with complex-type in cluster (#23299) * fixed * update * fixed --- .../tvf/test_local_tvf_with_complex_type.groovy | 22 ++++++--- ...t_local_tvf_with_complex_type_element_at.groovy | 52 ++++++++++++++-------- ...l_tvf_with_complex_type_insertinto_doris.groovy | 26 ++++++++--- 3 files changed, 69 insertions(+), 31 deletions(-) diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy index d7271501d8..cd5948b22c 100644 --- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type.groovy @@ -1,3 +1,5 @@ +import org.junit.Assert + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,21 +19,31 @@ // This suit test the `backends` tvf suite("test_local_tvf_with_complex_type", "p0") { - List<List<Object>> table = sql """ select * from backends(); """ - assertTrue(table.size() > 0) - def be_id = table[0][0] + List<List<Object>> backends = sql """ show backends """ def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/" + assertTrue(backends.size() > 0) + def be_id = backends[0][0] + // cluster mode need to make sure all be has this data + def outFilePath="/" + def transFile01="${dataFilePath}/complex_type.orc" + def transFile02="${dataFilePath}/complex_type.parquet" + for (List<Object> backend : backends) { + def be_host = backend[1] + scpFiles ("root", be_host, transFile01, outFilePath, false); + scpFiles ("root", be_host, transFile02, outFilePath, false); + } + qt_sql """ select * from local( - "file_path" = "${dataFilePath}/complex_type.orc", + "file_path" = "${outFilePath}/complex_type.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select * from local( - "file_path" = "${dataFilePath}/complex_type.parquet", + "file_path" = "${outFilePath}/complex_type.parquet", "backend_id" = "${be_id}", "format" = "parquet"); """ diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy index b27c6bfa53..91c00a9046 100644 --- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_element_at.groovy @@ -1,3 +1,5 @@ +import org.junit.Assert + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,11 +19,23 @@ // This suit test the `backends` tvf suite("test_local_tvf_with_complex_type_element_at", "p0") { - List<List<Object>> table = sql """ select * from backends(); """ - assertTrue(table.size() > 0) - def be_id = table[0][0] + List<List<Object>> backends = sql """ show backends """ + assertTrue(backends.size() > 0) + def be_id = backends[0][0] def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/" + + // cluster mode need to make sure all be has this data + def outFilePath="/" + def transFile01="${dataFilePath}/t.orc" + def transFile02="${dataFilePath}/t.parquet" + for (List<Object> backend : backends) { + def be_host = backend[1] + scpFiles ("root", be_host, transFile01, outFilePath, false); + scpFiles ("root", be_host, transFile02, outFilePath, false); + } + + /** * here is file schema * var schema = StructType( @@ -39,82 +53,82 @@ suite("test_local_tvf_with_complex_type_element_at", "p0") { qt_sql """ select * from local( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select count(*) from local( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select arr_arr[1][1] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select arr_map[1] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select arr_map[1]["WdTnFb-LHW8Nel-laB-HCQA"] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select map_map["W1iF16-DE1gzJx-avC-Mrf6"]["HJVQSC-46l3xm7-J6c-moIH"] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select map_arr[1] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select map_arr[1][7] from local ( - "file_path" = "${dataFilePath}/t.orc", + "file_path" = "${outFilePath}/t.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ select * from local( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet"); """ qt_sql """ select count(*) from local( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet"); """ qt_sql """ select arr_arr[1][1] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" qt_sql """ select arr_map[1] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" qt_sql """ select arr_map[1]["WdTnFb-LHW8Nel-laB-HCQA"] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" qt_sql """ select map_map["W1iF16-DE1gzJx-avC-Mrf6"]["HJVQSC-46l3xm7-J6c-moIH"] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" qt_sql """ select map_arr[1] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" qt_sql """ select map_arr[1][7] from local ( - "file_path" = "${dataFilePath}/t.parquet", + "file_path" = "${outFilePath}/t.parquet", "backend_id" = "${be_id}", "format" = "parquet");""" } diff --git a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy index bcc4526d36..fbe8fb5597 100644 --- a/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy +++ b/regression-test/suites/external_table_p0/tvf/test_local_tvf_with_complex_type_insertinto_doris.groovy @@ -1,3 +1,5 @@ +import org.junit.Assert + // Licensed to the Apache Software Foundation (ASF) under one // or more contributor license agreements. See the NOTICE file // distributed with this work for additional information @@ -17,12 +19,22 @@ // This suit test the `backends` tvf suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") { - List<List<Object>> table = sql """ select * from backends(); """ - assertTrue(table.size() > 0) - def be_id = table[0][0] + List<List<Object>> backends = sql """ select * from backends(); """ + assertTrue(backends.size() > 0) + def be_id = backends[0][0] def dataFilePath = context.config.dataPath + "/external_table_p0/tvf/" def table_name = "comp" + // cluster mode need to make sure all be has this data + def outFilePath="/" + def transFile01="${dataFilePath}/comp.orc" + def transFile02="${dataFilePath}/comp.parquet" + for (List<Object> backend : backends) { + def be_host = backend[1] + scpFiles ("root", be_host, transFile01, outFilePath, false); + scpFiles ("root", be_host, transFile02, outFilePath, false); + } + qt_sql """ADMIN SET FRONTEND CONFIG ('disable_nested_complex_type' = 'false')""" // create doris table @@ -46,13 +58,13 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") { qt_sql """ select * from local( - "file_path" = "${dataFilePath}/comp.orc", + "file_path" = "${outFilePath}/comp.orc", "backend_id" = "${be_id}", "format" = "orc");""" qt_sql """ insert into ${table_name} select * from local ( - "file_path" = "${dataFilePath}/comp.orc", + "file_path" = "${outFilePath}/comp.orc", "backend_id" = "${be_id}", "format" = "orc");""" @@ -60,13 +72,13 @@ suite("test_local_tvf_with_complex_type_insertinto_doris", "p0") { qt_sql """ select * from local( - "file_path" = "${dataFilePath}/comp.parquet", + "file_path" = "${outFilePath}/comp.parquet", "backend_id" = "${be_id}", "format" = "parquet"); """ qt_sql """ insert into ${table_name} select * from local( - "file_path" = "${dataFilePath}/comp.parquet", + "file_path" = "${outFilePath}/comp.parquet", "backend_id" = "${be_id}", "format" = "parquet"); """ --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
