This is an automated email from the ASF dual-hosted git repository. kxiao pushed a commit to branch branch-2.0 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 4213b19415accfb074c0fbd117deea2ac9e9ac70 Author: slothever <[email protected]> AuthorDate: Tue Oct 24 20:52:50 2023 +0800 [regression](multi-catalog) add aliyun dlf hive on oss and huawei obs test case (#25650) add aliyun dlf hive on oss and huawei obs test case now obs cases have some problem, will not fix this at this PR, just add comment. --- regression-test/conf/regression-conf.groovy | 4 ++ .../hive/test_cloud_accessible_obs.out | 7 +++ .../hive/test_cloud_accessible_oss.out | 10 +++ .../hive/test_cloud_accessible_obs.groovy | 47 ++++++++++++++ .../hive/test_cloud_accessible_oss.groovy | 73 ++++++++++++++++++++++ 5 files changed, 141 insertions(+) diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 9da502b8144..141dc57ed6b 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -151,9 +151,13 @@ extEsPort = 9200 extEsUser = "*******" extEsPassword = "***********" +enableObjStorageTest=false enableMaxComputeTest=false aliYunAk="***********" +dlfUid="***********" aliYunSk="***********" +hwYunAk="***********" +hwYunSk="***********" s3Endpoint = "cos.ap-hongkong.myqcloud.com" s3BucketName = "doris-build-hk-1308700295" diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out new file mode 100644 index 00000000000..2a935c23e0f --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_cloud_accessible_obs.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !hms_q1 -- +123 22 234 123.324235 sawer 234.1234 a23f 1234vb false 2023-04-23 21:23:34.123 2023-04-23 + +-- !hms_q2 -- +11625 62 234 2347.6236 awsali 546.2342 a23f 1234vb true 2023-04-22 21:21:34.123 2023-04-21 2023-04-24 +23621 23 234 345.12512356 aswwas 525.2352 a23f wsd false 2023-04-23 21:23:34.123 2023-04-23 2023-04-22 diff --git a/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out b/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out new file mode 100644 index 00000000000..97933cfc5d4 --- /dev/null +++ b/regression-test/data/external_table_p2/hive/test_cloud_accessible_oss.out @@ -0,0 +1,10 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !hms_q1 -- +123 22 234 123.324235 sawer 234.1234 a23f 1234vb false 2023-04-23 21:23:34.123 2023-04-23 + +-- !hms_q2 -- +11625 62 234 2347.6236 awsali 546.2342 a23f 1234vb true 2023-04-22 21:21:34.123 2023-04-21 2023-04-24 +23621 23 234 345.12512356 aswwas 525.2352 a23f wsd false 2023-04-23 21:23:34.123 2023-04-23 2023-04-22 + +-- !dlf_q1 -- +26 AAAAAAAAKBAAAAAA 1997-08-16 2000-08-15 site_4 2450594 2446944 Unknown William Carter 2 Also difficult women relax. Raw solutions exclude Often well-known police can hand acres. Healthy, sensible problems make more silent recordings Keith Williams 2 able 892 Cedar North Drive Suite 280 Midway Williamson County TN 31904 United States -5.00 0.07 diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy new file mode 100644 index 00000000000..b475088bd56 --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_obs.groovy @@ -0,0 +1,47 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cloud_accessible_obs", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableObjStorageTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String ak = context.config.otherConfigs.get("hwYunAk") + String sk = context.config.otherConfigs.get("hwYunSk") + String hms_catalog_name = "test_cloud_accessible_obs" + sql """drop catalog if exists ${hms_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} + PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}', + 'obs.endpoint' = 'obs.cn-north-4.myhuaweicloud.com', + 'obs.access_key' = '${ak}', + 'obs.secret_key' = '${sk}' + ); + """ + + logger.info("catalog " + hms_catalog_name + " created") + sql """switch ${hms_catalog_name};""" + logger.info("switched to catalog " + hms_catalog_name) + sql """ use cloud_accessible """ + qt_hms_q1 """ select * from types_obs order by hms_int """ + qt_hms_q2 """ select * from types_one_part_obs order by hms_int """ + + sql """drop catalog ${hms_catalog_name};""" + } +} diff --git a/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy new file mode 100644 index 00000000000..d7319a7ed08 --- /dev/null +++ b/regression-test/suites/external_table_p2/hive/test_cloud_accessible_oss.groovy @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_cloud_accessible_oss", "p2,external,hive,external_remote,external_remote_hive") { + String enabled = context.config.otherConfigs.get("enableObjStorageTest") + if (enabled != null && enabled.equalsIgnoreCase("true")) { + String extHiveHmsHost = context.config.otherConfigs.get("extHiveHmsHost") + String extHiveHmsPort = context.config.otherConfigs.get("extHiveHmsPort") + String ak = context.config.otherConfigs.get("aliYunAk") + String sk = context.config.otherConfigs.get("aliYunSk") + String hms_catalog_name = "test_cloud_accessible_oss" + sql """drop catalog if exists ${hms_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS ${hms_catalog_name} + PROPERTIES ( + 'type' = 'hms', + 'hive.metastore.uris' = 'thrift://${extHiveHmsHost}:${extHiveHmsPort}', + 'oss.endpoint' = 'oss-cn-beijing.aliyuncs.com', + 'oss.access_key' = '${ak}', + 'oss.secret_key' = '${sk}' + ); + """ + + logger.info("catalog " + hms_catalog_name + " created") + sql """switch ${hms_catalog_name};""" + logger.info("switched to catalog " + hms_catalog_name) + sql """ use cloud_accessible """ + qt_hms_q1 """ select * from types_oss order by hms_int """ + qt_hms_q2 """ select * from types_one_part_oss order by hms_int """ + + sql """drop catalog ${hms_catalog_name};""" + + // dlf case + String dlf_catalog_name = "test_cloud_accessible_dlf" + String dlf_uid = context.config.otherConfigs.get("dlfUid") + sql """drop catalog if exists ${dlf_catalog_name};""" + sql """ + CREATE CATALOG IF NOT EXISTS ${dlf_catalog_name} + PROPERTIES ( + "type"="hms", + "hive.metastore.type" = "dlf", + "dlf.endpoint" = "dlf.cn-beijing.aliyuncs.com", + "dlf.region" = "cn-beijing", + "dlf.proxy.mode" = "DLF_ONLY", + "dlf.uid" = "${dlf_uid}", + "dlf.access_key" = "${ak}", + "dlf.secret_key" = "${sk}", + "dlf.access.public" = "true" + ); + """ + logger.info("catalog " + dlf_catalog_name + " created") + sql """switch ${dlf_catalog_name};""" + logger.info("switched to catalog " + dlf_catalog_name) + sql """ use jz_datalake """ + qt_dlf_q1 """ select * from web_site where web_site_id='AAAAAAAAKBAAAAAA' order by web_site_sk,web_site_id limit 1; """ // test char,date,varchar,double,decimal + + sql """drop catalog ${dlf_catalog_name};""" + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
