This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin-on-parquet-v2 in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin-on-parquet-v2 by this push: new 0f94e81 KYLIN-4898 Automated test 0f94e81 is described below commit 0f94e8165427eeb0cbae516f9516d4742f7a1205 Author: helenzeng0503 <linghui_helen_z...@163.com> AuthorDate: Fri Jan 29 16:38:09 2021 +0800 KYLIN-4898 Automated test --- .../data/auto_config/auto_config_no_distinct.json | 420 ++++++++++++++++++++ .../auto_config/auto_config_override_conf.json | 423 +++++++++++++++++++++ .../generic_desc_data/generic_desc_data_3x.json | 0 .../generic_desc_data/generic_desc_data_4x.json | 187 +++++++++ .../data/happy_path/happy_path.sql | 43 +++ .../env/default/python.properties | 2 +- .../features/specs/auto_config/auto_config.spec | 13 + .../features/specs/happy_path/happy_path.spec | 5 + .../features/specs/project_model/model.spec | 7 + .../features/specs/project_model/project.spec | 11 + .../features/step_impl/auto_config/auto_config.py | 196 ++++++++++ .../features/step_impl/before_suite.py | 8 + .../features/step_impl/happy_path/happy_path.py | 68 ++++ .../features/step_impl/project_model/model.py | 28 ++ .../features/step_impl/project_model/project.py | 46 +++ .../{kylin_instance.yml => kylin_host.yml} | 9 +- .../kylin_instances/kylin_instance.yml | 2 +- .../CI/kylin-system-testing/kylin_utils/equals.py | 2 +- build/CI/kylin-system-testing/kylin_utils/shell.py | 5 - build/CI/kylin-system-testing/kylin_utils/util.py | 11 +- 20 files changed, 1471 insertions(+), 15 deletions(-) diff --git a/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json b/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json new file mode 100644 index 0000000..7ef7f0c --- /dev/null +++ b/build/CI/kylin-system-testing/data/auto_config/auto_config_no_distinct.json @@ -0,0 +1,420 @@ +{ + "cube_desc_data": + { + "uuid": "788f16f4-94b2-f7f4-0a62-5dc8a02ceee8", + "last_modified": 0, + "version": "4.0.0.0", + "name": "kylin_sales_cube_no_distinct", + "is_draft": false, + "model_name": "kylin_sales_model", + "description": "", + "null_string": null, + "dimensions": [ + { + "name": "TRANS_ID", + "table": "KYLIN_SALES", + "column": "TRANS_ID", + "derived": null + }, + { + "name": "YEAR_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "YEAR_BEG_DT" + ] + }, + { + "name": "MONTH_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "MONTH_BEG_DT" + ] + }, + { + "name": "WEEK_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "WEEK_BEG_DT" + ] + }, + { + "name": "USER_DEFINED_FIELD1", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": null, + "derived": [ + "USER_DEFINED_FIELD1" + ] + }, + { + "name": "USER_DEFINED_FIELD3", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": null, + "derived": [ + "USER_DEFINED_FIELD3" + ] + }, + { + "name": "META_CATEG_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null + }, + { + "name": "CATEG_LVL2_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null + }, + { + "name": "CATEG_LVL3_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null + }, + { + "name": "LSTG_FORMAT_NAME", + "table": "KYLIN_SALES", + "column": "LSTG_FORMAT_NAME", + "derived": null + }, + { + "name": "SELLER_ID", + "table": "KYLIN_SALES", + "column": "SELLER_ID", + "derived": null + }, + { + "name": "BUYER_ID", + "table": "KYLIN_SALES", + "column": "BUYER_ID", + "derived": null + }, + { + "name": "ACCOUNT_BUYER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL", + "derived": null + }, + { + "name": "ACCOUNT_SELLER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL", + "derived": null + }, + { + "name": "BUYER_COUNTRY", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_COUNTRY", + "derived": null + }, + { + "name": "SELLER_COUNTRY", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_COUNTRY", + "derived": null + }, + { + "name": "BUYER_COUNTRY_NAME", + "table": "BUYER_COUNTRY", + "column": "NAME", + "derived": null + }, + { + "name": "SELLER_COUNTRY_NAME", + "table": "SELLER_COUNTRY", + "column": "NAME", + "derived": null + }, + { + "name": "OPS_USER_ID", + "table": "KYLIN_SALES", + "column": "OPS_USER_ID", + "derived": null + }, + { + "name": "OPS_REGION", + "table": "KYLIN_SALES", + "column": "OPS_REGION", + "derived": null + } + ], + "measures": [ + { + "name": "GMV_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "BUYER_LEVEL_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL" + }, + "returntype": "bigint" + } + }, + { + "name": "SELLER_LEVEL_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" + }, + "returntype": "bigint" + } + }, + { + "name": "TRANS_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "constant", + "value": "1" + }, + "returntype": "bigint" + } + }, + { + "name": "TOP_SELLER", + "function": { + "expression": "TOP_N", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE", + "next_parameter": { + "type": "column", + "value": "KYLIN_SALES.SELLER_ID" + } + }, + "returntype": "topn(100,4)", + "configuration": { + "topn.encoding.KYLIN_SALES.SELLER_ID": "dict", + "topn.encoding_version.KYLIN_SALES.SELLER_ID": "1" + } + } + }, + { + "name": "PRICE_PERCENTILE", + "function": { + "expression": "PERCENTILE_APPROX", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE" + }, + "returntype": "percentile(100)" + } + } + ], + "dictionaries": [], + "rowkey": { + "rowkey_columns": [ + { + "column": "KYLIN_SALES.BUYER_ID", + "encoding": "integer:4", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.SELLER_ID", + "encoding": "integer:4", + "isShardBy": true + }, + { + "column": "KYLIN_SALES.TRANS_ID", + "encoding": "integer:4", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.PART_DT", + "encoding": "date", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LEAF_CATEG_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_COUNTRY.NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_COUNTRY.NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LSTG_FORMAT_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LSTG_SITE_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.OPS_USER_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.OPS_REGION", + "encoding": "dict", + "isShardBy": false + } + ] + }, + "hbase_mapping": { + "column_family": [ + { + "name": "F1", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "GMV_SUM", + "BUYER_LEVEL_SUM", + "SELLER_LEVEL_SUM", + "TRANS_CNT" + ] + } + ] + }, + { + "name": "F2", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "TOP_SELLER", + "PRICE_PERCENTILE" + ] + } + ] + } + ] + }, + "aggregation_groups": [ + { + "includes": [ + "KYLIN_SALES.PART_DT", + "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "KYLIN_SALES.LEAF_CATEG_ID", + "KYLIN_SALES.LSTG_FORMAT_NAME", + "KYLIN_SALES.LSTG_SITE_ID", + "KYLIN_SALES.OPS_USER_ID", + "KYLIN_SALES.OPS_REGION", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME", + "SELLER_COUNTRY.NAME" + ], + "select_rule": { + "hierarchy_dims": [ + [ + "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "KYLIN_SALES.LEAF_CATEG_ID" + ] + ], + "mandatory_dims": [ + "KYLIN_SALES.PART_DT" + ], + "joint_dims": [ + [ + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ], + [ + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" + ], + [ + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" + ], + [ + "KYLIN_SALES.LSTG_FORMAT_NAME", + "KYLIN_SALES.LSTG_SITE_ID" + ], + [ + "KYLIN_SALES.OPS_USER_ID", + "KYLIN_SALES.OPS_REGION" + ] + ] + } + } + ], + "signature": "tADjcemRs3OW+O5Qdhf0gQ==", + "notify_list": [], + "status_need_notify": [], + "partition_date_start": 1325376000000, + "partition_date_end": 3153600000000, + "auto_merge_time_ranges": [], + "volatile_range": 0, + "retention_range": 0, + "engine_type": 6, + "storage_type": 4, + "override_kylin_properties": { + "kylin.cube.aggrgroup.is-mandatory-only-valid": "true", + "kylin.engine.spark.rdd-partition-cut-mb": "500" + }, + "cuboid_black_list": [], + "parent_forward": 3, + "mandatory_dimension_set_list": [], + "snapshot_table_desc_list": [] + } +} \ No newline at end of file diff --git a/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json b/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json new file mode 100644 index 0000000..188c140 --- /dev/null +++ b/build/CI/kylin-system-testing/data/auto_config/auto_config_override_conf.json @@ -0,0 +1,423 @@ +{ + "cube_desc_data": + { + "uuid": "788f16f4-94b2-f7f4-0a62-5dc8a02ceee8", + "last_modified": 0, + "version": "4.0.0.0", + "name": "kylin_sales_cube_override_conf", + "is_draft": false, + "model_name": "kylin_sales_model", + "description": "", + "null_string": null, + "dimensions": [ + { + "name": "TRANS_ID", + "table": "KYLIN_SALES", + "column": "TRANS_ID", + "derived": null + }, + { + "name": "YEAR_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "YEAR_BEG_DT" + ] + }, + { + "name": "MONTH_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "MONTH_BEG_DT" + ] + }, + { + "name": "WEEK_BEG_DT", + "table": "KYLIN_CAL_DT", + "column": null, + "derived": [ + "WEEK_BEG_DT" + ] + }, + { + "name": "USER_DEFINED_FIELD1", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": null, + "derived": [ + "USER_DEFINED_FIELD1" + ] + }, + { + "name": "USER_DEFINED_FIELD3", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": null, + "derived": [ + "USER_DEFINED_FIELD3" + ] + }, + { + "name": "META_CATEG_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "META_CATEG_NAME", + "derived": null + }, + { + "name": "CATEG_LVL2_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "CATEG_LVL2_NAME", + "derived": null + }, + { + "name": "CATEG_LVL3_NAME", + "table": "KYLIN_CATEGORY_GROUPINGS", + "column": "CATEG_LVL3_NAME", + "derived": null + }, + { + "name": "LSTG_FORMAT_NAME", + "table": "KYLIN_SALES", + "column": "LSTG_FORMAT_NAME", + "derived": null + }, + { + "name": "SELLER_ID", + "table": "KYLIN_SALES", + "column": "SELLER_ID", + "derived": null + }, + { + "name": "BUYER_ID", + "table": "KYLIN_SALES", + "column": "BUYER_ID", + "derived": null + }, + { + "name": "ACCOUNT_BUYER_LEVEL", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_BUYER_LEVEL", + "derived": null + }, + { + "name": "ACCOUNT_SELLER_LEVEL", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_SELLER_LEVEL", + "derived": null + }, + { + "name": "BUYER_COUNTRY", + "table": "BUYER_ACCOUNT", + "column": "ACCOUNT_COUNTRY", + "derived": null + }, + { + "name": "SELLER_COUNTRY", + "table": "SELLER_ACCOUNT", + "column": "ACCOUNT_COUNTRY", + "derived": null + }, + { + "name": "BUYER_COUNTRY_NAME", + "table": "BUYER_COUNTRY", + "column": "NAME", + "derived": null + }, + { + "name": "SELLER_COUNTRY_NAME", + "table": "SELLER_COUNTRY", + "column": "NAME", + "derived": null + }, + { + "name": "OPS_USER_ID", + "table": "KYLIN_SALES", + "column": "OPS_USER_ID", + "derived": null + }, + { + "name": "OPS_REGION", + "table": "KYLIN_SALES", + "column": "OPS_REGION", + "derived": null + } + ], + "measures": [ + { + "name": "GMV_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE" + }, + "returntype": "decimal(19,4)" + } + }, + { + "name": "BUYER_LEVEL_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL" + }, + "returntype": "bigint" + } + }, + { + "name": "SELLER_LEVEL_SUM", + "function": { + "expression": "SUM", + "parameter": { + "type": "column", + "value": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" + }, + "returntype": "bigint" + } + }, + { + "name": "TRANS_CNT", + "function": { + "expression": "COUNT", + "parameter": { + "type": "constant", + "value": "1" + }, + "returntype": "bigint" + } + }, + { + "name": "TOP_SELLER", + "function": { + "expression": "TOP_N", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE", + "next_parameter": { + "type": "column", + "value": "KYLIN_SALES.SELLER_ID" + } + }, + "returntype": "topn(100,4)", + "configuration": { + "topn.encoding.KYLIN_SALES.SELLER_ID": "dict", + "topn.encoding_version.KYLIN_SALES.SELLER_ID": "1" + } + } + }, + { + "name": "PRICE_PERCENTILE", + "function": { + "expression": "PERCENTILE_APPROX", + "parameter": { + "type": "column", + "value": "KYLIN_SALES.PRICE" + }, + "returntype": "percentile(100)" + } + } + ], + "dictionaries": [], + "rowkey": { + "rowkey_columns": [ + { + "column": "KYLIN_SALES.BUYER_ID", + "encoding": "integer:4", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.SELLER_ID", + "encoding": "integer:4", + "isShardBy": true + }, + { + "column": "KYLIN_SALES.TRANS_ID", + "encoding": "integer:4", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.PART_DT", + "encoding": "date", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LEAF_CATEG_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "BUYER_COUNTRY.NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "SELLER_COUNTRY.NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LSTG_FORMAT_NAME", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.LSTG_SITE_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.OPS_USER_ID", + "encoding": "dict", + "isShardBy": false + }, + { + "column": "KYLIN_SALES.OPS_REGION", + "encoding": "dict", + "isShardBy": false + } + ] + }, + "hbase_mapping": { + "column_family": [ + { + "name": "F1", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "GMV_SUM", + "BUYER_LEVEL_SUM", + "SELLER_LEVEL_SUM", + "TRANS_CNT" + ] + } + ] + }, + { + "name": "F2", + "columns": [ + { + "qualifier": "M", + "measure_refs": [ + "TOP_SELLER", + "PRICE_PERCENTILE" + ] + } + ] + } + ] + }, + "aggregation_groups": [ + { + "includes": [ + "KYLIN_SALES.PART_DT", + "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "KYLIN_SALES.LEAF_CATEG_ID", + "KYLIN_SALES.LSTG_FORMAT_NAME", + "KYLIN_SALES.LSTG_SITE_ID", + "KYLIN_SALES.OPS_USER_ID", + "KYLIN_SALES.OPS_REGION", + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL", + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME", + "SELLER_COUNTRY.NAME" + ], + "select_rule": { + "hierarchy_dims": [ + [ + "KYLIN_CATEGORY_GROUPINGS.META_CATEG_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL2_NAME", + "KYLIN_CATEGORY_GROUPINGS.CATEG_LVL3_NAME", + "KYLIN_SALES.LEAF_CATEG_ID" + ] + ], + "mandatory_dims": [ + "KYLIN_SALES.PART_DT" + ], + "joint_dims": [ + [ + "BUYER_ACCOUNT.ACCOUNT_COUNTRY", + "BUYER_COUNTRY.NAME" + ], + [ + "SELLER_ACCOUNT.ACCOUNT_COUNTRY", + "SELLER_COUNTRY.NAME" + ], + [ + "BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL", + "SELLER_ACCOUNT.ACCOUNT_SELLER_LEVEL" + ], + [ + "KYLIN_SALES.LSTG_FORMAT_NAME", + "KYLIN_SALES.LSTG_SITE_ID" + ], + [ + "KYLIN_SALES.OPS_USER_ID", + "KYLIN_SALES.OPS_REGION" + ] + ] + } + } + ], + "signature": "tADjcemRs3OW+O5Qdhf0gQ==", + "notify_list": [], + "status_need_notify": [], + "partition_date_start": 1325376000000, + "partition_date_end": 3153600000000, + "auto_merge_time_ranges": [], + "volatile_range": 0, + "retention_range": 0, + "engine_type": 6, + "storage_type": 4, + "override_kylin_properties": { + "kylin.engine.spark-conf.spark.executor.memory": "2G", + "kylin.engine.spark-conf.spark.executor.cores": "2", + "kylin.engine.spark-conf.spark.executor.memoryOverhead": "256M", + "kylin.engine.spark-conf.spark.executor.instances": "3", + "kylin.engine.spark-conf.spark.sql.shuffle.partitions": "3" + }, + "cuboid_black_list": [], + "parent_forward": 3, + "mandatory_dimension_set_list": [], + "snapshot_table_desc_list": [] + } +} \ No newline at end of file diff --git a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json similarity index 100% rename from build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_3x.json rename to build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_3x.json diff --git a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json similarity index 81% rename from build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json rename to build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json index f322325..abecc0b 100644 --- a/build/CI/kylin-system-testing/meta_data/generic_desc_data/generic_desc_data_4x.json +++ b/build/CI/kylin-system-testing/data/generic_desc_data/generic_desc_data_4x.json @@ -192,6 +192,193 @@ "capacity": "MEDIUM", "projectName": "generic_test_project" }, + "snowflake_left_incre_model": + { + "uuid": "c465bc68-46fa-cc0b-254d-58383c01557a", + "last_modified": 0, + "version": "4.0.0.0", + "name": "snowflake_left_incre_model", + "owner": "ADMIN", + "is_draft": false, + "description": "Snowflakes on top of KylinSales.", + "fact_table": "DEFAULT.KYLIN_SALES", + "lookups": [ + { + "table": "DEFAULT.KYLIN_CAL_DT", + "kind": "LOOKUP", + "alias": "KYLIN_CAL_DT", + "join": { + "type": "left", + "primary_key": [ + "KYLIN_CAL_DT.CAL_DT" + ], + "foreign_key": [ + "KYLIN_SALES.PART_DT" + ] + } + }, + { + "table": "DEFAULT.KYLIN_CATEGORY_GROUPINGS", + "kind": "LOOKUP", + "alias": "KYLIN_CATEGORY_GROUPINGS", + "join": { + "type": "left", + "primary_key": [ + "KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID", + "KYLIN_CATEGORY_GROUPINGS.SITE_ID" + ], + "foreign_key": [ + "KYLIN_SALES.LEAF_CATEG_ID", + "KYLIN_SALES.LSTG_SITE_ID" + ] + } + }, + { + "table": "DEFAULT.KYLIN_ACCOUNT", + "kind": "LOOKUP", + "alias": "BUYER_ACCOUNT", + "join": { + "type": "left", + "primary_key": [ + "BUYER_ACCOUNT.ACCOUNT_ID" + ], + "foreign_key": [ + "KYLIN_SALES.BUYER_ID" + ] + } + }, + { + "table": "DEFAULT.KYLIN_ACCOUNT", + "kind": "LOOKUP", + "alias": "SELLER_ACCOUNT", + "join": { + "type": "left", + "primary_key": [ + "SELLER_ACCOUNT.ACCOUNT_ID" + ], + "foreign_key": [ + "KYLIN_SALES.SELLER_ID" + ] + } + }, + { + "table": "DEFAULT.KYLIN_COUNTRY", + "kind": "LOOKUP", + "alias": "BUYER_COUNTRY", + "join": { + "type": "left", + "primary_key": [ + "BUYER_COUNTRY.COUNTRY" + ], + "foreign_key": [ + "BUYER_ACCOUNT.ACCOUNT_COUNTRY" + ] + } + }, + { + "table": "DEFAULT.KYLIN_COUNTRY", + "kind": "LOOKUP", + "alias": "SELLER_COUNTRY", + "join": { + "type": "left", + "primary_key": [ + "SELLER_COUNTRY.COUNTRY" + ], + "foreign_key": [ + "SELLER_ACCOUNT.ACCOUNT_COUNTRY" + ] + } + } + ], + "dimensions": [ + { + "table": "KYLIN_SALES", + "columns": [ + "TRANS_ID", + "SELLER_ID", + "BUYER_ID", + "PART_DT", + "LEAF_CATEG_ID", + "LSTG_FORMAT_NAME", + "LSTG_SITE_ID", + "OPS_USER_ID", + "OPS_REGION" + ] + }, + { + "table": "KYLIN_CAL_DT", + "columns": [ + "CAL_DT", + "WEEK_BEG_DT", + "MONTH_BEG_DT", + "YEAR_BEG_DT" + ] + }, + { + "table": "KYLIN_CATEGORY_GROUPINGS", + "columns": [ + "USER_DEFINED_FIELD1", + "USER_DEFINED_FIELD3", + "META_CATEG_NAME", + "CATEG_LVL2_NAME", + "CATEG_LVL3_NAME", + "LEAF_CATEG_ID", + "SITE_ID" + ] + }, + { + "table": "BUYER_ACCOUNT", + "columns": [ + "ACCOUNT_ID", + "ACCOUNT_BUYER_LEVEL", + "ACCOUNT_SELLER_LEVEL", + "ACCOUNT_COUNTRY", + "ACCOUNT_CONTACT" + ] + }, + { + "table": "SELLER_ACCOUNT", + "columns": [ + "ACCOUNT_ID", + "ACCOUNT_BUYER_LEVEL", + "ACCOUNT_SELLER_LEVEL", + "ACCOUNT_COUNTRY", + "ACCOUNT_CONTACT" + ] + }, + { + "table": "BUYER_COUNTRY", + "columns": [ + "COUNTRY", + "NAME" + ] + }, + { + "table": "SELLER_COUNTRY", + "columns": [ + "COUNTRY", + "NAME" + ] + } + ], + "metrics": [ + "KYLIN_SALES.PRICE", + "KYLIN_SALES.ITEM_COUNT", + "KYLIN_SALES.ITEM_ID" + ], + "filter_condition": "KYLIN_SALES.PRICE > 25.0", + "partition_desc": { + "partition_date_column": "KYLIN_SALES.PART_DT", + "partition_time_column": null, + "partition_date_start": 1325376000000, + "partition_date_format": "yyyy-MM-dd", + "partition_time_format": "HH:mm:ss", + "partition_type": "APPEND", + "partition_condition_builder": "org.apache.kylin.metadata.model.PartitionDesc$DefaultPartitionConditionBuilder" + }, + "capacity": "MEDIUM", + "projectName": "generic_test_project" + }, "cube_desc_data": { "uuid": "86da18cb-6eb9-bf43-5c69-aae88a7b5915", diff --git a/build/CI/kylin-system-testing/data/happy_path/happy_path.sql b/build/CI/kylin-system-testing/data/happy_path/happy_path.sql new file mode 100644 index 0000000..4fd469b --- /dev/null +++ b/build/CI/kylin-system-testing/data/happy_path/happy_path.sql @@ -0,0 +1,43 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +SELECT +KYLIN_SALES.PART_DT, +KYLIN_CAL_DT.YEAR_BEG_DT, +SUM(BUYER_ACCOUNT.ACCOUNT_BUYER_LEVEL) as BUYER_LEVEL_SUM, +COUNT(DISTINCT KYLIN_SALES.SELLER_ID) as SELLER_CNT_HLL, +SUM(KYLIN_SALES.PRICE) as GMV, +PERCENTILE(KYLIN_SALES.PRICE, 0.5) as PRICE_PERCENTILE, +COUNT(DISTINCT KYLIN_SALES.ITEM_ID) as ITEM_COUNT +FROM KYLIN_SALES +INNER JOIN KYLIN_CAL_DT as KYLIN_CAL_DT +ON KYLIN_SALES.PART_DT = KYLIN_CAL_DT.CAL_DT +INNER JOIN KYLIN_CATEGORY_GROUPINGS +ON KYLIN_SALES.LEAF_CATEG_ID = KYLIN_CATEGORY_GROUPINGS.LEAF_CATEG_ID AND KYLIN_SALES.LSTG_SITE_ID = KYLIN_CATEGORY_GROUPINGS.SITE_ID +INNER JOIN KYLIN_ACCOUNT as BUYER_ACCOUNT +ON KYLIN_SALES.BUYER_ID = BUYER_ACCOUNT.ACCOUNT_ID +INNER JOIN KYLIN_ACCOUNT as SELLER_ACCOUNT +ON KYLIN_SALES.SELLER_ID = SELLER_ACCOUNT.ACCOUNT_ID +INNER JOIN KYLIN_COUNTRY as BUYER_COUNTRY +ON BUYER_ACCOUNT.ACCOUNT_COUNTRY = BUYER_COUNTRY.COUNTRY +INNER JOIN KYLIN_COUNTRY as SELLER_COUNTRY +ON SELLER_ACCOUNT.ACCOUNT_COUNTRY = SELLER_COUNTRY.COUNTRY +WHERE PART_DT >= '2012-12-25' and PART_DT < '2013-01-05' and SELLER_COUNTRY.COUNTRY in ('CN') +GROUP BY KYLIN_SALES.PART_DT, KYLIN_CAL_DT.YEAR_BEG_DT +ORDER BY KYLIN_SALES.PART_DT, KYLIN_CAL_DT.YEAR_BEG_DT; + diff --git a/build/CI/kylin-system-testing/env/default/python.properties b/build/CI/kylin-system-testing/env/default/python.properties index 4dc60a9..1432d27 100644 --- a/build/CI/kylin-system-testing/env/default/python.properties +++ b/build/CI/kylin-system-testing/env/default/python.properties @@ -15,7 +15,7 @@ # limitations under the License. # -GAUGE_PYTHON_COMMAND = python3 +GAUGE_PYTHON_COMMAND = python # Comma seperated list of dirs. path should be relative to project root. STEP_IMPL_DIR = features/step_impl diff --git a/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec b/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec new file mode 100644 index 0000000..a47e57a --- /dev/null +++ b/build/CI/kylin-system-testing/features/specs/auto_config/auto_config.spec @@ -0,0 +1,13 @@ +# Automically setting spark configurations +Tags: auto_config, 4.x + +## use default configuration +* use default configuration on cube "kylin_sales_cube" +## auto configuration with user-defined parameters +* auto configuration with user-defined parameters on cube "kylin_sales_cube" +## auto configuration with user-defined parameters +* auto configuration with user-defined parameters on "kylin_sales_cube" and "kylin_sales_cube_no_distinct" +## auto_config and override on cube level +* auto_config and override on cube level on cube "kylin_sales_cube_override_conf" +//## auto configuration off, override on cube level +//* auto configuration off, override on cube level \ No newline at end of file diff --git a/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec b/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec new file mode 100644 index 0000000..8526704 --- /dev/null +++ b/build/CI/kylin-system-testing/features/specs/happy_path/happy_path.spec @@ -0,0 +1,5 @@ +# Automated test for Happy Path +Tags: happy_path, 4.x + +## automated happy path +* automated happy path, check query using "meta_data/happy_path/happy_path.sql" \ No newline at end of file diff --git a/build/CI/kylin-system-testing/features/specs/project_model/model.spec b/build/CI/kylin-system-testing/features/specs/project_model/model.spec new file mode 100644 index 0000000..507230c --- /dev/null +++ b/build/CI/kylin-system-testing/features/specs/project_model/model.spec @@ -0,0 +1,7 @@ +# model management +Tags: model, 4.x + +## model clone +* in project "generic_test_project", clone model "generic_test_model" and name it "model_name_clone" +## model clone duplicated +* again, in project "generic_test_project", clone model "generic_test_model" and name it "model_name_clone" \ No newline at end of file diff --git a/build/CI/kylin-system-testing/features/specs/project_model/project.spec b/build/CI/kylin-system-testing/features/specs/project_model/project.spec new file mode 100644 index 0000000..e094a94 --- /dev/null +++ b/build/CI/kylin-system-testing/features/specs/project_model/project.spec @@ -0,0 +1,11 @@ +# project management +Tags: project, 4.x + +## create duplicate project +* create a project "test_project_description" with description "this is a test project" and check that duplicate name is not allowed +## look up the project list +* check that "test_project_description" is on the list +## update project description +* update the project "test_project_description" and edit the description to be "this is a test project V2" +## delete project and check +* delete project "test_project_description" and check that it's not on the list diff --git a/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py b/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py new file mode 100644 index 0000000..30ad76f --- /dev/null +++ b/build/CI/kylin-system-testing/features/step_impl/auto_config/auto_config.py @@ -0,0 +1,196 @@ +from getgauge.python import step +import os +import json +import pytest +from kylin_utils import util +from kylin_utils import shell +import re +import time +from getgauge.python import Messages + +global client +client = util.setup_instance('kylin_instance.yml') + + +@step("use default configuration on cube <cube_name>") +def default_config(cube_name): + sh_config(set_auto="True", executor_instance="5", instance_strategy="100,2,500,3,1000,4") + + time.sleep(60) + + resp = client.build_segment(start_time="1325376000000", + end_time="1388534400000", + cube_name=cube_name) + job_id = resp['uuid'] + step_id = job_id + "-01" + client.await_job(job_id) + resp = client.get_step_output(job_id=job_id, step_id=step_id) + output = resp.get('cmd_output') + + check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="5", partitionsSize="2") + + client.disable_cube(cube_name) + + time.sleep(30) + client.purge_cube(cube_name) + + time.sleep(30) + + +@step("auto configuration with user-defined parameters on cube <cube_name>") +def user_def_config(cube_name): + sh_config(set_auto="True", executor_instance="1", instance_strategy="100,1,500,2,1000,3") + + time.sleep(60) + + resp = client.build_segment(start_time="1325376000000", + end_time="1388534400000", + cube_name=cube_name) + + job_id = resp['uuid'] + step_id = job_id + "-01" + client.await_job(job_id) + resp = client.get_step_output(job_id=job_id, step_id=step_id) + output = resp.get('cmd_output') + + check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="1", partitionsSize="2") + + client.disable_cube(cube_name) + + time.sleep(30) + client.purge_cube(cube_name) + + time.sleep(30) + + +@step("auto configuration with user-defined parameters on <cube_name> and <cube_no_distinct>") +def user_defined_no_dist(cube_name, cube_no_distinct): + sh_config(set_auto="True", executor_instance="2", instance_strategy="100,4,500,6,1000,10") + + time.sleep(60) + + resp = client.build_segment(start_time="1325376000000", + end_time="1388534400000", + cube_name=cube_name) + + job_id = resp['uuid'] + step_id = job_id + "-01" + client.await_job(job_id) + resp = client.get_step_output(job_id=job_id, step_id=step_id) + output = resp.get('cmd_output') + + check_log(output=output, memorySize="4GB", coresSize="5", memoryOverheadSize="1GB", instancesSize="2", partitionsSize="2") + + client.disable_cube(cube_name) + + time.sleep(30) + client.purge_cube(cube_name) + + time.sleep(30) + + with open(os.path.join('meta_data/auto_config', 'auto_config_no_distinct.json'), 'r') as f: + cube_desc_data = json.load(f)['cube_desc_data'] + + client.create_cube('learn_kylin', cube_no_distinct, cube_desc_data=cube_desc_data) + + resp = client.build_segment(start_time="1325376000000", + end_time="1388534400000", + cube_name=cube_no_distinct) + + job_id = resp['uuid'] + step_id = job_id + "-01" + client.await_job(job_id) + resp = client.get_step_output(job_id=job_id, step_id=step_id) + output = resp.get('cmd_output') + + check_log(output=output, memorySize="1GB", coresSize="1", memoryOverheadSize="512MB", instancesSize="2", partitionsSize="2") + + client.disable_cube(cube_no_distinct) + + time.sleep(30) + client.delete_cube(cube_no_distinct) + + time.sleep(30) + + +@step("auto_config and override on cube level on cube <cube_override_conf>") +def override_on_cube(cube_override_conf): + sh_config(set_auto="True", executor_instance="5", instance_strategy="100,2,500,3,1000,4") + + time.sleep(60) + + with open(os.path.join('meta_data/auto_config', 'auto_config_override_conf.json'), 'r') as f: + cube_desc_data = json.load(f)['cube_desc_data'] + + client.create_cube('learn_kylin', cube_override_conf, cube_desc_data=cube_desc_data) + + resp = client.build_segment(start_time="1325376000000", + end_time="1388534400000", + cube_name=cube_override_conf) + + job_id = resp['uuid'] + step_id = job_id + "-01" + client.await_job(job_id) + resp = client.get_step_output(job_id=job_id, step_id=step_id) + output = resp.get('cmd_output') + + memory = re.findall("Override user-defined spark conf, set spark.executor.memory.*", output) + cores = re.findall("Override user-defined spark conf, set spark.executor.cores.*", output) + memoryOverhead = re.findall("Override user-defined spark conf, set spark.executor.memoryOverhead.*", output) + instances = re.findall("Override user-defined spark conf, set spark.executor.instances.*", output) + partitions = re.findall("Override user-defined spark conf, set spark.sql.shuffle.partitions.*", output) + assert memory[0] == "Override user-defined spark conf, set spark.executor.memory=2G.", \ + Messages.write_message("expected Override user-defined spark conf, set spark.executor.memory=2G; actually " + memory[0]) + assert cores[0] == "Override user-defined spark conf, set spark.executor.cores=2.", \ + Messages.write_message("expected Override user-defined spark conf, set spark.executor.cores=2; actually " + cores[0]) + assert memoryOverhead[0] == "Override user-defined spark conf, set spark.executor.memoryOverhead=256M.", \ + Messages.write_message("expected Override user-defined spark conf, set spark.executor.memoryOverhead=256M; actually " + memoryOverhead[0]) + assert instances[0] == "Override user-defined spark conf, set spark.executor.instances=3.",\ + Messages.write_message("expected Override user-defined spark conf, set spark.executor.instances=3; actually" + instances[0]) + assert partitions[0] == "Override user-defined spark conf, set spark.sql.shuffle.partitions=3.", \ + Messages.write_message("expected Override user-defined spark conf, set spark.sql.shuffle.partitions=3; actually " + partitions[0]) + + client.disable_cube(cube_override_conf) + + time.sleep(30) + client.delete_cube(cube_override_conf) + + time.sleep(30) + + +def sh_config(set_auto, executor_instance, instance_strategy): + sh = util.ssh_shell() + + sh_command = "cd $KYLIN_HOME/conf " \ + "&& sed -i -r '/kylin.spark-conf.auto.prior/d' kylin.properties" \ + "&& sed -i -r '/kylin.engine.base-executor-instance/d' kylin.properties" \ + "&& sed -i -r '/kylin.engine.executor-instance-strategy/d' kylin.properties" \ + "&& sed -i '$akylin.spark-conf.auto.prior={set_auto}' kylin.properties" \ + "&& sed -i '$akylin.engine.base-executor-instance={executor_instance}' kylin.properties" \ + "&& sed -i '$akylin.engine.executor-instance-strategy={instance_strategy}' kylin.properties".format( + set_auto=set_auto, executor_instance=executor_instance, instance_strategy=instance_strategy) + + resp = sh.command(sh_command) + + resp = sh.command("cd $KYLIN_HOME/bin" + "&& sh kylin.sh stop" + "&& sh kylin.sh start") + + +def check_log(output, memorySize, coresSize, memoryOverheadSize, instancesSize, partitionsSize): + memory = re.findall("Auto set spark conf: spark.executor.memory = .*", output) + cores = re.findall("Auto set spark conf: spark.executor.cores = .*", output) + memoryOverhead = re.findall("Auto set spark conf: spark.executor.memoryOverhead = .*", output) + instances = re.findall("Auto set spark conf: spark.executor.instances = .*", output) + partitions = re.findall("Auto set spark conf: spark.sql.shuffle.partitions = .*", output) + assert memory[0] == "Auto set spark conf: spark.executor.memory = {memory}.".format(memory = memorySize), \ + Messages.write_message("expected "+"Auto set spark conf: spark.executor.memory = {memory}.".format(memory = memorySize)+ ", actually " +memory[0]) + assert cores[0] == "Auto set spark conf: spark.executor.cores = {cores}.".format(cores = coresSize), \ + Messages.write_message("expected "+"Auto set spark conf: spark.executor.cores = {cores}.".format(cores = coresSize)+", actually " +cores[0]) + assert memoryOverhead[0] == "Auto set spark conf: spark.executor.memoryOverhead = {memoryOverhead}.".format(memoryOverhead = memoryOverheadSize), \ + Messages.write_message("expected "+"Auto set spark conf: spark.executor.memoryOverhead = {memoryOverhead}.".format(memoryOverhead = memoryOverheadSize)+", actually " +memoryOverhead[0]) + assert instances[0] == "Auto set spark conf: spark.executor.instances = {instances}.".format(instances = instancesSize), \ + Messages.write_message("expected "+"Auto set spark conf: spark.executor.instances = {instances}.".format(instances = instancesSize)+", actually " +instances[0]) + assert partitions[0] == "Auto set spark conf: spark.sql.shuffle.partitions = {partitions}.".format(partitions = partitionsSize), \ + Messages.write_message("expected "+"Auto set spark conf: spark.sql.shuffle.partitions = {partitions}.".format(partitions = partitionsSize)+", actually " +partitions[0]) + diff --git a/build/CI/kylin-system-testing/features/step_impl/before_suite.py b/build/CI/kylin-system-testing/features/step_impl/before_suite.py index 00fca21..e0843a2 100644 --- a/build/CI/kylin-system-testing/features/step_impl/before_suite.py +++ b/build/CI/kylin-system-testing/features/step_impl/before_suite.py @@ -43,6 +43,8 @@ def create_generic_model_and_cube(): model_desc_data = data.get('model_desc_data') model_name = model_desc_data.get('name') + snowflake_left_incre_model = data.get('snowflake_left_incre_model') + left_model_name = snowflake_left_incre_model.get('name') if not util.if_model_exists(kylin_client=client, model_name=model_name, project=project_name): resp = client.create_model(project_name=project_name, @@ -50,6 +52,12 @@ def create_generic_model_and_cube(): model_desc_data=model_desc_data) assert json.loads(resp['modelDescData'])['name'] == model_name + if not util.if_model_exists(kylin_client=client, model_name=left_model_name, project=project_name): + resp = client.create_model(project_name=project_name, + model_name=left_model_name, + model_desc_data=snowflake_left_incre_model) + assert json.loads(resp['modelDescData'])['name'] == left_model_name + cube_desc_data = data.get('cube_desc_data') cube_name = cube_desc_data.get('name') if not util.if_cube_exists(kylin_client=client, cube_name=cube_name, project=project_name): diff --git a/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py b/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py new file mode 100644 index 0000000..1d59e05 --- /dev/null +++ b/build/CI/kylin-system-testing/features/step_impl/happy_path/happy_path.py @@ -0,0 +1,68 @@ +from getgauge.python import step +import os +import json +import pytest +from kylin_utils import util +from kylin_utils import equals +from kylin_utils import shell +import re +import time + + +@step("automated happy path, check query using <sql_file>") +def happy_path(sql_file): + global client + client = util.setup_instance('kylin_instance.yml') + + resp1 = client.build_segment(start_time="1325376000000", + end_time="1356998400000", + cube_name="kylin_sales_cube") + + resp2 = client.build_segment(start_time="1356998400000", + end_time="1388620800000", + cube_name="kylin_sales_cube") + + job_id1 = resp1['uuid'] + job_id2 = resp2['uuid'] + client.await_job(job_id1) + client.await_job(job_id2) + + resp = client.merge_segment(cube_name="kylin_sales_cube", + start_time="1325376000000", + end_time="1388620800000") + + job_id = resp['uuid'] + client.await_job(job_id) + + resp = client.refresh_segment(cube_name="kylin_sales_cube", + start_time="1325376000000", + end_time="1388620800000") + job_id = resp['uuid'] + client.await_job(job_id) + + with open(sql_file, 'r', encoding='utf8') as sql: + sql = sql.read() + + equals.compare_sql_result(sql=sql, project='learn_kylin', kylin_client=client, expected_result=None) + + resp = client.disable_cube(cube_name="kylin_sales_cube") + assert resp['status'] == 'DISABLED' + + time.sleep(10) + + client.purge_cube(cube_name="kylin_sales_cube") + time.sleep(30) + + resp = client.get_cube_instance(cube_name="kylin_sales_cube") + + assert len(resp['segments']) == 0 + + + + + + + + + + diff --git a/build/CI/kylin-system-testing/features/step_impl/project_model/model.py b/build/CI/kylin-system-testing/features/step_impl/project_model/model.py new file mode 100644 index 0000000..f159d93 --- /dev/null +++ b/build/CI/kylin-system-testing/features/step_impl/project_model/model.py @@ -0,0 +1,28 @@ +from getgauge.python import step +import os +import json +import pytest +from kylin_utils import util + +global client +client = util.setup_instance('kylin_instance.yml') + + +@step("in project <project_name>, clone model <model_name> and name it <clone_name>") +def check_model_clone(project_name, model_name, clone_name): + clone = client.clone_model(project_name, model_name, clone_name) + assert util.if_model_exists(kylin_client=client, model_name=clone_name, project=project_name) == 1 + model_desc = client.list_model_desc(project_name, model_name) + clone_model_desc = client.list_model_desc(project_name, clone_name) + check_list = ['fact_table', 'lookups', 'dimensions', 'metrics', 'filter_condition', 'partition_desc'] + for i in range(len(check_list)): + assert model_desc[0][check_list[i]] == clone_model_desc[0][check_list[i]] + + +@step("again, in project <project_name>, clone model <model_name> and name it <clone_name>") +def check_clone_duplicated(project_name, model_name, clone_name): + with pytest.raises(Exception, match=r'Model name .* is duplicated, could not be created.'): + clone = client.clone_model(project_name, model_name, clone_name) + + + diff --git a/build/CI/kylin-system-testing/features/step_impl/project_model/project.py b/build/CI/kylin-system-testing/features/step_impl/project_model/project.py new file mode 100644 index 0000000..61a088f --- /dev/null +++ b/build/CI/kylin-system-testing/features/step_impl/project_model/project.py @@ -0,0 +1,46 @@ +from getgauge.python import step +import os +import json +import pytest +from kylin_utils import util + +global client +client = util.setup_instance('kylin_instance.yml') + +@step("create a project <project_name> with description <project_description> and check that duplicate name is not allowed") +def create_duplicate_project(project_name, project_description): + client.create_project(project_name, description=project_description) + with pytest.raises(Exception, match=r'The project named .* already exists'): + client.create_project(project_name, description=project_description) + + +@step("check that <project_name> is on the list") +def check_project(project_name): + + assert util.if_project_exists(kylin_client=client, project=project_name) == 1 + + +@step("update the project <project_name> and edit the description to be <project_description>") +def update_project_description(project_name, project_description): + update = client.update_project(project_name, description=project_description) + resp = client.list_projects() + update = False + for i in range(len(resp)): + update = update | ((resp[i]['name'] == project_name) & (resp[i]['description'] == project_description)) + assert update is True + + +@step("delete project <project_name> and check that it's not on the list") +def check_delete_project(project_name): + delete = client.delete_project(project_name) + resp = client.list_projects() + exist = False + for i in range(len(resp)): + exist = exist | (resp[i]['name'] == project_name) + assert exist is False + + + + + + diff --git a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml b/build/CI/kylin-system-testing/kylin_instances/kylin_host.yml similarity index 88% copy from build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml copy to build/CI/kylin-system-testing/kylin_instances/kylin_host.yml index 5454a41..c75bbf5 100644 --- a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml +++ b/build/CI/kylin-system-testing/kylin_instances/kylin_host.yml @@ -14,9 +14,6 @@ # limitations under the License. --- -# All mode -- host: localhost - port: 7070 - version: 4.x - hadoop_platform: HDP2.4 - deploy_mode: ALL \ No newline at end of file +- host: kylin-all + username: root + password: hadoop \ No newline at end of file diff --git a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml index 5454a41..1aa1a08 100644 --- a/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml +++ b/build/CI/kylin-system-testing/kylin_instances/kylin_instance.yml @@ -15,7 +15,7 @@ --- # All mode -- host: localhost +- host: kylin-all port: 7070 version: 4.x hadoop_platform: HDP2.4 diff --git a/build/CI/kylin-system-testing/kylin_utils/equals.py b/build/CI/kylin-system-testing/kylin_utils/equals.py index 2cceff1..e352736 100644 --- a/build/CI/kylin-system-testing/kylin_utils/equals.py +++ b/build/CI/kylin-system-testing/kylin_utils/equals.py @@ -241,7 +241,7 @@ def compare_sql_result(sql, project, kylin_client, compare_level="data_set", cub kylin_resp = kylin_client.execute_query(cube_name=cube, project_name=project, sql=sql) - assert kylin_resp.get('isException') is False, 'Thown Exception when execute ' + sql + assert kylin_resp.get('isException') is False, 'Thrown Exception when execute ' + sql pushdown_resp = kylin_client.execute_query(project_name=pushdown_project, sql=sql) assert pushdown_resp.get('isException') is False diff --git a/build/CI/kylin-system-testing/kylin_utils/shell.py b/build/CI/kylin-system-testing/kylin_utils/shell.py index 72b734a..31af6af 100644 --- a/build/CI/kylin-system-testing/kylin_utils/shell.py +++ b/build/CI/kylin-system-testing/kylin_utils/shell.py @@ -135,8 +135,3 @@ def exec(script): # pylint: disable=redefined-builtin def shell(): return Bash() - - -if __name__ == '__main__': - sh = sshshell('10.1.3.94', username='root', password='hadoop') - print(sh.command('pwd')) \ No newline at end of file diff --git a/build/CI/kylin-system-testing/kylin_utils/util.py b/build/CI/kylin-system-testing/kylin_utils/util.py index f29e034..44d3e4a 100644 --- a/build/CI/kylin-system-testing/kylin_utils/util.py +++ b/build/CI/kylin-system-testing/kylin_utils/util.py @@ -18,7 +18,7 @@ from selenium import webdriver from yaml import load, loader import os - +from kylin_utils import shell from kylin_utils import kylin @@ -79,3 +79,12 @@ def if_model_exists(kylin_client, model_name, project): if len(resp) == 1: exists = 1 return exists + +def ssh_shell(config_file='kylin_host.yml'): + instances_file = os.path.join('kylin_instances/', config_file) + stream = open(instances_file, 'r') + for item in load(stream, Loader=loader.SafeLoader): + host = item['host'] + username = item['username'] + password = item['password'] + return shell.SSHShell(host=host, username=username, password=password)