KYLIN-1249 A client library to help automatic cube Signed-off-by: Hongbin Ma <mahong...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/5593d824 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/5593d824 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/5593d824 Branch: refs/heads/1.3.x Commit: 5593d8246e4378b3fdf8982c1fddfadb1bf146b7 Parents: 2b7ae84 Author: huang...@mininglamp.com <huang...@mininglamp.com> Authored: Wed Feb 24 23:17:56 2016 +0800 Committer: Hongbin Ma <mahong...@apache.org> Committed: Thu Mar 17 18:12:56 2016 +0800 ---------------------------------------------------------------------- tools/kylin_client_tool/client/__init__.py | 2 + .../client/client_interface.py | 226 ++++++++++++++ tools/kylin_client_tool/client/client_parse.py | 96 ++++++ tools/kylin_client_tool/cube_def.csv | 2 + tools/kylin_client_tool/cube_names.csv | 2 + tools/kylin_client_tool/jobs/__init__.py | 2 + tools/kylin_client_tool/jobs/admin.py | 66 ++++ tools/kylin_client_tool/jobs/build.py | 76 +++++ tools/kylin_client_tool/jobs/cube.py | 187 +++++++++++ tools/kylin_client_tool/kylin_client_tool.py | 5 + tools/kylin_client_tool/models/__init__.py | 2 + tools/kylin_client_tool/models/cube.py | 309 +++++++++++++++++++ tools/kylin_client_tool/models/dimension.py | 83 +++++ tools/kylin_client_tool/models/hbase.py | 100 ++++++ tools/kylin_client_tool/models/io/__init__.py | 2 + tools/kylin_client_tool/models/io/base.py | 65 ++++ tools/kylin_client_tool/models/io/cube.py | 90 ++++++ tools/kylin_client_tool/models/io/dimension.py | 55 ++++ tools/kylin_client_tool/models/io/measure.py | 56 ++++ tools/kylin_client_tool/models/io/readers.py | 96 ++++++ tools/kylin_client_tool/models/job.py | 119 +++++++ tools/kylin_client_tool/models/measure.py | 38 +++ tools/kylin_client_tool/models/object.py | 12 + tools/kylin_client_tool/models/request.py | 187 +++++++++++ tools/kylin_client_tool/models/rowkey.py | 79 +++++ tools/kylin_client_tool/rest/__init__.py | 2 + tools/kylin_client_tool/rest/apis.py | 102 ++++++ tools/kylin_client_tool/scheduler/__init__.py | 2 + .../scheduler/workers/__init__.py | 2 + .../kylin_client_tool/scheduler/workers/cube.py | 122 ++++++++ tools/kylin_client_tool/settings/__init__.py | 2 + tools/kylin_client_tool/settings/settings.py | 15 + tools/kylin_client_tool/setup-mac.sh | 20 ++ tools/kylin_client_tool/setup.sh | 20 ++ 34 files changed, 2244 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/client/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/client/__init__.py b/tools/kylin_client_tool/client/__init__.py new file mode 100644 index 0000000..c942174 --- /dev/null +++ b/tools/kylin_client_tool/client/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Ni Chunen' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/client/client_interface.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/client/client_interface.py b/tools/kylin_client_tool/client/client_interface.py new file mode 100644 index 0000000..ceb6095 --- /dev/null +++ b/tools/kylin_client_tool/client/client_interface.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +__author__ = 'Ni Chunen' +import time +import datetime + +from apscheduler.schedulers.background import BackgroundScheduler + +from apscheduler.schedulers.blocking import BlockingScheduler + +from scheduler.workers.cube import CubeWorker +from jobs.cube import CubeJob +from models.io.readers import CSVReader +from jobs.build import CubeBuildJob + + +class ClientJob: + @staticmethod + def build(cube_name_list, endtime=None): + run_cube_job_id = '1' + check_cube_job_id = '2' + scheduler = BackgroundScheduler() + CubeWorker.job_instance_dict = {} + + for cube_name in cube_name_list: + CubeWorker.job_instance_dict[cube_name] = None + + CubeWorker.scheduler = scheduler + CubeWorker.run_cube_job_id = run_cube_job_id + CubeWorker.check_cube_job_id = check_cube_job_id + # start the run cube job immediately + CubeWorker.run_cube_job(endtime) + + scheduler.add_job(CubeWorker.run_cube_job, 'interval', seconds=30, id=run_cube_job_id, args=[endtime]) + scheduler.add_job(CubeWorker.check_cube_job, 'interval', seconds=30, id=check_cube_job_id) + scheduler.start() + + while True: + if CubeWorker.all_finished(): + print "all cube jobs are finished" + scheduler.remove_job(check_cube_job_id) + scheduler.remove_job(run_cube_job_id) + scheduler.shutdown() + break + + time.sleep(15) + + @staticmethod + def init(cube_name_list): + pass + + @staticmethod + def build_cube_from_csv(csv_file, database, endtime=None, timed_build=None, crontab_options=None): + cube_dic_list = CSVReader.get_cube_desc_list_from_csv(csv_file, database) + cube_name_list = [] + + for cube_dic in cube_dic_list: + cube_name_list.append(cube_dic['cube_desc'].name) + + print "building cubes for", cube_name_list + + if timed_build is not None and crontab_options is not None: + scheduler = BlockingScheduler() + if timed_build is 'i' and crontab_options is not None: + scheduler.add_job(ClientJob.build, 'interval', hours=int(crontab_options), + args=[cube_name_list, endtime]) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + scheduler.shutdown() + elif timed_build is 't' and crontab_options is not None: + time_list = crontab_options.split(',') + if time_list.__len__() == 6: + scheduler.add_job(ClientJob.build, 'date', + run_date=datetime.datetime(int(time_list[0]), int(time_list[1]), + int(time_list[2]), int(time_list[3]), + int(time_list[4]), int(time_list[5])), + args=[cube_name_list, endtime]) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + scheduler.shutdown() + else: + print 'Bad command line!' + else: + print 'Bad command line!' + else: + ClientJob.build(cube_name_list, endtime) + + @staticmethod + def build_cube_from_names_or_file(cube_name, names_file, endtime=None, timed_build=None, crontab_options=None): + cube_name_list_from_file = [] + + if names_file is not None: + cube_name_list_from_file = CSVReader.get_cube_names_from_csv(names_file) + if cube_name is not None: + cube_name_list = cube_name.split(',') + + cube_name_list += cube_name_list_from_file + print "building cubes for", cube_name_list + + if timed_build is not None and crontab_options is not None: + scheduler = BlockingScheduler() + + if timed_build is 'i' and crontab_options is not None: + scheduler.add_job(ClientJob.build, 'interval', hours=int(crontab_options), + args=[cube_name_list, endtime]) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + scheduler.shutdown() + elif timed_build is 't' and crontab_options is not None: + time_list = crontab_options.split(',') + + if time_list.__len__() == 6: + scheduler.add_job(ClientJob.build, 'date', + run_date=datetime.datetime(int(time_list[0]), int(time_list[1]), + int(time_list[2]), int(time_list[3]), + int(time_list[4]), int(time_list[5])), + args=[cube_name_list, endtime]) + try: + scheduler.start() + except (KeyboardInterrupt, SystemExit): + scheduler.shutdown() + else: + print 'Bad command line!' + else: + print 'Bad command line!' + else: + ClientJob.build(cube_name_list, endtime) + + @staticmethod + def create_cube_from_csv(csv_file, project, database): + cube_dic_list = CSVReader.get_cube_desc_list_from_csv(csv_file, database) + + for cube_dic in cube_dic_list: + print 'creating cube for', cube_dic['cube_desc'].name, '@project=', project + # create cube under project default + # print cube_desc.to_json() + cube_request_result = CubeJob.create_cube(cube_dic['cube_desc'], cube_dic['model_desc'], project) + print 'result=', 'OK' if cube_request_result else 'FAILED' + + @staticmethod + def check_job_status(cube_name, names_file, status): + cube_name_list = [] + job_instance_list = [] + job_status = None + + if status is not None: + if status is 'R': + job_status = CubeJob.RUNNING_JOB_STATUS + elif status is 'F': + job_status = CubeJob.FINISHED_JOB_STATUS + elif status is 'D': + job_status = CubeJob.DISCARDED_JOB_STATUS + else: + job_status = CubeJob.ERROR_JOB_STATUS + + if cube_name is not None: + cube_name_list = cube_name.split(',') + elif names_file is not None: + cube_name_list = CSVReader.get_cube_names_from_csv(names_file) + + if cube_name_list.__len__() > 0: + for cube_name in cube_name_list: + job_instance_list += CubeJob.get_cube_job(cube_name, job_status) + else: + job_instance_list = CubeJob.get_cube_job(None, job_status) + + if job_instance_list.__len__() == 0: + print "No job found." + + for job_instance in job_instance_list: + print "JOB name " + job_instance.name + " of cube " + job_instance.related_cube + "'s status is " + job_instance.get_status() + + @staticmethod + def cancel_job(cube_name, names_file): + cube_name_list = [] + job_instance_list = [] + + if cube_name is not None: + cube_name_list = cube_name.split(',') + + if names_file is not None: + cube_name_list += CSVReader.get_cube_names_from_csv(names_file) + + if cube_name_list.__len__() > 0: + for cube_name in cube_name_list: + job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.ERROR_JOB_STATUS) + job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.RUNNING_JOB_STATUS) + else: + job_instance_list = CubeJob.get_cube_job(None, CubeJob.ERROR_JOB_STATUS) + + if job_instance_list.__len__() == 0: + print "No job found." + + for job_instance in job_instance_list: + print "Cancel job " + job_instance.name + "\n" + CubeBuildJob.cancel_job(job_instance.uuid) + + @staticmethod + def resume_job(cube_name, names_file): + cube_name_list = [] + job_instance_list = [] + + if cube_name is not None: + cube_name_list = cube_name.split(',') + + if names_file is not None: + cube_name_list += CSVReader.get_cube_names_from_csv(names_file) + + if cube_name_list.__len__() > 0: + for cube_name in cube_name_list: + job_instance_list += CubeJob.get_cube_job(cube_name, CubeJob.ERROR_JOB_STATUS) + else: + job_instance_list = CubeJob.get_cube_job(None, CubeJob.ERROR_JOB_STATUS) + + if job_instance_list.__len__() == 0: + print "No error job found." + + for job_instance in job_instance_list: + print "Resume job " + job_instance.name + "\n" + CubeBuildJob.resume_job(job_instance.uuid) + + @staticmethod + def __init__(): + pass http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/client/client_parse.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/client/client_parse.py b/tools/kylin_client_tool/client/client_parse.py new file mode 100644 index 0000000..8834bc9 --- /dev/null +++ b/tools/kylin_client_tool/client/client_parse.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +__author__ = 'Ni Chunen' + +from optparse import OptionParser +from settings.settings import KYLIN_REST_HOST +from client.client_interface import ClientJob + + +def menu_parser(): + print 'Current kylin rest host is ' + KYLIN_REST_HOST + ', if not, please quit and modify it from your setting file.' + parser = OptionParser() + parser.add_option("-c", "--create_cubes", action="store_true", + dest="create_cubes", + help="Create cubes with descriptions in the csv file to your project.") + parser.add_option("-b", "--build_cubes", action="store_true", + dest="build_cubes", + help="Build cubes with descriptions in the csv file to your project.") + parser.add_option("-s", "--check_job_status", action="store_true", + dest="check_job_status", + help="Check job status with options.") + parser.add_option("-k", "--cancel_job", action="store_true", + dest="cancel_job", + help="Cancel jobs with options.") + parser.add_option("-r", "--resume_job", action="store_true", + dest="resume_job", + help="Resume jobs with options.") + + parser.add_option("-D", "--database", + dest="database", + default="default", + help="Specify your database,[default=%default].") + parser.add_option("-P", "--project", + dest="project", + default="learn_kylin", + help="Specify your project,[default=%default].") + parser.add_option("-T", "--time", + dest="time", + default=None, + help="Set the end time of cube building,[default=%default].") + parser.add_option("-F", "--cubeDefFile", + dest="cubeDefFile", + default="cube_def.csv", + help="Specify your cubes definition file,[default=%default].") + parser.add_option("-f", "--cubeNameFile", + dest="cubeNameFile", + default=None, + help="Specify your cube names' file,[default=%default].") + parser.add_option("-S", "--status", + dest="status", + default=None, + help="Specify the job status, R for Running, E for Error, F for Finished, D for Discarded, [default=%default].") + parser.add_option("-C", "--cube_name", + dest="cube_name", + default=None, + help="Specify the cube name, [default=%default].") + parser.add_option("-B", "--schedule_build", + dest="schedule_build", + default=None, + help="Schedule cube building with options, 'i' for intervally build, 't' for time build, [default=%default].") + parser.add_option("-O", "--crontab_options", + dest="crontab_options", + default=None, + help="Set the options of timed building, like '-B i -O 24' for building every 24 hours, '-B t -O 2016,3,1,0,0,0' for building at '2016-3-1 0:0:0', [default=%default].") + + (options, args) = parser.parse_args() + status = True + + if options.create_cubes == True and options.cubeDefFile is not None and status == True: + ClientJob.create_cube_from_csv(options.cubeDefFile, options.project, options.database) + status = False + + if options.build_cubes == True and ( + options.cubeNameFile is not None or options.cube_name is not None) and status == True: + ClientJob.build_cube_from_names_or_file(options.cube_name, options.cubeNameFile, options.time, + options.schedule_build, options.crontab_options) + status = False + + if options.build_cubes == True and options.cubeDefFile is not None and status == True: + ClientJob.build_cube_from_csv(options.cubeDefFile, options.database, options.time, options.schedule_build, + options.crontab_options) + status = False + + if options.check_job_status == True and status == True: + ClientJob.check_job_status(options.cube_name, options.cubeNameFile, options.status) + status = False + + if options.cancel_job == True and status == True: + ClientJob.cancel_job(options.cube_name, options.cubeNameFile) + status = False + + if options.resume_job == True and status == True: + ClientJob.resume_job(options.cube_name, options.cubeNameFile) + status = False + + if status: + print 'Bad command line!' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/cube_def.csv ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/cube_def.csv b/tools/kylin_client_tool/cube_def.csv new file mode 100644 index 0000000..45f7880 --- /dev/null +++ b/tools/kylin_client_tool/cube_def.csv @@ -0,0 +1,2 @@ + client_tool_test1|kylin_sales|PART_DT,date;TRANS_ID,bigint;LSTG_FORMAT_NAME,string|PRICE,max,decimal|no_dictionary=5/TRANS_ID;mandatory_dimension=PART_DT;partition_date_column=PART_DT;partition_date_start=2010-01-01| + client_tool_test2|kylin_sales|PART_DT,date;SELLER_ID,bigint;SLR_SEGMENT_CD,smallint|ITEM_COUNT,sum,bigint|aggregation_group=PART_DT/SELLER_ID,PART_DT/SLR_SEGMENT_CD;partition_date_column=PART_DT;partition_date_start=2010-01-01| http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/cube_names.csv ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/cube_names.csv b/tools/kylin_client_tool/cube_names.csv new file mode 100644 index 0000000..c22d621 --- /dev/null +++ b/tools/kylin_client_tool/cube_names.csv @@ -0,0 +1,2 @@ +client_tool_test1 +client_tool_test2 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/jobs/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/jobs/__init__.py b/tools/kylin_client_tool/jobs/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/jobs/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/jobs/admin.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/jobs/admin.py b/tools/kylin_client_tool/jobs/admin.py new file mode 100644 index 0000000..cc41b7e --- /dev/null +++ b/tools/kylin_client_tool/jobs/admin.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from rest.apis import KylinRestApi + + +class AdminJob: + @staticmethod + def get_env(): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_get('admin/env', '') + + if KylinRestApi.is_response_ok(response): + status = 0 + # elif response is not None and response.json() and "does not exist" in str(response.json()): + # status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return response + + @staticmethod + def get_conf(): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_get('admin/config', '') + + if KylinRestApi.is_response_ok(response): + status = 0 + # elif response is not None and response.json() and "does not exist" in str(response.json()): + # status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return response + + @staticmethod + def cleanup_storage(): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_delete('admin/storage', '') + + if KylinRestApi.is_response_ok(response): + status = 0 + # elif response is not None and response.json() and "does not exist" in str(response.json()): + # status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return status http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/jobs/build.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/jobs/build.py b/tools/kylin_client_tool/jobs/build.py new file mode 100644 index 0000000..7a46240 --- /dev/null +++ b/tools/kylin_client_tool/jobs/build.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from rest.apis import KylinRestApi +from models.job import JobInstance + + +class CubeBuildJob: + @staticmethod + def rebuild_cube(cube_name, job_build_request): + job_instance = None + + try: + Kylin_rest_api = KylinRestApi() + response = Kylin_rest_api.http_put('cubes/' + cube_name + '/rebuild', '', + payload=job_build_request.to_json()) + + if KylinRestApi.is_response_ok(response): + job_instance = JobInstance.from_json(response.json()) + else: + print response.json() + except Exception, ex: + print ex + + return job_instance + + @staticmethod + def get_job(job_uuid): + job_instance = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_get('jobs/' + job_uuid, '') + + if KylinRestApi.is_response_ok(response): + job_instance = JobInstance.from_json(response.json()) + else: + print response.json() + except Exception, ex: + pass + + return job_instance + + @staticmethod + def cancel_job(job_uuid): + job_instance = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('jobs/' + job_uuid + '/cancel', '') + + if KylinRestApi.is_response_ok(response): + job_instance = JobInstance.from_json(response.json()) + else: + print response.json() + except Exception, ex: + pass + + return job_instance + + @staticmethod + def resume_job(job_uuid): + job_instance = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('jobs/' + job_uuid + '/resume', '') + + if KylinRestApi.is_response_ok(response): + job_instance = JobInstance.from_json(response.json()) + else: + print response.json() + except Exception, ex: + pass + + return job_instance http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/jobs/cube.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/jobs/cube.py b/tools/kylin_client_tool/jobs/cube.py new file mode 100644 index 0000000..4c6f775 --- /dev/null +++ b/tools/kylin_client_tool/jobs/cube.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.request import CubeRequest, JobListRequest +from models.cube import CubeInstance +from models.job import JobInstance +from rest.apis import KylinRestApi + + +class CubeJob: + RUNNING_JOB_STATUS = [0, 1, 2] + FINISHED_JOB_STATUS = [4] + ERROR_JOB_STATUS = [8] + DISCARDED_JOB_STATUS = [16] + + @staticmethod + def create_cube(cube_desc, model_desc, project=None): + cube_request_result = None + + try: + # set last modified time to 0 + cube_desc.last_modified = 0 + cube_request = CubeRequest.get_cube_request_from_cube_desc(cube_desc, model_desc, project) + + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_post('cubes', '', payload=cube_request.to_json()) + + if KylinRestApi.is_response_ok(response): + cube_request_result = CubeRequest.from_json(response.json()) + # set result to null if the operation is not successful + if not cube_request_result.successful: + cube_request_result = None + print response.json() + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return cube_request_result + + @staticmethod + def update_cube(cube_desc, model_desc, project=None): + cube_request_result = None + + try: + cube_request = CubeRequest.get_cube_request_from_cube_desc(cube_desc, model_desc, project) + + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('cubes', '', payload=cube_request.to_json()) + + if KylinRestApi.is_response_ok(response): + cube_request_result = CubeRequest.from_json(response.json()) + # set result to null if the operation is not successful + if not cube_request_result.successful: + cube_request_result = None + print response.json() + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return cube_request_result + + @staticmethod + def delete_cube(cube_name): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_delete('cubes/' + cube_name, '') + # print response.json() + + if KylinRestApi.is_response_ok(response): + status = 0 + elif response is not None and response.json() and "not found" in str(response.json()): + status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return status + + @staticmethod + def disable_cube(cube_name): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('cubes/' + cube_name + '/disable', '') + + if KylinRestApi.is_response_ok(response): + status = 0 + elif response is not None and response.json() and "is DISABLED" in str(response.json()): + status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return status + + @staticmethod + def enable_cube(cube_name): + status = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('cubes/' + cube_name + '/enable', '') + + if KylinRestApi.is_response_ok(response): + status = 0 + elif response is not None and response.json() and "is READY" in str(response.json()): + status = 0 + else: + print response.json() + # cube_request = None + except Exception, ex: + pass + + return status + + @staticmethod + def get_cube_job(cube_name, status_list=None): + job_list_req = JobListRequest() + job_list_req.cubeName = cube_name + job_list_req.limit = 10000 + job_list_req.offset = 0 + job_list_req.status = status_list + + job_instance_list = [] + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_get('jobs/', job_list_req.to_query_string()) + + if KylinRestApi.is_response_ok(response): + job_instance_list = [JobInstance.from_json(json_dict) for json_dict in response.json()] + else: + print response.json() + except Exception, ex: + pass + + return job_instance_list + + @staticmethod + def list_cubes(cube_name=None, project_name=None): + offset, limit = 0, 10000 + query_string = '' + ('cubeName=' + cube_name + '&' if cube_name else '') + \ + ('projectName=' + project_name + '&' if project_name else '') + \ + ('offset=' + str(offset) + '&') + ('limit=' + str(limit) + '&') + cube_instance_list = [] + + try: + kylin_rest_api = KylinRestApi() + + response = kylin_rest_api.http_get('cubes/', query_string) + + if KylinRestApi.is_response_ok(response): + cube_instance_list = [CubeInstance.from_json(json_dict) for json_dict in response.json()] + else: + print response.json() + except Exception, ex: + pass + + return cube_instance_list + + @staticmethod + def update_cube_cost(cube_name, cost): + cube_instance = None + + try: + kylin_rest_api = KylinRestApi() + response = kylin_rest_api.http_put('cubes/' + cube_name + '/cost', 'cost=' + str(cost)) + + if KylinRestApi.is_response_ok(response): + cube_instance = CubeInstance.from_json(response.json()) + else: + print response.json() + except Exception, ex: + pass + + return cube_instance http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/kylin_client_tool.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/kylin_client_tool.py b/tools/kylin_client_tool/kylin_client_tool.py new file mode 100644 index 0000000..29d1819 --- /dev/null +++ b/tools/kylin_client_tool/kylin_client_tool.py @@ -0,0 +1,5 @@ +__author__ = 'Ni Chunen' +from client.client_parse import menu_parser + +if __name__ == '__main__': + menu_parser() http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/__init__.py b/tools/kylin_client_tool/models/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/models/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/cube.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/cube.py b/tools/kylin_client_tool/models/cube.py new file mode 100644 index 0000000..c2980e6 --- /dev/null +++ b/tools/kylin_client_tool/models/cube.py @@ -0,0 +1,309 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.object import JsonSerializableObj +from models.dimension import DimensionDesc +from models.measure import MeasureDesc +from models.rowkey import RowKeyDesc +from models.hbase import HBaseMappingDesc + + +class CubeDesc(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.cube.model.CubeDesc + """ + + class CubeDescSetting: + APPEND_COUNT_MEASURE = 'append_count_measure' + NO_DICTIONARY = 'no_dictionary' + MANDATORY_DIMENSION = 'mandatory_dimension' + AGGREGATION_GROUP = 'aggregation_group' + PARTITION_DATE_COLUMN = 'partition_date_column' + PARTITION_DATE_START = 'partition_date_start' + + @staticmethod + def get_value(settings, name, default): + value = settings.get(name) + + if value is None: + return default + if value and type(value) == list and value[0].lower() == 'false': + return False + if value and type(value) == list and value[0].lower() == 'true': + return True + + return value + + def __init__(self): + JsonSerializableObj.__init__(self) + self.name = None + self.description = '' + self.dimensions = None + self.measures = None + self.rowkey = None + self.notify_list = [] + self.hbase_mapping = None + self.model_name = '' + self.retention_range = '0' + + # self.uuid = None + # self.last_modified = None + # self.fact_table = None + # self.null_string = None + # self.filter_condition = None + # self.cube_partition_desc = None + # self.signature = None + + def append_count_measure(self): + no_count_measure = True + if self.measures: + for measure in self.measures: + if measure.name == '_COUNT_': + no_count_measure = False + + if no_count_measure: + if not self.measures: self.measures = [] + + self.measures.append(MeasureDesc.get_count_measure(len(self.measures) + 1)) + + def apply_settings(self, settings): + append_count = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.APPEND_COUNT_MEASURE, True) + no_dictionary = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.NO_DICTIONARY, None) + mandatory_dimension = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.MANDATORY_DIMENSION, + None) + aggregation_group = CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.AGGREGATION_GROUP, + None) + + # apply append_count setting + if append_count is None or append_count is True: + self.append_count_measure() + # update hbase_mapping + self.hbase_mapping = HBaseMappingDesc.get_from_measures(self.measures) + + # apply no_dictionary setting + if no_dictionary and type(no_dictionary) == list and self.rowkey.rowkey_columns: + rowkey_column_cnt = len(self.rowkey.rowkey_columns) + for no_dictionary_tuple in no_dictionary: + fields = no_dictionary_tuple.split('/') + rowkey_size = fields[0] + column_name = fields[1] + for i in range(rowkey_column_cnt): + if self.rowkey.rowkey_columns[i].column == column_name: + self.rowkey.rowkey_columns[i].length = int(rowkey_size) + self.rowkey.rowkey_columns[i].dictionary = None + + if aggregation_group and type(aggregation_group) == list: + agg_groups = [] + for aggregation_group_tuple in aggregation_group: + fields = aggregation_group_tuple.split('/') + one_group = [] + for i in range(len(fields)): + one_group.append(fields[i]) + agg_groups.append(one_group) + self.rowkey.aggregation_groups = agg_groups + + # apply mandatory_dimension setting + if mandatory_dimension and type( + mandatory_dimension) == list and self.rowkey.rowkey_columns and self.rowkey.aggregation_groups: + rowkey_column_cnt = len(self.rowkey.rowkey_columns) + for m_dim in mandatory_dimension: + for i in range(rowkey_column_cnt): + if self.rowkey.rowkey_columns[i].column == m_dim: + self.rowkey.rowkey_columns[i].mandatory = True + + agg_group_cnt = len(self.rowkey.aggregation_groups) + for i in range(agg_group_cnt): + new_agg_group = [] + for dim in self.rowkey.aggregation_groups[i]: + if mandatory_dimension.count(dim) <= 0: + new_agg_group.append(dim) + self.rowkey.aggregation_groups[i] = new_agg_group + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + cd = CubeDesc() + + cd.name = json_dict.get('name') + cd.description = json_dict.get('description') + if json_dict.get('dimensions') and type(json_dict.get('dimensions')) == list: + dimension_list = json_dict.get('dimensions') + cd.dimensions = [DimensionDesc.from_json(dimension) for dimension in dimension_list] + if json_dict.get('measures') and type(json_dict.get('measures')) == list: + measure_list = json_dict.get('measures') + cd.measures = [MeasureDesc.from_json(measure) for measure in measure_list] + # deserialize json for rowkey + cd.rowkey = RowKeyDesc.from_json(json_dict.get('rowkey')) + cd.notify_list = json_dict.get('notify_list') + # cd.capacity = json_dict.get('capacity') + # deserialize json for hbase_mapping + cd.hbase_mapping = HBaseMappingDesc.from_json(json_dict.get('hbase_mapping')) + cd.retention_range = json_dict.get('retention_range') + return cd + # cd.uuid = json_dict.get('uuid') + # cd.last_modified = json_dict.get('last_modified') + # cd.fact_table = json_dict.get('fact_table') + # cd.null_string = json_dict.get('null_string') + # cd.filter_condition = json_dict.get('filter_condition') + # deserialize json for cube_partition_desc + # cd.cube_partition_desc = CubePartitionDesc.from_json(json_dict.get('cube_partition_desc')) + # deserialize json for dimensions + + # deserialize json for measures + # cd.signature = json_dict.get('signature') + + +class CubeModel(JsonSerializableObj): + def __init__(self): + JsonSerializableObj.__init__(self) + self.name = None + self.fact_table = None + self.lookups = [] + self.filter_condition = '' + self.capacity = 'MEDIUM' + self.partition_desc = None + self.last_modified = 0 + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + cm = CubeModel() + + cm.name = json_dict.get('name') + cm.fact_table = json_dict.get('json_dict') + if json_dict.get('lookups') and type(json_dict.get('lookups')) == list: + lookups_list = json_dict.get('lookups') + cm.lookups = lookups_list + cm.filter_condition = json_dict.get('json_dict') + cm.capacity = json_dict.get('json_dict') + cm.partition_desc = CubePartitionDesc.from_json(json_dict.get('partition_desc')) + cm.last_modified = json_dict.get('last_modified') + return cm + + +class CubePartitionDesc(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.cube.model.v1.CubePartitionDesc + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.partition_date_column = '' + self.partition_date_start = '' + self.cube_partition_type = 'APPEND' + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + cpd = CubePartitionDesc() + + cpd.partition_date_column = json_dict.get('partition_date_column') + cpd.partition_date_start = json_dict.get('partition_date_start') + cpd.cube_partition_type = json_dict.get('cube_partition_type') + + return cpd + + @staticmethod + def get_default(): + json_dict = {'partition_date_start': 0, 'cube_partition_type': 'APPEND', 'partition_date_column': None} + + return CubePartitionDesc.from_json(json_dict) + + @staticmethod + def get_from_setting(settings): + desc = CubePartitionDesc() + + desc.partition_date_start = \ + CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.PARTITION_DATE_START, [None])[0] + desc.cube_partition_type = 'APPEND' + desc.partition_date_column = \ + CubeDesc.CubeDescSetting.get_value(settings, CubeDesc.CubeDescSetting.PARTITION_DATE_COLUMN, [None])[0] + return desc + + +class CubeInstance(JsonSerializableObj): + def __init__(self): + JsonSerializableObj.__init__(self) + + self.name = None + self.owner = None + self.version = None + self.descName = None + self.cost = None + self.status = None + self.segments = None + self.create_time = None + self.size_kb = None + self.source_records_count = None + self.source_records_size = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + ci = CubeInstance() + + ci.name = json_dict.get('name') + ci.owner = json_dict.get('owner') + ci.version = json_dict.get('version') + ci.descName = json_dict.get('descName') + ci.cost = json_dict.get('cost') + ci.status = json_dict.get('status') + # deserialize json for segments + if json_dict.get('segments') and type(json_dict.get('segments')) == list: + segment_list = json_dict.get('segments') + ci.segments = [CubeSegment.from_json(segment) for segment in segment_list] + ci.create_time = json_dict.get('create_time') + ci.size_kb = json_dict.get('size_kb') + ci.source_records_count = json_dict.get('source_records_count') + ci.source_records_size = json_dict.get('source_records_size') + + return ci + + +class CubeSegment(JsonSerializableObj): + def __init__(self): + JsonSerializableObj.__init__(self) + + self.uuid = None + self.name = None + self.storage_location_identifier = None + self.date_range_start = None + self.date_range_end = None + self.status = None + self.size_kb = None + self.source_records = None + self.source_records_size = None + self.last_build_time = None + self.last_build_job_id = None + self.create_time = None + self.binary_signature = None + self.dictionaries = None + self.snapshots = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + cs = CubeSegment() + + cs.uuid = json_dict.get('uuid') + cs.name = json_dict.get('name') + cs.storage_location_identifier = json_dict.get('storage_location_identifier') + cs.date_range_start = json_dict.get('date_range_start') + cs.date_range_end = json_dict.get('date_range_end') + cs.status = json_dict.get('status') + cs.size_kb = json_dict.get('size_kb') + cs.source_records = json_dict.get('source_records') + cs.source_records_size = json_dict.get('source_records_size') + cs.last_build_time = json_dict.get('last_build_time') + cs.last_build_job_id = json_dict.get('last_build_job_id') + cs.create_time = json_dict.get('create_time') + cs.binary_signature = json_dict.get('binary_signature') + cs.dictionaries = json_dict.get('dictionaries') + cs.snapshots = json_dict.get('snapshots') + + return cs http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/dimension.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/dimension.py b/tools/kylin_client_tool/models/dimension.py new file mode 100644 index 0000000..0d9446d --- /dev/null +++ b/tools/kylin_client_tool/models/dimension.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + + +class DimensionDesc: + """ + python class mapping to org.apache.kylin.cube.model.DimensionDesc + """ + + def __init__(self): + self.id = None + self.name = None + self.join = None + self.hierarchy = None + self.table = None + self.column = [] + self.datatype = None + self.derived = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + dd = DimensionDesc() + + dd.id = json_dict.get('id') + dd.name = json_dict.get('name') + # deserialize json for join + dd.join = JoinDesc.from_json(json_dict.get('join')) + # deserialize json for hierarchy + if json_dict.get('hierarchy') and type(json_dict.get('hierarchy')) == list: + hierarchy_list = json_dict.get('hierarchy') + dd.hierarchy = [HierarchyDesc.from_json(hierarchy) for hierarchy in hierarchy_list] + dd.table = json_dict.get('table') + dd.column = json_dict.get('column') + dd.datatype = json_dict.get('datatype') + dd.derived = json_dict.get('derived') + + return dd + + +class JoinDesc: + """ + python class mapping to org.apache.kylin.metadata.model.JoinDesc + """ + + def __init__(self): + self.type = None + self.primary_key = None + self.foreign_key = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + jd = JoinDesc() + + jd.type = json_dict.get('type') + jd.primary_key = json_dict.get('primary_key') + jd.foreign_key = json_dict.get('foreign_key') + + return jd + + +class HierarchyDesc: + """ + python class mapping to org.apache.kylin.cube.model.HierarchyDesc + """ + + def __init__(self): + self.level = None + self.column = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + hd = HierarchyDesc() + + hd.level = json_dict.get('level') + hd.column = json_dict.get('column') + + return hd http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/hbase.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/hbase.py b/tools/kylin_client_tool/models/hbase.py new file mode 100644 index 0000000..e8dac1a --- /dev/null +++ b/tools/kylin_client_tool/models/hbase.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + + +class HBaseMappingDesc: + """ + python class mapping to org.apache.kylin.cube.model.HBaseMappingDesc + """ + + def __init__(self): + self.column_family = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + hmd = HBaseMappingDesc() + + # deserialize json for columns + if json_dict.get('column_family') and type(json_dict.get('column_family')) == list: + column_family_list = json_dict.get('column_family') + hmd.column_family = [HBaseColumnFamilyDesc.from_json(column_family) for column_family in column_family_list] + + return hmd + + @staticmethod + def get_from_measures(measures): + hbmd = HBaseMappingDesc() + hbmd.column_family = [] + + if not measures: return hbmd + + start, step, measure_cnt = 0, 5, len(measures) + family_id = 1 + while start < measure_cnt: + hbcfd = HBaseColumnFamilyDesc() + hbcfd.name = 'F' + str(family_id) + hbcfd.columns = [] + + hbcd = HBaseColumnDesc() + hbcd.qualifier = 'M' + hbcd.measure_refs = [] + + for measure in measures[start:start + step]: + hbcd.measure_refs.append(measure.name) + + # append column instance + hbcfd.columns.append(hbcd) + # append column family instance + hbmd.column_family.append(hbcfd) + + start += step + family_id += 1 + + return hbmd + + +class HBaseColumnFamilyDesc: + """ + python class mapping to org.apache.kylin.cube.model.HBaseColumnFamilyDesc + """ + + def __init__(self): + self.name = None + self.columns = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + hcfd = HBaseColumnFamilyDesc() + + hcfd.name = json_dict.get('name') + # deserialize json for columns + if json_dict.get('columns') and type(json_dict.get('columns')) == list: + column_list = json_dict.get('columns') + hcfd.columns = [HBaseColumnDesc.from_json(column) for column in column_list] + + return hcfd + + +class HBaseColumnDesc: + """ + python class mapping to org.apache.kylin.cube.model.HBaseColumnDesc + """ + + def __init__(self): + self.qualifier = None + self.measure_refs = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + hcd = HBaseColumnDesc() + + hcd.qualifier = json_dict.get('qualifier') + hcd.measure_refs = json_dict.get('measure_refs') + + return hcd http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/__init__.py b/tools/kylin_client_tool/models/io/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/models/io/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/base.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/base.py b/tools/kylin_client_tool/models/io/base.py new file mode 100644 index 0000000..cdca483 --- /dev/null +++ b/tools/kylin_client_tool/models/io/base.py @@ -0,0 +1,65 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + + +class CSV: + def __init__(self, csv_line, sep): + self.csv_line = csv_line + self.content_list = csv_line.split(sep) if csv_line and isinstance(csv_line, str) else None + + def get_property(self, ind): + if self.content_list and len(self.content_list) > ind: + return self.content_list[ind] + return None + + def is_object_valid(self, cls): + if hasattr(cls, 'not_null_attrs'): + for attr in getattr(cls, 'not_null_attrs'): + if not hasattr(self, attr) or not getattr(self, attr): + return False + return True + + +class FunctionDesc: + """ + python class mapping to org.apache.kylin.metadata.model.FunctionDesc + """ + + def __init__(self): + self.expression = None + self.parameter = None + self.returntype = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + fd = FunctionDesc() + + fd.expression = json_dict.get('expression') + # deserialize json for parameter + fd.parameter = ParameterDesc.from_json(json_dict.get('parameter')) + fd.returntype = json_dict.get('returntype') + + return fd + + +class ParameterDesc: + """ + python class mapping to org.apache.kylin.metadata.model.ParameterDesc + """ + + def __init__(self): + self.type = None + self.value = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + pd = ParameterDesc() + + pd.type = json_dict.get('type') + pd.value = json_dict.get('value') + + return pd http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/cube.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/cube.py b/tools/kylin_client_tool/models/io/cube.py new file mode 100644 index 0000000..9474855 --- /dev/null +++ b/tools/kylin_client_tool/models/io/cube.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.io.base import CSV +from models.io.dimension import DimensionCSV +from models.io.measure import MeasureCSV + + +class CubeCSV(CSV): + columns = {'name': 0, 'table': 1, 'dimensions': 2, 'measures': 3, 'settings': 4, 'filter': 5} + not_null_attrs = ['name', 'table', 'dimensions'] + + def __init__(self, csv_line, db): + CSV.__init__(self, csv_line, '|') + + self.name = self.get_property(CubeCSV.columns['name']) + self.table = self.get_property(CubeCSV.columns['table']) + self.dimensions = None + self.measures = None + self.settings = {} + self.filter = None + + dimensions_csv = self.get_property(CubeCSV.columns['dimensions']) + self.dimensions = DimensionCSV.get_list_from_csv(dimensions_csv, self.table, db) + + measures_csv = self.get_property(CubeCSV.columns['measures']) + self.measures = MeasureCSV.get_list_from_csv(measures_csv, 'column') + # print measures_csv, len(self.measures) if self.measures else 0 + + settings_csv = self.get_property(CubeCSV.columns['settings']) + self.settings = CubeSettingCSV.get_settings_from_csv(settings_csv) + + filter_csv = self.get_property(CubeCSV.columns['filter']) + self.filter = filter_csv + + def is_valid(self): + if not self.is_object_valid(CubeCSV): + print self.to_cube_desc_json(), 'not ok' + return False + + if self.dimensions: + dim_name_dict = {} + for dimension in self.dimensions: + if not dimension.is_valid() or dimension.name in dim_name_dict: + print dimension.to_dimension_desc_json(), 'not ok' + return False + dim_name_dict[dimension.name] = True + + if self.measures: + measure_name_dict = {} + for measure in self.measures: + if not measure.is_valid() or measure.name in measure_name_dict: + print measure.to_measure_desc_json(), 'not ok' + return False + measure_name_dict[measure.name] = True + + return True + + def to_cube_desc_json(self): + json_dict = {'name': self.name, 'fact_table': self.table, 'capacity': 'MEDIUM', + 'dimensions': [dimension.to_dimension_desc_json() for dimension in self.dimensions], + 'measures': [measure.to_measure_desc_json() for measure in self.measures]} + + return json_dict + + +class CubeSettingCSV(CSV): + columns = {'name': 0, 'value': 1} + + def __init__(self, csv_line): + CSV.__init__(self, csv_line, '=') + + self.name = self.get_property(CubeSettingCSV.columns['name']) + self.value = self.get_property(CubeSettingCSV.columns['value']) + + if self.value: + self.value = self.value.split(',') + + @staticmethod + def get_settings_from_csv(csv_line): + if not csv_line: return {} + + settings = {} + setting_csv_list = csv_line.split(';') + for csv in setting_csv_list: + setting_csv = CubeSettingCSV(csv) + if setting_csv: + settings[setting_csv.name] = setting_csv.value + + return settings http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/dimension.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/dimension.py b/tools/kylin_client_tool/models/io/dimension.py new file mode 100644 index 0000000..5276743 --- /dev/null +++ b/tools/kylin_client_tool/models/io/dimension.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.io.base import CSV + + +class DimensionCSV(CSV): + columns = {'column': 0, 'data_type': 1} + not_null_attrs = ['id', 'table', 'column', 'name'] + + def __init__(self, csv_line, db): + CSV.__init__(self, csv_line, ',') + + self.id = None + self.table = None + self.column = [] + self.column.append(self.get_property(DimensionCSV.columns['column'])) + self.data_type = self.get_property(DimensionCSV.columns['data_type']) + self.name = self.column[0] + + def to_dimension_desc_json(self): + json_dict = {'id': self.id, 'table': self.table, 'datatype': self.data_type, + 'column': self.column, 'name': self.name} + + return json_dict + + def is_valid(self): + return self.is_object_valid(DimensionCSV) + + @staticmethod + def get_from_csv(csv_line, id, table, db): + d_csv = None + + if csv_line: + d_csv = DimensionCSV(csv_line, db) + d_csv.id = id + d_csv.table = db + '.' + table + d_csv.name = d_csv.table + '.' + d_csv.name + + return d_csv + + @staticmethod + def get_list_from_csv(csv_line, table, db): + if not csv_line: return [] + + dimension_list = [] + dimension_csv_list = csv_line.split(';') + dim_id = 1 + for csv in dimension_csv_list: + dim_csv = DimensionCSV.get_from_csv(csv, dim_id, table, db) + if dim_csv: + dimension_list.append(dim_csv) + dim_id += 1 + + return dimension_list http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/measure.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/measure.py b/tools/kylin_client_tool/models/io/measure.py new file mode 100644 index 0000000..90ed65d --- /dev/null +++ b/tools/kylin_client_tool/models/io/measure.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.io.base import CSV, FunctionDesc, ParameterDesc + + +class MeasureCSV(CSV): + columns = {'value': 0, 'expression': 1, 'return_type': 2} + not_null_attrs = ['id', 'name', 'function'] + + def __init__(self, csv_line): + CSV.__init__(self, csv_line, ',') + parameter = ParameterDesc() + parameter.value = self.get_property(MeasureCSV.columns['value']) + function = FunctionDesc() + function.expression = self.get_property(MeasureCSV.columns['expression']) + function.parameter = parameter + function.returntype = self.get_property(MeasureCSV.columns['return_type']) + self.id = None + self.function = function + self.name = str(self.function.expression) + '_' + str(self.function.parameter.value) + + def to_measure_desc_json(self): + json_dict = {'id': self.id, 'name': self.name, 'dependent_measure_ref': None, + 'function': {'returntype': self.function.returntype, 'expression': self.function.expression, + 'parameter': {'type': self.function.parameter.type, + 'value': self.function.parameter.value}}} + return json_dict + + def is_valid(self): + return self.is_object_valid(MeasureCSV) + + @staticmethod + def get_from_csv(csv_line, id, type): + m_csv = MeasureCSV(csv_line) + + m_csv.id = id + m_csv.function.parameter.type = type + + return m_csv + + @staticmethod + def get_list_from_csv(csv_line, type): + if not csv_line: return [] + + measure_list = [] + measure_csv_list = csv_line.split(';') + m_id = 1 + + for csv in measure_csv_list: + m_csv = MeasureCSV.get_from_csv(csv, m_id, type) + if m_csv: + measure_list.append(m_csv) + m_id += 1 + + return measure_list http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/io/readers.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/io/readers.py b/tools/kylin_client_tool/models/io/readers.py new file mode 100644 index 0000000..437671c --- /dev/null +++ b/tools/kylin_client_tool/models/io/readers.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +import sys, time, datetime +from models.io.cube import CubeCSV +from models.cube import CubeDesc, CubeModel +from models.cube import CubePartitionDesc +from models.hbase import HBaseMappingDesc +from models.rowkey import RowKeyDesc + + +class CSVReader: + APPEND_COUNT_MEASURE = 'append_count_measure' + + @staticmethod + def get_cube_desc_from_csv_line(csv_line, db): + cube_dic = {} + cube_csv = CubeCSV(csv_line.strip(), db) + + if not cube_csv.is_valid(): + raise Exception + + cube_desc = CubeDesc() + model_desc = CubeModel() + cube_desc.name = cube_csv.name + cube_desc.model_name = (cube_csv.name).upper() + cube_desc.dimensions = cube_csv.dimensions + cube_desc.measures = cube_csv.measures + if not cube_desc.rowkey: + cube_desc.rowkey = RowKeyDesc.get_from_dimensions(cube_desc.dimensions) + if not cube_desc.hbase_mapping: + # print cube_desc.measures + cube_desc.hbase_mapping = HBaseMappingDesc.get_from_measures(cube_desc.measures) + + model_desc.name = cube_csv.name + model_desc.fact_table = (db + '.' + cube_csv.table).upper() + if not model_desc.partition_desc: + model_desc.partition_desc = CubePartitionDesc.get_from_setting(settings=cube_csv.settings) + if not (model_desc.partition_desc.partition_date_column is None): + model_desc.partition_desc.partition_date_column \ + = (db + '.' + cube_csv.table + '.' + model_desc.partition_desc.partition_date_column).upper() + timestamp = int(time.mktime(datetime.datetime.strptime(model_desc.partition_desc.partition_date_start, + "%Y-%m-%d").timetuple())) - time.timezone + model_desc.partition_desc.partition_date_start = timestamp * 1000 + + # print cube_desc.rowkey.rowkey_columns[0].dictionary + + # apply settings + cube_desc.apply_settings(settings=cube_csv.settings) + if cube_csv.filter is None: + model_desc.filter_condition = '' + else: + model_desc.filter_condition = cube_csv.filter + + cube_dic['cube_desc'] = cube_desc + cube_dic['model_desc'] = model_desc + # print cube_desc.rowkey.rowkey_columns[0].dictionary + + return cube_dic + + @staticmethod + def get_cube_desc_list_from_csv(csv_file, db): + fd = open(csv_file, 'r') + csv_lines = fd.readlines() + + cube_desc_list = [] + + for csv_line in csv_lines: + if csv_line: + try: + cube_dic = CSVReader.get_cube_desc_from_csv_line(csv_line, db) + + cube_desc_list.append(cube_dic) + except Exception, ex: + import traceback + + traceback.print_exc() + print "can't parse this csv for cube", csv_line + sys.exit(1) + pass + + return cube_desc_list + + @staticmethod + def get_cube_names_from_csv(csv_file): + fd = open(csv_file, 'r') + csv_lines = fd.readlines() + + cube_names_list = [] + + for csv_line in csv_lines: + csv_line = csv_line.strip() + if csv_line: + cube_names_list.append(csv_line) + + return cube_names_list http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/job.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/job.py b/tools/kylin_client_tool/models/job.py new file mode 100644 index 0000000..74b59be --- /dev/null +++ b/tools/kylin_client_tool/models/job.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.object import JsonSerializableObj + + +class CubeJobStatus: + RUNNING = 'RUNNING' + ERROR = 'ERROR' + FINISHED = 'FINISHED' + DISCARD = 'DISCARDED' + + +class JobInstance(JsonSerializableObj): + def __init__(self): + JsonSerializableObj.__init__(self) + + self.uuid = None + self.last_modified = None + self.name = None + self.type = None + self.duration = None + self.related_cube = None + self.related_segment = None + self.exec_start_time = None + self.exec_end_time = None + self.mr_waiting = None + self.steps = None + self.submitter = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + ji = JobInstance() + + ji.uuid = json_dict.get('uuid') + ji.last_modified = json_dict.get('last_modified') + ji.name = json_dict.get('name') + ji.type = json_dict.get('type') + ji.duration = json_dict.get('duration') + ji.related_cube = json_dict.get('related_cube') + ji.related_segment = json_dict.get('related_segment') + ji.exec_start_time = json_dict.get('exec_start_time') + ji.exec_end_time = json_dict.get('exec_end_time') + ji.mr_waiting = json_dict.get('mr_waiting') + # deserialize json for steps + if json_dict.get('steps') and type(json_dict.get('steps')) == list: + step_list = json_dict.get('steps') + ji.steps = [JobStep.from_json(step) for step in step_list] + ji.submitter = json_dict.get('submitter') + + return ji + + def get_status(self): + if not self.steps: + return CubeJobStatus.ERROR + + for job_step in self.steps: + if job_step.step_status in CubeJobStatus.ERROR: + return CubeJobStatus.ERROR + if job_step.step_status in CubeJobStatus.DISCARD: + return CubeJobStatus.DISCARD + + # check the last step + job_step = self.steps[-1] + if job_step.step_status not in CubeJobStatus.FINISHED: + return CubeJobStatus.RUNNING + + return CubeJobStatus.FINISHED + + def get_current_step(self): + if not self.steps: + return 0 + + step_id = 1 + for job_step in self.steps: + if job_step.step_status not in CubeJobStatus.FINISHED: + return step_id + step_id += 1 + + return len(self.steps) + + +class JobStep(JsonSerializableObj): + def __init__(self): + JsonSerializableObj.__init__(self) + + self.name = None + self.sequence_id = None + self.exec_cmd = None + self.interrupt_cmd = None + self.exec_start_time = None + self.exec_end_time = None + self.exec_wait_time = None + self.step_status = None + self.cmd_type = None + self.info = None + self.run_async = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + js = JobStep() + + js.name = json_dict.get('name') + js.sequence_id = json_dict.get('sequence_id') + js.exec_cmd = json_dict.get('exec_cmd') + js.interrupt_cmd = json_dict.get('interrupt_cmd') + js.exec_start_time = json_dict.get('exec_start_time') + js.exec_end_time = json_dict.get('exec_end_time') + js.exec_wait_time = json_dict.get('exec_wait_time') + js.step_status = json_dict.get('step_status') + js.cmd_type = json_dict.get('cmd_type') + js.info = json_dict.get('info') + js.run_async = json_dict.get('run_async') + + return js http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/measure.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/measure.py b/tools/kylin_client_tool/models/measure.py new file mode 100644 index 0000000..b26b623 --- /dev/null +++ b/tools/kylin_client_tool/models/measure.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' +from models.io.base import FunctionDesc + + +class MeasureDesc: + """ + python class mapping to org.apache.kylin.metadata.model.MeasureDesc + """ + + def __init__(self): + self.id = None + self.name = None + self.function = None + self.dependent_measure_ref = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + md = MeasureDesc() + + md.id = json_dict.get('id') + md.name = json_dict.get('name') + # deserialize json for function + md.function = FunctionDesc.from_json(json_dict.get('function')) + md.dependent_measure_ref = json_dict.get('dependent_measure_ref') + + return md + + @staticmethod + def get_count_measure(id): + json_dict = {'id': id, 'name': '_COUNT_', + 'function': {'expression': 'COUNT', 'returntype': 'bigint', + 'parameter': {'type': 'constant', 'value': 1}}, + 'dependent_measure_ref': None} + + return MeasureDesc.from_json(json_dict) http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/object.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/object.py b/tools/kylin_client_tool/models/object.py new file mode 100644 index 0000000..d07f89f --- /dev/null +++ b/tools/kylin_client_tool/models/object.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +import json + + +class JsonSerializableObj: + def __init__(self): + pass + + def to_json(self): + return json.dumps(self, default=lambda o: o.__dict__) http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/request.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/request.py b/tools/kylin_client_tool/models/request.py new file mode 100644 index 0000000..29cd022 --- /dev/null +++ b/tools/kylin_client_tool/models/request.py @@ -0,0 +1,187 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +from models.object import JsonSerializableObj +from models.cube import CubeDesc, CubeModel + + +class CubeRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.CubeRequest + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.uuid = None + self.cubeName = None + self.cubeDescData = None + self.modelDescData = None + self.successful = None + self.message = None + self.cubeDescName = None + self.project = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + cr = CubeRequest() + + cr.uuid = json_dict.get('uuid') + cr.cubeName = json_dict.get('cubeName') + cr.cubeDescData = json_dict.get('cubeDescData') + cr.modelDescData = json_dict.get('modelDescData') + cr.successful = json_dict.get('successful') + cr.message = json_dict.get('message') + cr.cubeDescName = json_dict.get('cubeDescName') + cr.project = json_dict.get('project') + + return cr + + @staticmethod + def get_cube_request_from_cube_desc(cube_desc, model_desc, project=None): + if not cube_desc or not isinstance(cube_desc, CubeDesc): return None + if not model_desc or not isinstance(model_desc, CubeModel): return None + + cr = CubeRequest() + + # cr.uuid = cube_desc.uuid + cr.cubeDescData = cube_desc.to_json() + # print cr.cubeDescData + cr.modelDescData = model_desc.to_json() + cr.cubeName = cube_desc.name + + cr.project = project + + return cr + + +class JobBuildRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.JobBuildRequest + """ + BUILD = 'BUILD' + MERGE = 'MERGE' + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.startTime = None + self.endTime = None + self.buildType = JobBuildRequest.BUILD + + +class JobListRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.JobListRequest + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.cubeName = None + self.projectName = None + self.offset = None + self.limit = None + self.status = None + + def to_query_string(self): + qs = "" + + if self.cubeName: + qs += "cubeName=" + self.cubeName + "&" + if self.projectName: + qs += "projectName=" + self.projectName + "&" + if self.offset is not None: + qs += "offset=" + str(self.offset) + "&" + if self.limit is not None: + qs += "limit=" + str(self.limit) + "&" + if self.status: + for status in self.status: + qs += "status=" + str(status) + "&" + + return qs[:-1] if qs else "" + + +class ProjectRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.CreateProjectRequest + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.name = None + self.description = None + + +class SQLRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.SQLRequest + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.sql = None + self.project = None + self.offset = None + self.limit = None + self.acceptPartial = None + + +class PrepareSqlRequest(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.PrepareSqlRequest + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.sql = None + self.project = None + self.offset = None + self.limit = None + self.acceptPartial = None + self.params = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + psr = PrepareSqlRequest() + + psr.sql = json_dict.get('sql') + psr.project = json_dict.get('project') + psr.offset = json_dict.get('offset') + psr.limit = json_dict.get('limit') + psr.acceptPartial = json_dict.get('acceptPartial') + if json_dict.get('params') and type(json_dict.get('params')) == list: + param_list = json_dict.get('params') + psr.params = [StateParam.from_json(param) for param in param_list] + + return psr + + +class StateParam(JsonSerializableObj): + """ + python class mapping to org.apache.kylin.rest.request.PrepareSqlRequest.StateParam + """ + + def __init__(self): + JsonSerializableObj.__init__(self) + + self.className = None + self.value = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + sp = StateParam() + + sp.className = json_dict.get('className') + sp.value = json_dict.get('value') + + return sp http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/models/rowkey.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/models/rowkey.py b/tools/kylin_client_tool/models/rowkey.py new file mode 100644 index 0000000..f370864 --- /dev/null +++ b/tools/kylin_client_tool/models/rowkey.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + + +class RowKeyDesc: + """ + python class mapping to org.apache.kylin.cube.model.RowKeyDesc + """ + + def __init__(self): + self.rowkey_columns = None + self.aggregation_groups = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + rkd = RowKeyDesc() + + # deserialize json for rowkey_columns + if json_dict.get('rowkey_columns') and type(json_dict.get('rowkey_columns')) == list: + rowkey_column_list = json_dict.get('rowkey_columns') + rkd.rowkey_columns = [RowKeyColDesc.from_json(rowkey_column) for rowkey_column in rowkey_column_list] + rkd.aggregation_groups = json_dict.get('aggregation_groups') + + return rkd + + @staticmethod + def get_from_dimensions(dimensions): + rkd = RowKeyDesc() + rkd.rowkey_columns = [] + rkd.aggregation_groups = [] + + if not dimensions: return rkd + + aggregation_group = [] + for dimension in dimensions: + rkd.rowkey_columns.append(RowKeyColDesc.get_from_dimension(dimension)) + aggregation_group.append(dimension.column[0]) + rkd.aggregation_groups = [aggregation_group] + + return rkd + + +class RowKeyColDesc: + """ + python class mapping to org.apache.kylin.cube.model.RowKeyColDesc + """ + + def __init__(self): + self.column = None + self.length = None + self.dictionary = None + self.mandatory = None + + @staticmethod + def from_json(json_dict): + if not json_dict or type(json_dict) != dict: return None + + rkcd = RowKeyColDesc() + + rkcd.column = json_dict.get('column') + rkcd.length = json_dict.get('length') + rkcd.dictionary = json_dict.get('dictionary') + rkcd.mandatory = json_dict.get('mandatory') + + return rkcd + + @staticmethod + def get_from_dimension(dimension): + rkcd = RowKeyColDesc() + + if dimension: + rkcd.column = dimension.column[0] + rkcd.length = 0 + rkcd.mandatory = False + rkcd.dictionary = 'true' + + return rkcd http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/rest/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/rest/__init__.py b/tools/kylin_client_tool/rest/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/rest/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/rest/apis.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/rest/apis.py b/tools/kylin_client_tool/rest/apis.py new file mode 100644 index 0000000..b4d7a63 --- /dev/null +++ b/tools/kylin_client_tool/rest/apis.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' + +import json +import sys +import requests +from requests.auth import HTTPBasicAuth +from settings import settings + + +class KylinRestApi: + cooikes = None + + def __init__(self): + self.host = settings.KYLIN_REST_HOST + self.port = settings.KYLIN_REST_PORT + self.user = settings.KYLIN_USER + self.password = settings.KYLIN_PASSWORD + self.rest_path_prefix = settings.KYLIN_REST_PATH_PREFIX + + if not KylinRestApi.cooikes: + KylinRestApi.cooikes = KylinRestApi.login(self) + + if not KylinRestApi.cooikes: + print "can't set cookies, exiting..." + sys.exit(1) + + @staticmethod + def login(kylin_rest_api): + if kylin_rest_api.user and kylin_rest_api.password: + # auth and get back cookies + headers = {} + headers['content-type'] = 'application/json' + req_response = requests.post(kylin_rest_api.get_api_url('user/authentication', ''), \ + auth=HTTPBasicAuth(kylin_rest_api.user, kylin_rest_api.password)) + return req_response.cookies + + return None + + @staticmethod + def is_response_ok(response): + return str(response.status_code) == '200' + + def get_api_url(self, uri, query_string): + return self.host + ':' + str(self.port) + self.rest_path_prefix \ + + '/' + uri + '?' + query_string + + def http_get(self, uri, query_string, headers=None): + api_url = self.get_api_url(uri, query_string) + + headers = headers if headers and type(headers) == dict else {} + headers['content-type'] = 'application/json' + + req_response = requests.get(api_url, headers=headers, cookies=KylinRestApi.cooikes) + + return req_response + + def http_post(self, uri, query_string, headers=None, payload=None): + api_url = self.get_api_url(uri, query_string) + + headers = headers if headers and type(headers) == dict else {} + headers['content-type'] = 'application/json' + + if payload: + data = payload if type(payload) == str else json.dumps(payload) + req_response = requests.post(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes) + else: + req_response = requests.post(api_url, headers=headers, cookies=KylinRestApi.cooikes) + + return req_response + + def http_put(self, uri, query_string, headers=None, payload=None): + api_url = self.get_api_url(uri, query_string) + + headers = headers if headers and type(headers) == dict else {} + headers['content-type'] = 'application/json' + + if payload: + data = payload if type(payload) == str else json.dumps(payload) + req_response = requests.put(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes) + else: + req_response = requests.put(api_url, headers=headers, cookies=KylinRestApi.cooikes) + + return req_response + + def http_delete(self, uri, query_string, headers=None, payload=None): + api_url = self.get_api_url(uri, query_string) + + headers = headers if headers and type(headers) == dict else {} + headers['content-type'] = 'application/json' + + # print payload + + if payload: + data = payload if type(payload) == str else json.dumps(payload) + req_response = requests.delete(api_url, data=data, headers=headers, cookies=KylinRestApi.cooikes) + else: + req_response = requests.delete(api_url, headers=headers, cookies=KylinRestApi.cooikes) + + # print str(req_response.json()) + + return req_response http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/scheduler/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/scheduler/__init__.py b/tools/kylin_client_tool/scheduler/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/scheduler/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua' http://git-wip-us.apache.org/repos/asf/kylin/blob/5593d824/tools/kylin_client_tool/scheduler/workers/__init__.py ---------------------------------------------------------------------- diff --git a/tools/kylin_client_tool/scheduler/workers/__init__.py b/tools/kylin_client_tool/scheduler/workers/__init__.py new file mode 100644 index 0000000..1b249ac --- /dev/null +++ b/tools/kylin_client_tool/scheduler/workers/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +__author__ = 'Huang, Hua'