This is an automated email from the ASF dual-hosted git repository. xxyu pushed a commit to branch kylin5 in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/kylin5 by this push: new 9bc9a36829 KYLIN-5465 add dashboard Ui and api (#2097) 9bc9a36829 is described below commit 9bc9a36829f272d2049a3a4691be9c696306f9e0 Author: Syleechan <38198463+syleec...@users.noreply.github.com> AuthorDate: Fri Jun 30 14:55:35 2023 +0800 KYLIN-5465 add dashboard Ui and api (#2097) * [KYLIN-5465] add dashboard Ui and api * [KYLIN-5465] fix dependency * [KYLIN-5465] fix job metrics not refresh issue * [KYLIN-5465] fix code smell * [KYLIN-5465] fix code smell 1 * [KYLIN-5465] fix code smell 1 * [KYLIN-5465] modify method usage of UI --------- Co-authored-by: cli2 <c...@ebay.com> --- kystudio/package.json | 4 +- kystudio/src/components/dashboard/BarEcharts.vue | 71 +++ kystudio/src/components/dashboard/LineEcharts.vue | 71 +++ kystudio/src/components/dashboard/chartOption.js | 99 +++ kystudio/src/components/dashboard/dashboard.vue | 702 +++++++++++++++++++++ kystudio/src/config/index.js | 5 + kystudio/src/directive/index.js | 2 +- kystudio/src/locale/en.js | 1 + kystudio/src/router/index.js | 5 + kystudio/src/service/api.js | 4 +- kystudio/src/service/dashboard.js | 23 + kystudio/src/store/dashboard.js | 20 + kystudio/src/store/index.js | 2 + kystudio/src/store/types.js | 7 + src/common-server/pom.xml | 8 + .../kylin/rest/controller/DashboardController.java | 92 +++ .../apache/kylin/rest/service/BasicService.java | 4 + .../kylin/common/response/MetricsResponse.java | 34 + .../src/main/resources/metadata-jdbc-h2.properties | 8 +- .../main/resources/metadata-jdbc-mysql.properties | 6 + .../metadata/query/JdbcQueryHistoryStore.java | 115 +++- .../kylin/metadata/query/QueryHistoryDAO.java | 8 + .../query/QueryHistoryRealizationTable.java | 5 + .../apache/kylin/metadata/query/QueryMetrics.java | 6 + .../kylin/metadata/query/QueryMetricsContext.java | 3 + .../kylin/metadata/query/RDBMSQueryHistoryDAO.java | 18 + .../metadata/query/util/QueryHisStoreUtil.java | 20 + .../metadata/query/RDBMSQueryHistoryDaoTest.java | 43 ++ .../kylin/rest/service/DashboardService.java | 221 +++++++ .../kylin/rest/service/QueryHistoryService.java | 59 +- .../kylin/rest/service/DashboardServiceTest.java | 394 ++++++++++++ 31 files changed, 2046 insertions(+), 14 deletions(-) diff --git a/kystudio/package.json b/kystudio/package.json index d7ed33f2e8..952b2aa7b2 100644 --- a/kystudio/package.json +++ b/kystudio/package.json @@ -18,8 +18,9 @@ "@tweenjs/tween.js": "17.1.1", "brace": "0.10.0", "d3": "3.5.17", + "daterangepicker": "^3.1.0", "dayjs": "1.7.7", - "echarts": "4.2.0-rc.1", + "echarts": "^4.9.0", "express-http-proxy": "0.11.0", "jquery": "3.5.0", "js-base64": "2.1.9", @@ -44,6 +45,7 @@ "vue-router": "2.8.1", "vue-virtual-scroller": "^1.0.10", "vue2-ace-editor": "0.0.3", + "vue2-daterange-picker": "^0.6.8", "vuex": "2.5.0" }, "devDependencies": { diff --git a/kystudio/src/components/dashboard/BarEcharts.vue b/kystudio/src/components/dashboard/BarEcharts.vue new file mode 100644 index 0000000000..70d1fa4409 --- /dev/null +++ b/kystudio/src/components/dashboard/BarEcharts.vue @@ -0,0 +1,71 @@ +<template> + <div :id="uuid" :style="style"></div> +</template> + +<script> +import * as echarts from 'echarts' +import {Component, Watch} from 'vue-property-decorator' +import Vue from 'vue' + +const idGen = () => { + return new Date().getTime() + 1 +} + +@Component({ + props: { + height: { + type: String, + default: '300px' + }, + width: { + type: String, + default: '450px' + }, + + options: { + type: Object, + default: null + } + } +}) + +export default class BarEcharts extends Vue { + @Watch('width') + onWithChange () { + if (this.myChart) { + setTimeout(() => { + this.myChart.resize({ + animation: { + duration: 300 + } + }) + }, 0) + } + } + + @Watch('options', {deep: true}) + onOptionChange () { + if (this.myChart) { + this.myChart.dispose() + this.myChart = echarts.init(document.getElementById(this.uuid)) + this.myChart.setOption(this.options, {notMerge: true}) + } + } + + get style () { + return { + height: this.height, + width: this.width + } + } + + created () { + this.uuid = idGen() + } + + mounted () { + this.myChart = echarts.init(document.getElementById(this.uuid)) + this.myChart.setOption(this.options) + } +} +</script> diff --git a/kystudio/src/components/dashboard/LineEcharts.vue b/kystudio/src/components/dashboard/LineEcharts.vue new file mode 100644 index 0000000000..478f34123d --- /dev/null +++ b/kystudio/src/components/dashboard/LineEcharts.vue @@ -0,0 +1,71 @@ +<template> + <div :id="uuid" :style="style"></div> +</template> + +<script> +import * as echarts from 'echarts' +import {Component, Watch} from 'vue-property-decorator' +import Vue from 'vue' + +const idGen = () => { + return new Date().getTime() - 1 +} + +@Component({ + props: { + height: { + type: String, + default: '300px' + }, + width: { + type: String, + default: '450px' + }, + + options: { + type: Object, + default: null + } + } +}) + +export default class LineEcharts extends Vue { + @Watch('width') + onWithChange () { + if (this.myChart) { + setTimeout(() => { + this.myChart.resize({ + animation: { + duration: 300 + } + }) + }, 0) + } + } + + @Watch('options', {deep: true}) + onOptionChange () { + if (this.myChart) { + this.myChart.dispose() + this.myChart = echarts.init(document.getElementById(this.uuid)) + this.myChart.setOption(this.options, {notMerge: true}) + } + } + + get style () { + return { + height: this.height, + width: this.width + } + } + + created () { + this.uuid = idGen() + } + + mounted () { + this.myChart = echarts.init(document.getElementById(this.uuid)) + this.myChart.setOption(this.options) + } +} +</script> diff --git a/kystudio/src/components/dashboard/chartOption.js b/kystudio/src/components/dashboard/chartOption.js new file mode 100644 index 0000000000..936c2b0441 --- /dev/null +++ b/kystudio/src/components/dashboard/chartOption.js @@ -0,0 +1,99 @@ +export default { + barChartOptions: (xData, yData) => { + return { + title: { + text: '', + left: 'center', + top: -5 + }, + height: 255, + grid: { + top: 25, + left: 45, + right: 15, + bottom: 0 + }, + tooltip: { + trigger: 'axis', + axisPointer: { + type: 'shadow' + } + }, + xAxis: [ + { + type: 'category', + data: xData || [], + axisTick: { + alignWithLabel: true + } + } + ], + yAxis: [ + { + type: 'value', + axisTick: false + } + ], + series: [ + { + name: 'value', + type: 'bar', + barWidth: '60%', + data: yData || [], + label: { + show: false + }, + itemStyle: { + normal: { + color: function (params) { + const colorList = ['#5470c6', '#91cc75', '#fac858', '#ee6666', '#73c0de', '#3ba272', '#fc8452', '#9a60b4', '#ea7ccc'] + return colorList[params.dataIndex] + } + } + } + } + ] + } + }, + lineChartOptions: (xData, yData) => { + return { + title: { + text: '', + left: 'center', + top: 15 + }, + grid: { + top: 45, + left: 45, + right: 15, + height: 235 + }, + tooltip: { + trigger: 'axis' + }, + xAxis: [ + { + type: 'category', + data: xData || [], + axisTick: { + alignWithLabel: true + } + } + ], + yAxis: [ + { + type: 'value', + axisTick: false + } + ], + series: [ + { + name: 'value', + type: 'line', + color: '#5470c6', + data: yData || [] + } + ] + } + } +} diff --git a/kystudio/src/components/dashboard/dashboard.vue b/kystudio/src/components/dashboard/dashboard.vue new file mode 100644 index 0000000000..986bd635a8 --- /dev/null +++ b/kystudio/src/components/dashboard/dashboard.vue @@ -0,0 +1,702 @@ +<template> + <div class="dashboard" v-loading="isLoading"> + <header class="dashboard-header"> + <h1 class="ksd-title-label" v-if="projectSettings">{{projectSettings.project || projectSettings.alias}}</h1> + <div id="datepicker"> + <date-range-picker v-model="pickerDates" :locale-data="{format: 'yyyy-mm-dd'}" @update="changeDateRange"> + <template v-slot:input="picker" style="min-width: 350px;" > + {{ pickerDates.startDate }} - {{ pickerDates.endDate }} + </template> + </date-range-picker> + </div> + </header> + + <section class="dashboard-body" v-if="projectSettings"> + <!-- Model Metrics --> + <div class = "row"> + <div class="el-col-sm-1"> + <el-tooltip placement="bottom" :content="'As of ' + currentTime"> + <div class="square-big" tool-palcement="bottom"> + <div class = "title"> + TOTAL MODEL COUNT + </div> + <div class="metric" v-if="totalModel || totalModel === 0"> + {{totalModel}} + </div> + <div class="metric" v-if="!totalModel && (totalModel !== 0)"> + -- + </div> + <a class="description" @click="toModel()"> + More Details + </a> + </div> + </el-tooltip> + <el-tooltip placement="bottom"> + <div slot="content">{{"Max:" + maxExpansionRate + " | Min:" + minExpansionRate}}<br/>{{"As of " + currentTime}}</div> + <div class="square-big" tool-palcement="bottom"> + <div class = "title"> + AVG MODEL EXPANSION + </div> + <div class="metric" v-if="avgExpansionRate || avgExpansionRate === 0"> + {{avgExpansionRate}} + </div> + <div class="metric" v-if="!avgExpansionRate && (avgExpansionRate !== 0)"> + -- + </div> + <a class="description" @click="toModel()"> + More Details + </a> + </div> + </el-tooltip> + </div> + </div> + <!-- Query Metrics --> + <div class = "col-sm-10"> + <div class = "row"> + <div class = "col-sm-2" style="width: 20%"> + <div class = "square" :class="{'square-active': currentSquare === 'queryCount'}" @click="queryCountChart()"> + <div class = "title"> + QUERY<br/>COUNT + </div> + <div class = "metric" v-if="queryCount || queryCount === 0"> + {{queryCount}} + </div> + <div class="metric" v-if="!queryCount && (queryCount !== 0)"> + -- + </div> + <a class="description" @click="toQuery()"> + More Details + </a> + </div> + </div> + <div class="col-sm-2" style="width: 20%"> + <div class="square1" :class="{'square-active': currentSquare === 'queryAvg'}" @click="queryAvgChart()" > + <div class="title"> + AVG QUERY LATENCY + </div> + <div class="metric" v-if="avgQueryLatency || avgQueryLatency === 0"> + {{numFilter(avgQueryLatency / 1000)}}<span class="unit"> sec</span> + </div> + <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency !== 0)"> + -- + </div> + <a class="description" @click="toQuery()"> + More Details + </a> + </div> + </div> + <!-- Job Metrics --> + <div class="col-sm-2" style="width: 20%"> + <div class="square2" :class="{'square-active': currentSquare === 'jobCount'}" @click="jobCountChart()" > + <div class="title"> + JOB<br/>COUNT + </div> + <div class="metric" v-if="jobCount || jobCount === 0"> + {{jobCount}} + </div> + <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency !== 0)"> + -- + </div> + <a class="description" @click="toJob()"> + More Details + </a> + </div> + </div> + <div class="col-sm-2" style="width: 20%"> + <div class="square3" :class="{'square-active': currentSquare === 'jobAvgBuild'}" @click="jobAvgBuildChart()" > + <div class="title"> + AVG BUILD TIME PER {{jobUnit}} + </div> + <div class="metric" v-if="jobAvgBuildTime || jobAvgBuildTime === 0"> + {{jobAvgBuildTime}}<span class="unit"> {{jobTimeUnit}}</span> + </div> + <div class="metric" v-if="!avgQueryLatency && (avgQueryLatency !== 0)"> + -- + </div> + <a class="description" @click="toJob()"> + More Details + </a> + </div> + </div> + </div> + </div> + <!-- charts --> + <div class="row"> + <div class="col-sm-2" v-if="barChartOptions"> + <div class="barChartSquare"> + <div class="form-control"> + Show Value: <input type="checkbox" v-model="barChartOptions.series[0].label.show" @click="showBarChartValue()"> + </div> + <BarEcharts :options="barChartOptions"/> + </div> + </div> + <div class="col-sm-2" v-if="lineChartOptions"> + <div class="lineChartSquare"> + <select class="select-control" + v-model="currentSelectFilter" + @change="getCurrentSelectedLineChart"> + <option v-for="filter in chartFilter" :value="filter.name"> + {{filter.name}} + </option> + </select> + <LineEcharts :options="lineChartOptions"/> + </div> + </div> + </div> + </section> + <kylin-empty-data v-else> + </kylin-empty-data> + </div> +</template> + +<script> +import Vue from 'vue' +import {mapActions, mapGetters} from 'vuex' +import { Component } from 'vue-property-decorator' +import {handleError, handleSuccessAsync} from '../../util/index' +import DateRangePicker from 'vue2-daterange-picker' +import 'vue2-daterange-picker/dist/vue2-daterange-picker.css' +import moment from 'moment' +import baseOptions from './chartOption' +import BarEcharts from './BarEcharts' +import LineEcharts from './LineEcharts' +Vue.component('date-range-picker', DateRangePicker) + +@Component({ + methods: { + ...mapActions({ + getModelStatistics: 'GET_MODEL_STATISTICS', + getQueryStatistics: 'GET_QUERY_STATISTICS', + fetchProjectSettings: 'FETCH_PROJECT_SETTINGS', + getChartData: 'GET_CHART_DATA', + getJobStatistics: 'GET_JOB_STATISTICS' + }) + }, + computed: { + ...mapGetters([ + 'currentSelectedProject', + 'currentProjectData' + ]) + }, + components: {DateRangePicker, BarEcharts, LineEcharts}, + watch: { + pickerDates: { + handler (newValue, oldValue) { + }, + deep: true + }, + barChartOptions: { + handler (newValue, oldValue) { + }, + deep: true + }, + lineChartOptions: { + handler (newValue, oldValue) { + }, + deep: true + }, + currentSelectFilter: { + handler (newValue, oldValue) { + }, + deep: true + } + } +}) + +export default class Dashboard extends Vue { + // name = 'Dashboard' + isLoading = false + projectSettings = null; + totalModel = 0; + avgExpansionRate = 0; + minExpansionRate = 0; + maxExpansionRate = 0; + currentTime = moment().format('YYYY-MM-DD'); + queryCount = 0; + avgQueryLatency = 0; + currentSquare = ''; + pickerDates = { + startDate: '', + endDate: '' + }; + options = null + // category: JOB, QUERY + // metric: QUERY:{count, avg_query_latency} + // dimension : filter by model or date(day, week) + barChart = null; + lineChart = null; + barChartOptions = null; + lineChartOptions = null; + category = ['QUERY', 'JOB']; + metric = [ + {name: 'query count', value: 'QUERY_COUNT'}, + {name: 'avg query latency', value: 'AVG_QUERY_LATENCY'}, + {name: 'job count', value: 'JOB_COUNT'}, + {name: 'avg build time', value: 'AVG_JOB_BUILD_TIME'} + ]; + dimension = [ + {name: 'project', value: 'PROJECT'}, + {name: 'model', value: 'MODEL'}, + {name: 'day', value: 'DAY'}, + {name: 'week', value: 'WEEK'}, + {name: 'month', value: 'MONTH'} + ]; + chartFilter = [ + {name: 'Daily', value: 'day'}, + {name: 'Weekly', value: 'week'}, + {name: 'Monthly', value: 'month'} + ]; + currentSelectFilter = this.chartFilter[1].name + barChartCategory = null; + barChartMetric = null; + lineChartCategory = null; + lineChartMetric = null; + chartType = ''; + barChartDimension = this.dimension[1].value; + lineChartDimension = this.dimension[3].value; + jobUnit = 'MB'; + jobTimeUnit = 'sec'; + jobCount = 0; + jobTotalByteSize = 0; + jobTotalLatency = 0; + jobAvgBuildTime = 0; + toModel () { + this.$router.push('/studio/model') + } + toQuery () { + this.$router.push('/query/queryhistory') + } + toJob () { + this.$router.push('/monitor/job') + } + showLoading () { + this.isLoading = true + } + changeDateRange (val) { + this.pickerDates = val + this.pickerDates.startDate = moment(this.pickerDates.startDate).format('YYYY-MM-DD') + this.pickerDates.endDate = moment(this.pickerDates.endDate).format('YYYY-MM-DD') + this.refresh() + } + refresh () { + this.getModelInfo() + this.getQueryMetrics() + this.createCharts() + this.getJobMetrics() + } + hideLoading () { + this.isLoading = false + } + numFilter (value) { + const realVal = parseFloat(value).toFixed(2) + return realVal + } + async getCurrentSettings () { + this.showLoading() + try { + const projectName = this.currentProjectData.name + const response = await this.fetchProjectSettings({ projectName }) + const result = await handleSuccessAsync(response) + const quotaSize = result.storage_quota_size / 1024 / 1024 / 1024 / 1024 + this.projectSettings = {...result, storage_quota_tb_size: quotaSize.toFixed(2)} + } catch (e) { + handleError(e) + } + this.hideLoading() + } + getModelInfo () { + this.getModelStatistics({projectName: this.currentProjectData.name, modelName: null}).then((res) => { + const data = res.data + if (data) { + this.totalModel = data.totalModel + this.avgExpansionRate = this.numFilter(data.avgModelExpansion) + this.minExpansionRate = this.numFilter(data.minModelExpansion) + this.maxExpansionRate = this.numFilter(data.maxModelExpansion) + } + }) + } + getQueryMetrics () { + this.getQueryStatistics({projectName: this.currentProjectData.name, + startTime: String(this.pickerDates.startDate), + endTime: String(this.pickerDates.endDate), + modelName: null}).then((res) => { + const data = res.data + if (data) { + this.queryCount = data.queryCount + this.avgQueryLatency = data.avgQueryLatency + } + }) + } + getJobMetrics () { + this.getJobStatistics({projectName: this.currentProjectData.name, + startTime: String(this.pickerDates.startDate), + endTime: String(this.pickerDates.endDate), + modelName: null}).then((res) => { + const data = res.data + if (data) { + this.jobCount = data.jobCount + this.jobTotalByteSize = data.jobTotalByteSize + this.jobTotalLatency = data.jobTotalLatency + if (this.jobTotalByteSize > 0) { + this.jobAvgBuildTime = this.numFilter(this.determineAvgJobBuildTimeUnit( + this.jobTotalLatency / this.jobTotalByteSize)) + } else { + this.jobAvgBuildTime = 0 + } + } + }) + } + data () { + const startDate = moment().subtract('days', 5).format('YYYY-MM-DD') + const endDate = moment().subtract('days', -1).format('YYYY-MM-DD') + return { + pickerDates: { + startDate, + endDate + } + } + } + mounted () { + this.getCurrentSettings() + this.getModelInfo() + this.getQueryMetrics() + this.queryCountChart() + this.getJobMetrics() + } + // create chart + queryCountChart () { + this.currentSquare = 'queryCount' + this.barChartCategory = this.category[0] + this.barChartMetric = this.metric[0].value + this.lineChartCategory = this.category[0] + this.lineChartMetric = this.metric[0].value + this.createCharts() + } + queryAvgChart () { + this.currentSquare = 'queryAvg' + this.barChartCategory = this.category[0] + this.barChartMetric = this.metric[1].value + this.lineChartCategory = this.category[0] + this.lineChartMetric = this.metric[1].value + this.createCharts() + } + jobCountChart () { + this.currentSquare = 'jobCount' + this.barChartCategory = this.category[1] + this.barChartMetric = this.metric[2].value + this.lineChartCategory = this.category[1] + this.lineChartMetric = this.metric[2].value + this.createCharts() + } + jobAvgBuildChart () { + this.currentSquare = 'jobAvgBuild' + this.barChartCategory = this.category[1] + this.barChartMetric = this.metric[3].value + this.lineChartCategory = this.category[1] + this.lineChartMetric = this.metric[3].value + this.createCharts() + } + createCharts () { + this.createChart(this.barChartDimension, this.barChartCategory, this.barChartMetric, 'bar') + this.createChart(this.lineChartDimension, this.lineChartCategory, this.lineChartMetric, 'line') + } + async createChart (dimension, category, metric, chartType) { + const startTime = String(this.pickerDates.startDate) + const endTime = String(this.pickerDates.endDate) + if (chartType === 'bar') { + let xdata = [] + let ydata = [] + this.barChartOptions = baseOptions.barChartOptions(xdata, ydata) + this.barChartOptions.series.type = chartType + const data = await this.getChartDataResult(category, dimension, metric, + this.currentProjectData.name, null, startTime, endTime) + for (let key in data) { + xdata.push(key) + if (this.barChartMetric === this.metric[1].value) { + ydata.push(this.numFilter(data[key] / 1000)) + } else if (this.barChartMetric === this.metric[3].value) { + ydata.push(this.numFilter(this.determineAvgJobBuildTime(data[key]))) + } else { + ydata.push(data[key]) + } + } + this.barChartOptions.xAxis.data = xdata + this.barChartOptions.series.data = ydata + this.barChartOptions.title.text = metric + ' BY ' + dimension + console.log(this.barChartOptions) + } else if (chartType === 'line') { + let xdata = [] + let ydata = [] + this.lineChartOptions = baseOptions.lineChartOptions(xdata, ydata) + this.lineChartOptions.series.type = chartType + const data = await this.getChartDataResult(category, dimension, metric, + this.currentProjectData.name, null, startTime, endTime) + for (let key in data) { + xdata.push(key) + if (this.lineChartMetric === this.metric[1].value) { + ydata.push(this.numFilter(data[key] / 1000)) + } else if (this.lineChartMetric === this.metric[3].value) { + ydata.push(this.numFilter(this.determineAvgJobBuildTime(data[key]))) + } else { + ydata.push(data[key]) + } + } + this.lineChartOptions.title.text = metric + ' BY ' + dimension + this.lineChartOptions.xAxis.data = xdata + this.lineChartOptions.series.data = ydata + } + } + // if data >= 100 ms/byte then transform display unit sec/MB to min/MB + determineAvgJobBuildTimeUnit (data) { + let avgTime = data < 100 ? data * 1024 * 1024 / 1000 : data * 1024 * 1024 / 1000 / 60 + this.jobTimeUnit = data < 100 ? 'sec' : 'min' + return avgTime + } + determineAvgJobBuildTime (data) { + return this.jobTimeUnit === 'sec' ? data * 1024 * 1024 / 1000 : data * 1024 * 1024 / 1000 / 60 + } + async getChartDataResult (category, dimension, metric, projectName, modelName, startTime, endTime) { + const res = await this.getChartData({ + category: category, + dimension: dimension, + metric: metric, + projectName: projectName, + modelName: modelName, + startTime: startTime, + endTime: endTime + }) + if (res) return res.data + return null + } + // refresh line chart by dimension + getCurrentSelectedLineChart () { + let temp = this.currentSelectFilter + for (let k in this.chartFilter) { + if (this.chartFilter[k].name === temp) { + temp = this.chartFilter[k].value + break + } + } + for (let k in this.dimension) { + if (temp === this.dimension[k].name) { + this.lineChartDimension = this.dimension[k].value + break + } + } + this.createChart(this.lineChartDimension, this.lineChartCategory, this.lineChartMetric, 'line') + } + // checkbox display + showBarChartValue () { + this.barChartOptions.series[0].label.show = this.barChartOptions.series[0].label.show === false + } +} +</script> + +<style lang="less"> +.dashboard { + height: 100%; + padding: 20px; + .dashboard-header { + margin-bottom: 20px; + } + .dashboard-header h1 { + font-size: 18px; + } + .dashboard-body { + height: 100%; + } + .square-big { + border: 5px solid #ddd; + text-align: center; + width: 185px; + height: 225px; + padding: 25px 0; + margin-bottom: 30px; + .title { + font-size: 24px; + height: 55px; + } + .metric { + padding: 15px 0; + font-size: 50px ; + font-weight: bolder; + height: 100px; + color: #06d; + .unit { + font-size: 35px; + } + } + .description { + font-size: 18px; + color: #6a6a6a; + } + } + .square { + border: 2px solid #ddd; + text-align: center; + width: 170px; + height: 165px; + cursor: zoom-in; + padding-top: 20px; + padding-bottom: 15px; + margin-left: 230px; + .title { + font-size: 20px; + height: 65px; + } + .metric { + font-size: 35px ; + font-weight: bolder; + color: #8b1; + display: inline-block; + white-space: nowrap; + .unit { + font-size: 16px; + } + } + .description { + font-size: 15px; + color: #6a6a6a; + display: block; + } + } + .square-active { + border: 2px solid #6a6a6a; + } + .square1 { + margin-top: -204px; + border: 2px solid #ddd; + text-align: center; + width: 170px; + height: 165px; + cursor: zoom-in; + padding-top: 20px; + padding-bottom: 15px; + margin-left: 440px; + .title { + font-size: 20px; + height: 65px; + } + .metric { + font-size: 35px ; + font-weight: bolder; + color: #8b1; + display: inline-block; + white-space: nowrap; + .unit { + font-size: 16px; + } + } + .description { + font-size: 15px; + color: #6a6a6a; + display: block; + } + } + .square2 { + margin-top: -204px; + border: 2px solid #ddd; + text-align: center; + width: 170px; + height: 165px; + cursor: zoom-in; + padding-top: 20px; + padding-bottom: 15px; + margin-left: 650px; + .title { + font-size: 20px; + height: 65px; + } + .metric { + font-size: 35px ; + font-weight: bolder; + color: #8b1; + display: inline-block; + white-space: nowrap; + .unit { + font-size: 16px; + } + } + .description { + font-size: 15px; + color: #6a6a6a; + display: block; + } + } + .square3 { + margin-top: -204px; + border: 2px solid #ddd; + text-align: center; + width: 170px; + height: 165px; + cursor: zoom-in; + padding-top: 20px; + padding-bottom: 15px; + margin-left: 860px; + .title { + font-size: 20px; + height: 65px; + } + .metric { + font-size: 35px ; + font-weight: bolder; + color: #8b1; + display: inline-block; + white-space: nowrap; + .unit { + font-size: 16px; + } + } + .description { + font-size: 15px; + color: #6a6a6a; + display: block; + } + } + .barChartSquare { + margin-top: 30px; + border: 2px solid #ddd; + text-align: center; + width: 450px; + height: 335px; + cursor: zoom-in; + padding-top: 10px; + padding-bottom: 15px; + margin-left: 230px; + } + .lineChartSquare { + margin-top: -365px; + border: 2px solid #ddd; + text-align: right; + width: 450px; + height: 360px; + cursor: zoom-in; + margin-left: 710px; + } + .form-control { + border-radius: 0px !important; + box-shadow: none; + border-color: #5470c6; + padding-top: -25px; + padding-bottom: 15px; + width: 105px; + } + .select-control { + border-radius: 0px !important; + box-shadow: none; + border-color: #5470c6; + width: 80px; + padding: 4px; + } + #datepicker { + font-family: "Avenir", Helvetica, Arial, sans-serif; + -webkit-font-smoothing: antialiased; + -moz-osx-font-smoothing: grayscale; + text-align: right; + color: #2c3e50; + margin-right: 280px; + height: 24px; + margin-top: -25px; + } +} +</style> diff --git a/kystudio/src/config/index.js b/kystudio/src/config/index.js index b8ecc8af02..f35e572b7d 100644 --- a/kystudio/src/config/index.js +++ b/kystudio/src/config/index.js @@ -89,6 +89,11 @@ export const menusData = [ name: 'group', path: '/admin/group', icon: 'el-ksd-icon-nav_user_group_24' + }, + { + name: 'dashboard', + path: '/dashboard', + icon: 'el-ksd-icon-nav_dashboard_24' } ] diff --git a/kystudio/src/directive/index.js b/kystudio/src/directive/index.js index 86b2c3cecb..f66286e96d 100644 --- a/kystudio/src/directive/index.js +++ b/kystudio/src/directive/index.js @@ -4,7 +4,6 @@ import Scrollbar from 'smooth-scrollbar' import store from '../store' import { stopPropagation, on } from 'util/event' import { closestElm } from 'util' -// import commonTip from 'components/common/common_tip' import ElementUI from 'kyligence-kylin-ui' const nodeList = [] const ctx = '@@clickoutsideContext' @@ -707,6 +706,7 @@ Vue.directive('custom-tooltip', { } }) + // MutationObserver 方式监听 dom attrs 的改变 function licenseDom (id) { if (!parentList[id]) return diff --git a/kystudio/src/locale/en.js b/kystudio/src/locale/en.js index 22357326e7..25a367e66d 100644 --- a/kystudio/src/locale/en.js +++ b/kystudio/src/locale/en.js @@ -469,6 +469,7 @@ exports.default = { insight: 'Insight', monitor: 'Monitor', system: 'System', + dashboard: 'Dashboard', setting: 'Setting', project: 'Project', auto: 'Auto-Modeling', diff --git a/kystudio/src/router/index.js b/kystudio/src/router/index.js index 4c75e0a7e7..29e93ab054 100644 --- a/kystudio/src/router/index.js +++ b/kystudio/src/router/index.js @@ -10,6 +10,7 @@ import Insight from 'components/query/insight' import queryHistory from 'components/query/query_history' import jobs from 'components/monitor/batchJobs/jobs' import streamingJobs from 'components/monitor/streamingJobs/streamingJobs' +import dashboard from 'components/dashboard/dashboard' import { bindRouterGuard } from './routerGuard.js' Vue.use(Router) @@ -54,6 +55,10 @@ let routerOptions = { name: 'Setting', path: '/setting', component: () => import('../components/setting/setting.vue') + }, { + name: 'Dashboard', + path: '/dashboard', + component: dashboard }, { name: 'Source', path: 'studio/source', diff --git a/kystudio/src/service/api.js b/kystudio/src/service/api.js index 7a22502816..ffaa073188 100644 --- a/kystudio/src/service/api.js +++ b/kystudio/src/service/api.js @@ -8,6 +8,7 @@ import userApi from './user' import systemApi from './system' import datasourceApi from './datasource' import monitorApi from './monitor' +import dashboardApi from './dashboard' // console.log(base64) Vue.use(VueResource) export default { @@ -18,5 +19,6 @@ export default { user: userApi, system: systemApi, datasource: datasourceApi, - monitor: monitorApi + monitor: monitorApi, + dashboard: dashboardApi } diff --git a/kystudio/src/service/dashboard.js b/kystudio/src/service/dashboard.js new file mode 100644 index 0000000000..a1102a2da7 --- /dev/null +++ b/kystudio/src/service/dashboard.js @@ -0,0 +1,23 @@ +import Vue from 'vue' +import VueResource from 'vue-resource' +import { apiUrl } from '../config' + +Vue.use(VueResource) + +export default { + getModelStatistics: (para) => { + return Vue.resource(apiUrl + 'dashboard/metric/model').get(para) + }, + getQueryStatistics: (para) => { + return Vue.resource(apiUrl + 'dashboard/metric/query').get(para) + }, + // category: JOB, QUERY + // dimension: QUERY:{count, avg_query_latency} + // metric: filter by model or date(day, week) + getChartData: (category, dimension, metric, projectName, modelName, startTime, endTime) => { + return Vue.resource(apiUrl + `dashboard/chart/${category}/${dimension}/${metric}?projectName=${projectName}&modelName=${modelName}&startTime=${startTime}&endTime=${endTime}`).get() + }, + getJobStatistics: (para) => { + return Vue.resource(apiUrl + 'dashboard/metric/job').get(para) + } +} diff --git a/kystudio/src/store/dashboard.js b/kystudio/src/store/dashboard.js new file mode 100644 index 0000000000..ca49dcba6a --- /dev/null +++ b/kystudio/src/store/dashboard.js @@ -0,0 +1,20 @@ +import * as types from './types' +import api from '../service/api' + +export default { + state: {}, + actions: { + [types.GET_MODEL_STATISTICS]: function ({commit, state}, param) { + return api.dashboard.getModelStatistics(param) + }, + [types.GET_QUERY_STATISTICS]: function ({commit, state}, param) { + return api.dashboard.getQueryStatistics(param) + }, + [types.GET_CHART_DATA]: function ({commit, state}, param) { + return api.dashboard.getChartData(param.category, param.metric, param.dimension, param.projectName, param.modelName, param.startTime, param.endTime) + }, + [types.GET_JOB_STATISTICS]: function ({commit, state}, param) { + return api.dashboard.getJobStatistics(param) + } + } +} diff --git a/kystudio/src/store/index.js b/kystudio/src/store/index.js index ec0956b973..dc51ad0d43 100644 --- a/kystudio/src/store/index.js +++ b/kystudio/src/store/index.js @@ -12,6 +12,7 @@ import system from './system' import monitor from './monitor' import capacity from './capacity' import * as actionTypes from './types' +import dashboard from './dashboard' export default new Vuex.Store({ modules: { @@ -24,6 +25,7 @@ export default new Vuex.Store({ system: system, monitor: monitor, capacity: capacity, + dashboard: dashboard, modals: {} } }) diff --git a/kystudio/src/store/types.js b/kystudio/src/store/types.js index 7ce238b2e7..d852acf269 100644 --- a/kystudio/src/store/types.js +++ b/kystudio/src/store/types.js @@ -439,3 +439,10 @@ export const RESET_CAPACITY_DATA = 'RESET_CAPACITY_DATA' export const SET_NODES_LIST = 'SET_NODES_LIST' export const SET_NODES_INFOS = 'SET_NODES_INFOS' export const SET_SYSTEM_CAPACITY_INFO = 'SET_SYSTEM_CAPACITY_INFO' + +// dashboard +export const GET_MODEL_STATISTICS = 'GET_MODEL_STATISTICS' +export const GET_QUERY_STATISTICS = 'GET_QUERY_STATISTICS' +export const GET_CHART_DATA = 'GET_CHART_DATA' +export const GET_JOB_STATISTICS = 'GET_JOB_STATISTICS' + diff --git a/src/common-server/pom.xml b/src/common-server/pom.xml index d85c09ad2c..b3dc4ed315 100644 --- a/src/common-server/pom.xml +++ b/src/common-server/pom.xml @@ -142,5 +142,13 @@ <artifactId>junit-vintage-engine</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-core-metrics</artifactId> + </dependency> + <dependency> + <groupId>org.apache.kylin</groupId> + <artifactId>kylin-modeling-service</artifactId> + </dependency> </dependencies> </project> diff --git a/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java b/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java new file mode 100644 index 0000000000..0c2f2648d7 --- /dev/null +++ b/src/common-server/src/main/java/org/apache/kylin/rest/controller/DashboardController.java @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.controller; + +import org.apache.kylin.common.response.MetricsResponse; +import org.apache.kylin.query.exception.UnsupportedQueryException; +import org.apache.kylin.rest.service.DashboardService; +import org.apache.kylin.rest.service.ModelService; +import org.apache.kylin.rest.service.QueryService; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Controller; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestMethod; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.ResponseBody; + +@Controller +@RequestMapping(value = "/api/dashboard") +public class DashboardController extends NBasicController { + public static final Logger logger = LoggerFactory.getLogger(DashboardController.class); + + @Autowired + DashboardService dashboardService; + + @Autowired + ModelService modelService; + + @Autowired + QueryService queryService; + + + @RequestMapping(value = "/metric/model", method = { RequestMethod.GET }) + @ResponseBody + public MetricsResponse getCubeMetrics(@RequestParam(value = "projectName") String projectName, + @RequestParam(value = "modelName", required = false) String modelName) { + dashboardService.checkAuthorization(projectName); + return dashboardService.getModelMetrics(projectName, modelName); + } + + @RequestMapping(value = "/metric/query", method = RequestMethod.GET) + @ResponseBody + public MetricsResponse getQueryMetrics(@RequestParam(value = "projectName", required = false) String projectName, + @RequestParam(value = "modelName", required = false) String cubeName, + @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) { + dashboardService.checkAuthorization(projectName); + return dashboardService.getQueryMetrics(projectName, startTime, endTime); + } + + @RequestMapping(value = "/metric/job", method = RequestMethod.GET) + @ResponseBody + public MetricsResponse getJobMetrics(@RequestParam(value = "projectName", required = false) String projectName, + @RequestParam(value = "modelName", required = false) String cubeName, + @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) { + dashboardService.checkAuthorization(projectName); + return dashboardService.getJobMetrics(projectName, startTime, endTime); + } + + @RequestMapping(value = "/chart/{category}/{metric}/{dimension}", method = RequestMethod.GET) + @ResponseBody + public MetricsResponse getChartData(@PathVariable String dimension, @PathVariable String metric, + @PathVariable String category, @RequestParam(value = "projectName", required = false) String projectName, + @RequestParam(value = "modelName", required = false) String cubeName, + @RequestParam(value = "startTime") String startTime, @RequestParam(value = "endTime") String endTime) { + dashboardService.checkAuthorization(projectName); + try { + return dashboardService.getChartData(category, projectName, startTime, endTime, dimension, metric); + } catch (Exception e) { + throw new UnsupportedQueryException("Category or Metric is not right: { " + e.getMessage() + " }"); + } + } + + +} diff --git a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java index f157021be1..c08995edc6 100644 --- a/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java +++ b/src/common-service/src/main/java/org/apache/kylin/rest/service/BasicService.java @@ -161,4 +161,8 @@ public abstract class BasicService { return null; }, project); } + + public NProjectManager getProjectManager(){ + return NProjectManager.getInstance(getConfig()); + } } diff --git a/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java b/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java new file mode 100644 index 0000000000..1eda0e22b5 --- /dev/null +++ b/src/core-common/src/main/java/org/apache/kylin/common/response/MetricsResponse.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.common.response; + +import java.util.TreeMap; + +public class MetricsResponse extends TreeMap<String, Float> { + + public static final long serialVersionUID = 1L; + + public void increase(String key, Float increased) { + if (this.containsKey(key)) { + this.put(key, (this.get(key) + increased)); + } else { + this.put(key, increased); + } + } +} diff --git a/src/core-common/src/main/resources/metadata-jdbc-h2.properties b/src/core-common/src/main/resources/metadata-jdbc-h2.properties index 77328950c1..21a9ddd38d 100644 --- a/src/core-common/src/main/resources/metadata-jdbc-h2.properties +++ b/src/core-common/src/main/resources/metadata-jdbc-h2.properties @@ -65,12 +65,16 @@ create.queryhistoryrealization.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \ `duration` BIGINT, \ `query_time` BIGINT, \ `project_name` VARCHAR(255), \ + `query_first_day_of_month` BIGINT, \ + `query_first_day_of_week` BIGINT, \ + `query_day` BIGINT, \ primary key (`id`,`project_name`) \ ); - create.queryhistoryrealization.store.tableindex1=CREATE INDEX %s_ix1 ON %s ( query_time ); create.queryhistoryrealization.store.tableindex2=CREATE INDEX %s_ix2 ON %s ( model ); - +create.queryhistoryrealization.store.tableindex3=CREATE INDEX %s_ix3 ON %s ( query_first_day_of_month ); +create.queryhistoryrealization.store.tableindex4=CREATE INDEX %s_ix4 ON %s ( query_first_day_of_week ); +create.queryhistoryrealization.store.tableindex5=CREATE INDEX %s_ix5 ON %s ( query_day ); # RAW RECOMMENDATION STORE create.rawrecommendation.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \ `id` int not null auto_increment, \ diff --git a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties index 6a2df4c3b1..48299736ce 100644 --- a/src/core-common/src/main/resources/metadata-jdbc-mysql.properties +++ b/src/core-common/src/main/resources/metadata-jdbc-mysql.properties @@ -87,11 +87,17 @@ create.queryhistoryrealization.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \ `duration` BIGINT, \ `query_time` BIGINT, \ `project_name` VARCHAR(255), \ + `query_first_day_of_month` BIGINT, \ + `query_first_day_of_week` BIGINT, \ + `query_day` BIGINT, \ primary key (`id`,`project_name`) \ ) DEFAULT CHARSET=utf8; create.queryhistoryrealization.store.tableindex1=ALTER table %s ADD INDEX %s_ix1(`query_time`); create.queryhistoryrealization.store.tableindex2=ALTER table %s ADD INDEX %s_ix2(`model`); +create.queryhistoryrealization.store.tableindex3=ALTER table %s ADD INDEX %s_ix3(`query_first_day_of_month`); +create.queryhistoryrealization.store.tableindex4=ALTER table %s ADD INDEX %s_ix4(`query_first_day_of_week`); +create.queryhistoryrealization.store.tableindex5=ALTER table %s ADD INDEX %s_ix5(`query_day`); #### JDBC STREAMING JOB STATS STORE create.streamingjobstats.store.table=CREATE TABLE IF NOT EXISTS `%s` ( \ diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java index 76bf56aedf..6a696c0fdd 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/JdbcQueryHistoryStore.java @@ -91,6 +91,8 @@ public class JdbcQueryHistoryStore { public static final String ID_TABLE_ALIAS = "idTable"; public static final String DELETE_REALIZATION_LOG = "Delete {} row query history realization takes {} ms"; + public static final String UNSUPPORTED_MESSAGE = "Unsupported time window!"; + private final QueryHistoryTable queryHistoryTable; private final QueryHistoryRealizationTable queryHistoryRealizationTable; @@ -348,6 +350,19 @@ public class JdbcQueryHistoryStore { } } + public List<QueryStatistics> queryCountAndAvgDurationRealization(long startTime, long endTime, String project) { + try (SqlSession session = sqlSessionFactory.openSession()) { + QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); + SelectStatementProvider statementProvider = select(count(queryHistoryRealizationTable.queryId).as(COUNT), + avg(queryHistoryRealizationTable.duration).as("mean")).from(queryHistoryRealizationTable) + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)).build() + .render(RenderingStrategies.MYBATIS3); + return mapper.selectMany(statementProvider); + } + } + public List<QueryStatistics> queryCountByModel(long startTime, long endTime, String project) { try (SqlSession session = sqlSessionFactory.openSession()) { QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); @@ -429,6 +444,26 @@ public class JdbcQueryHistoryStore { } } + public List<QueryStatistics> queryAvgDurationRealizationByTime(long startTime, long endTime, String timeDimension, + String project) { + try (SqlSession session = sqlSessionFactory.openSession()) { + QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); + SelectStatementProvider statementProvider = queryAvgDurationRealizationByTimeProvider(startTime, endTime, + timeDimension, project); + return mapper.selectMany(statementProvider); + } + } + + public List<QueryStatistics> queryCountRealizationByTime(long startTime, long endTime, String timeDimension, + String project) { + try (SqlSession session = sqlSessionFactory.openSession()) { + QueryStatisticsMapper mapper = session.getMapper(QueryStatisticsMapper.class); + SelectStatementProvider statementProvider = queryCountRealizationByTimeProvider(startTime, endTime, + timeDimension, project); + return mapper.selectMany(statementProvider); + } + } + public void deleteQueryHistory() { long startTime = System.currentTimeMillis(); try (SqlSession session = sqlSessionFactory.openSession()) { @@ -605,7 +640,13 @@ public class JdbcQueryHistoryStore { .map(queryHistoryRealizationTable.queryTime) .toPropertyWhenPresent("queryTime", realizationMetrics::getQueryTime) .map(queryHistoryRealizationTable.projectName) - .toPropertyWhenPresent("projectName", realizationMetrics::getProjectName).build() + .toPropertyWhenPresent("projectName", realizationMetrics::getProjectName) + .map(queryHistoryRealizationTable.queryDay) + .toPropertyWhenPresent("queryDay", realizationMetrics::getQueryDay) + .map(queryHistoryRealizationTable.queryFirstDayOfWeek) + .toPropertyWhenPresent("queryFirstDayOfWeek", realizationMetrics::getQueryFirstDayOfWeek) + .map(queryHistoryRealizationTable.queryFirstDayOfMonth) + .toPropertyWhenPresent("queryFirstDayOfMonth", realizationMetrics::getQueryFirstDayOfMonth).build() .render(RenderingStrategies.MYBATIS3); } @@ -768,7 +809,7 @@ public class JdbcQueryHistoryStore { .groupBy(queryHistoryTable.queryDay) // .build().render(RenderingStrategies.MYBATIS3); } else { - throw new IllegalStateException("Unsupported time window!"); + throw new IllegalStateException(UNSUPPORTED_MESSAGE); } } @@ -799,7 +840,75 @@ public class JdbcQueryHistoryStore { .groupBy(queryHistoryTable.queryDay) // .build().render(RenderingStrategies.MYBATIS3); } else { - throw new IllegalStateException("Unsupported time window!"); + throw new IllegalStateException(UNSUPPORTED_MESSAGE); + } + } + + private SelectStatementProvider queryAvgDurationRealizationByTimeProvider(long startTime, long endTime, + String timeDimension, String project) { + if (timeDimension.equalsIgnoreCase(MONTH)) { + return select(queryHistoryRealizationTable.queryFirstDayOfMonth.as("time"), + avg(queryHistoryRealizationTable.duration).as("mean")) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryFirstDayOfMonth) // + .build().render(RenderingStrategies.MYBATIS3); + } else if (timeDimension.equalsIgnoreCase(WEEK)) { + return select(queryHistoryRealizationTable.queryFirstDayOfWeek.as("time"), + avg(queryHistoryRealizationTable.duration).as("mean")) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryFirstDayOfWeek) // + .build().render(RenderingStrategies.MYBATIS3); + } else if (timeDimension.equalsIgnoreCase(DAY)) { + return select(queryHistoryRealizationTable.queryDay.as("time"), + avg(queryHistoryRealizationTable.duration).as("mean")) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryDay) // + .build().render(RenderingStrategies.MYBATIS3); + } else { + throw new IllegalStateException(UNSUPPORTED_MESSAGE); + } + } + + private SelectStatementProvider queryCountRealizationByTimeProvider(long startTime, long endTime, + String timeDimension, String project) { + if (timeDimension.equalsIgnoreCase(MONTH)) { + return select(queryHistoryRealizationTable.queryFirstDayOfMonth.as("time"), + count(queryHistoryRealizationTable.id).as(COUNT)) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryFirstDayOfMonth) // + .build().render(RenderingStrategies.MYBATIS3); + } else if (timeDimension.equalsIgnoreCase(WEEK)) { + return select(queryHistoryRealizationTable.queryFirstDayOfWeek.as("time"), + count(queryHistoryRealizationTable.id).as(COUNT)) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryFirstDayOfWeek) // + .build().render(RenderingStrategies.MYBATIS3); + } else if (timeDimension.equalsIgnoreCase(DAY)) { + return select(queryHistoryRealizationTable.queryDay.as("time"), + count(queryHistoryRealizationTable.id).as(COUNT)) // + .from(queryHistoryRealizationTable) // + .where(queryHistoryRealizationTable.queryTime, isGreaterThanOrEqualTo(startTime)) // + .and(queryHistoryRealizationTable.queryTime, isLessThan(endTime)) // + .and(queryHistoryRealizationTable.projectName, isEqualTo(project)) // + .groupBy(queryHistoryRealizationTable.queryDay) // + .build().render(RenderingStrategies.MYBATIS3); + } else { + throw new IllegalStateException(UNSUPPORTED_MESSAGE); } } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java index d954368f35..882dc78477 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryDAO.java @@ -29,6 +29,8 @@ public interface QueryHistoryDAO { QueryStatistics getQueryCountAndAvgDuration(long startTime, long endTime, String project); + QueryStatistics getQueryCountAndAvgDurationRealization(long startTime, long endTime, String project); + QueryStatistics getQueryCountByRange(long startTime, long endTime, String project); long getQueryHistoryCountBeyondOffset(long offset, String project); @@ -37,10 +39,16 @@ public interface QueryHistoryDAO { List<QueryStatistics> getQueryCountByTime(long startTime, long endTime, String timeDimension, String project); + List<QueryStatistics> getQueryCountRealizationByTime(long startTime, long endTime, String timeDimension, + String project); + List<QueryStatistics> getAvgDurationByModel(long startTime, long endTime, String project); List<QueryStatistics> getAvgDurationByTime(long startTime, long endTime, String timeDimension, String project); + List<QueryStatistics> getAvgDurationRealizationByTime(long startTime, long endTime, String timeDimension, + String project); + String getQueryMetricMeasurement(); void deleteQueryHistoriesIfMaxSizeReached(); diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java index 3bbc2f4505..754257f87d 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryHistoryRealizationTable.java @@ -32,6 +32,11 @@ public class QueryHistoryRealizationTable extends SqlTable { public final SqlColumn<String> model = column("model", JDBCType.VARCHAR); public final SqlColumn<String> layoutId = column("layout_id", JDBCType.VARCHAR); public final SqlColumn<String> indexType = column("index_type", JDBCType.VARCHAR); + public final SqlColumn<Long> queryFirstDayOfMonth = column("query_first_day_of_month", JDBCType.BIGINT); + public final SqlColumn<Long> queryFirstDayOfWeek = column("query_first_day_of_week", JDBCType.BIGINT); + public final SqlColumn<Long> queryDay = column("query_day", JDBCType.BIGINT); + + public final SqlColumn<Long> id = column("id", JDBCType.BIGINT); public QueryHistoryRealizationTable(String tableName) { super(tableName); diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java index 92323af54f..031bfc2f06 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetrics.java @@ -143,6 +143,12 @@ public class QueryMetrics extends SchedulerEventNotifier { protected List<String> snapshots; + protected long queryFirstDayOfMonth; + + protected long queryFirstDayOfWeek; + + protected long queryDay; + // For serialize public RealizationMetrics() { } diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java index fc726dc804..341f7ecb48 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/QueryMetricsContext.java @@ -242,6 +242,9 @@ public class QueryMetricsContext extends QueryMetrics { realizationMetrics.setDuration(queryDuration); realizationMetrics.setQueryTime(queryTime); realizationMetrics.setProjectName(projectName); + realizationMetrics.setQueryDay(queryDay); + realizationMetrics.setQueryFirstDayOfWeek(queryFirstDayOfWeek); + realizationMetrics.setQueryFirstDayOfMonth(queryFirstDayOfMonth); realizationMetrics.setStreamingLayout(realization.isStreamingLayout()); realizationMetrics.setSnapshots(realization.getSnapshots()); realizationMetricList.add(realizationMetrics); diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java index 14fdf0c4a1..64d874d1e6 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDAO.java @@ -195,6 +195,14 @@ public class RDBMSQueryHistoryDAO implements QueryHistoryDAO { return result.get(0); } + public QueryStatistics getQueryCountAndAvgDurationRealization(long startTime, long endTime, String project) { + List<QueryStatistics> result = jdbcQueryHisStore.queryCountAndAvgDurationRealization(startTime, endTime, + project); + if (CollectionUtils.isEmpty(result)) + return new QueryStatistics(); + return result.get(0); + } + public List<QueryStatistics> getQueryCountByModel(long startTime, long endTime, String project) { return jdbcQueryHisStore.queryCountByModel(startTime, endTime, project); } @@ -216,6 +224,11 @@ public class RDBMSQueryHistoryDAO implements QueryHistoryDAO { return jdbcQueryHisStore.queryCountByTime(startTime, endTime, timeDimension, project); } + public List<QueryStatistics> getQueryCountRealizationByTime(long startTime, long endTime, String timeDimension, + String project) { + return jdbcQueryHisStore.queryCountRealizationByTime(startTime, endTime, timeDimension, project); + } + public List<QueryStatistics> getAvgDurationByModel(long startTime, long endTime, String project) { return jdbcQueryHisStore.queryAvgDurationByModel(startTime, endTime, project); } @@ -225,6 +238,11 @@ public class RDBMSQueryHistoryDAO implements QueryHistoryDAO { return jdbcQueryHisStore.queryAvgDurationByTime(startTime, endTime, timeDimension, project); } + public List<QueryStatistics> getAvgDurationRealizationByTime(long startTime, long endTime, String timeDimension, + String project) { + return jdbcQueryHisStore.queryAvgDurationRealizationByTime(startTime, endTime, timeDimension, project); + } + @Override public Map<String, Long> getQueryCountByProject() { return jdbcQueryHisStore.getCountGroupByProject(); diff --git a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java index f6589ed5d0..d8b2c05613 100644 --- a/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java +++ b/src/core-metadata/src/main/java/org/apache/kylin/metadata/query/util/QueryHisStoreUtil.java @@ -47,6 +47,8 @@ import org.apache.kylin.common.Singletons; import org.apache.kylin.common.logging.LogOutputStream; import org.apache.kylin.common.persistence.metadata.jdbc.JdbcUtil; import org.apache.kylin.common.util.SetThreadName; +import org.apache.kylin.common.util.SetThreadName; +import org.apache.kylin.metadata.epoch.EpochManager; import org.apache.kylin.metadata.project.NProjectManager; import org.apache.kylin.metadata.project.ProjectInstance; import org.apache.kylin.metadata.query.QueryHistoryDAO; @@ -74,6 +76,12 @@ public class QueryHisStoreUtil { private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX1 = "create.queryhistoryrealization.store.tableindex1"; private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX2 = "create.queryhistoryrealization.store.tableindex2"; + private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX3 = "create.queryhistoryrealization.store.tableindex3"; + + private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX4 = "create.queryhistoryrealization.store.tableindex4"; + + private static final String CREATE_QUERY_HISTORY_REALIZATION_INDEX5 = "create.queryhistoryrealization.store.tableindex5"; + private QueryHisStoreUtil() { } @@ -158,6 +166,18 @@ public class QueryHisStoreUtil { String.format(Locale.ROOT, properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX2), qhRealizationTableName, qhRealizationTableName).getBytes(Charset.defaultCharset())), Charset.defaultCharset())); + sr.runScript(new InputStreamReader(new ByteArrayInputStream(// + String.format(Locale.ROOT, properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX3), + qhRealizationTableName, qhRealizationTableName).getBytes(Charset.defaultCharset())), + Charset.defaultCharset())); + sr.runScript(new InputStreamReader(new ByteArrayInputStream(// + String.format(Locale.ROOT, properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX4), + qhRealizationTableName, qhRealizationTableName).getBytes(Charset.defaultCharset())), + Charset.defaultCharset())); + sr.runScript(new InputStreamReader(new ByteArrayInputStream(// + String.format(Locale.ROOT, properties.getProperty(CREATE_QUERY_HISTORY_REALIZATION_INDEX5), + qhRealizationTableName, qhRealizationTableName).getBytes(Charset.defaultCharset())), + Charset.defaultCharset())); } } diff --git a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java index a6bccb5038..b30e13d383 100644 --- a/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java +++ b/src/core-metadata/src/test/java/org/apache/kylin/metadata/query/RDBMSQueryHistoryDaoTest.java @@ -43,6 +43,7 @@ public class RDBMSQueryHistoryDaoTest extends NLocalFileMetadataTestCase { String PROJECT = "default"; public static final String WEEK = "week"; public static final String DAY = "day"; + public static final String MONTH = "month"; public static final String NORMAL_USER = "normal_user"; public static final String ADMIN = "ADMIN"; @@ -283,6 +284,48 @@ public class RDBMSQueryHistoryDaoTest extends NLocalFileMetadataTestCase { Assert.assertEquals(3, monthQueryStatistics.get(0).getCount()); } + @Test + public void testGetQueryRealizationByTime() { + // 2020-01-29 23:25:12 + queryHistoryDAO.insert(createQueryMetrics(1580311512000L, 1L, true, PROJECT, true)); + // 2020-01-30 23:25:12 + queryHistoryDAO.insert(createQueryMetrics(1580397912000L, 1L, false, PROJECT, true)); + // 2020-01-31 23:25:12 + queryHistoryDAO.insert(createQueryMetrics(1580484312000L, 1L, false, PROJECT, true)); + // 2021-01-29 23:25:12 + queryHistoryDAO.insert(createQueryMetrics(1611933912000L, 1L, false, PROJECT, true)); + + // filter from 2020-01-26 23:25:11 to 2020-01-31 23:25:13 + List<QueryStatistics> dayQueryStatistics = queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L, + 1580484313000L, DAY, PROJECT); + Assert.assertEquals(0, dayQueryStatistics.size()); + + List<QueryStatistics> weekQueryStatistics = queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L, + 1580484313000L, WEEK, PROJECT); + Assert.assertEquals(0, weekQueryStatistics.size()); + + List<QueryStatistics> monthQueryStatistics = queryHistoryDAO.getQueryCountRealizationByTime(1580052311000L, + 1580484313000L, MONTH, PROJECT); + Assert.assertEquals(0, monthQueryStatistics.size()); + + + dayQueryStatistics = queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L, + DAY, PROJECT); + Assert.assertEquals(0, dayQueryStatistics.size()); + weekQueryStatistics = queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L, + WEEK, PROJECT); + Assert.assertEquals(0, weekQueryStatistics.size()); + monthQueryStatistics = queryHistoryDAO.getAvgDurationRealizationByTime(1580052311000L, 1580484313000L, + WEEK, PROJECT); + Assert.assertEquals(0, monthQueryStatistics.size()); + + QueryStatistics statistics = queryHistoryDAO.getQueryCountAndAvgDurationRealization(1580052311000L, 1580484313000L, + PROJECT); + Assert.assertEquals(0, statistics.getCount()); + Assert.assertEquals(0, statistics.getMeanDuration(), 0.1); + + } + @Test public void testGetAvgDurationByTime() throws Exception { // 2020-01-29 23:25:12 diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java new file mode 100644 index 0000000000..07c1511f93 --- /dev/null +++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/DashboardService.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.service; + +import lombok.extern.slf4j.Slf4j; +import org.apache.kylin.common.response.MetricsResponse; +import org.apache.kylin.metadata.cube.realization.HybridRealization; +import org.apache.kylin.metadata.project.ProjectInstance; +import org.apache.kylin.query.exception.UnsupportedQueryException; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.response.JobStatisticsResponse; +import org.apache.kylin.rest.response.NDataModelOldParams; +import org.apache.kylin.rest.response.NDataModelResponse; +import org.apache.kylin.rest.response.QueryStatisticsResponse; +import org.apache.kylin.rest.util.ModelUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.security.access.AccessDeniedException; +import org.springframework.security.access.prepost.PreAuthorize; +import org.springframework.stereotype.Component; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +@Slf4j +@Component("dashboardService") +public class DashboardService extends BasicService { + + public static final Logger logger = LoggerFactory.getLogger(DashboardService.class); + public static final String DAY = "day"; + public static final String AVG_QUERY_LATENCY = "AVG_QUERY_LATENCY"; + public static final String JOB = "JOB"; + public static final String AVG_JOB_BUILD_TIME = "AVG_JOB_BUILD_TIME"; + private static final String QUERY = "QUERY"; + private static final String QUERY_COUNT = "QUERY_COUNT"; + private static final String JOB_COUNT = "JOB_COUNT"; + @Autowired + ModelService modelService; + + @Autowired + QueryHistoryService queryHistoryService; + + @Autowired + JobService jobService; + + public MetricsResponse getModelMetrics(String projectName, String modelName) { + MetricsResponse modelMetrics = new MetricsResponse(); + long totalModelSize = 0; + long totalRecordSize = 0; + List<NDataModelResponse> models = modelService.getCubes0(modelName, projectName);//5.0 cube is model + Integer totalModel = models.size(); + ProjectInstance project = getProjectManager().getProject(projectName); + totalModel += project.getRealizationCount(HybridRealization.REALIZATION_TYPE); + modelMetrics.increase("totalModel", totalModel.floatValue()); + + Float minModelExpansion = Float.POSITIVE_INFINITY; + Float maxModelExpansion = Float.NEGATIVE_INFINITY; + + for (NDataModelResponse dataModel : models) { + NDataModelOldParams params = dataModel.getOldParams(); + if (params.getInputRecordSizeBytes() > 0) { + totalModelSize += params.getSizeKB() * 1024; + totalRecordSize += params.getInputRecordSizeBytes();//size / 1024 * 1024 * 1024 = x GB + Float modelExpansion = Float.valueOf(dataModel.getExpansionrate()); + + if (modelExpansion > maxModelExpansion) { + maxModelExpansion = modelExpansion; + } + if (modelExpansion < minModelExpansion) { + minModelExpansion = modelExpansion; + } + } + } + + Float avgModelExpansion = 0f; + if (totalRecordSize != 0) { + avgModelExpansion = Float.valueOf(ModelUtils.computeExpansionRate(totalModelSize, totalRecordSize)); + } + + modelMetrics.increase("avgModelExpansion", avgModelExpansion); + modelMetrics.increase("maxModelExpansion", maxModelExpansion); + modelMetrics.increase("minModelExpansion", minModelExpansion); + + return modelMetrics; + } + + public MetricsResponse getQueryMetrics(String projectName, String startTime, String endTime) { + MetricsResponse queryMetrics = new MetricsResponse(); + QueryStatisticsResponse queryStatistics = queryHistoryService.getQueryStatisticsByRealization(projectName, + convertToTimestamp(startTime), convertToTimestamp(endTime)); + Float queryCount = (float) queryStatistics.getCount(); + Float avgQueryLatency = (float) queryStatistics.getMean(); + queryMetrics.increase("queryCount", queryCount); + queryMetrics.increase("avgQueryLatency", avgQueryLatency); + return queryMetrics; + } + + public MetricsResponse getJobMetrics(String projectName, String startTime, String endTime) { + MetricsResponse jobMetrics = new MetricsResponse(); + JobStatisticsResponse jobStats = jobService.getJobStats(projectName, convertToTimestamp(startTime), + convertToTimestamp(endTime)); + Float jobCount = (float) jobStats.getCount(); + Float jobTotalByteSize = (float) jobStats.getTotalByteSize(); + Float jobTotalLatency = (float) jobStats.getTotalDuration(); + jobMetrics.increase("jobCount", jobCount); + jobMetrics.increase("jobTotalByteSize", jobTotalByteSize); + jobMetrics.increase("jobTotalLatency", jobTotalLatency); + return jobMetrics; + } + + public MetricsResponse getChartData(String category, String projectName, String startTime, String endTime, + String dimension, String metric) { + long _startTime = convertToTimestamp(startTime); + long _endTime = convertToTimestamp(endTime); + switch (category) { + case QUERY: { + switch (metric) { + case QUERY_COUNT: + Map<String, Object> queryCounts = queryHistoryService.getQueryCountByRealization(projectName, + _startTime, _endTime, dimension.toLowerCase()); + return transformChartData(queryCounts); + + case AVG_QUERY_LATENCY: + Map<String, Object> avgDurations = queryHistoryService.getAvgDurationByRealization(projectName, + _startTime, _endTime, dimension.toLowerCase()); + return transformChartData(avgDurations); + default: + throw new UnsupportedQueryException("Metric should be COUNT or AVG_QUERY_LATENCY"); + } + } + case JOB: { + switch (metric) { + case JOB_COUNT: + Map<String, Integer> jobCounts = jobService.getJobCount(projectName, _startTime, _endTime, + dimension.toLowerCase()); + MetricsResponse counts = new MetricsResponse(); + jobCounts.forEach((k, v) -> counts.increase(k, Float.valueOf(v))); + return counts; + case AVG_JOB_BUILD_TIME: + Map<String, Double> jobDurationPerByte = jobService.getJobDurationPerByte(projectName, _startTime, + _endTime, dimension.toLowerCase()); + MetricsResponse avgBuild = new MetricsResponse(); + jobDurationPerByte.forEach((k, v) -> avgBuild.increase(k, Float.valueOf(String.valueOf(v)))); + return avgBuild; + default: + throw new UnsupportedQueryException("Metric should be JOB_COUNT or AVG_JOB_BUILD_TIME"); + } + } + default: + throw new UnsupportedQueryException("Category should either be QUERY or JOB"); + } + } + + private MetricsResponse transformChartData(Map<String, Object> responses) { + MetricsResponse metrics = new MetricsResponse(); + for (Map.Entry<String, Object> entry : responses.entrySet()) { + String metric = entry.getKey(); + float value = Float.parseFloat(entry.getValue().toString()); + metrics.increase(metric, value); + } + return metrics; + } + + private long convertToTimestamp(String time) { + Date date; + try { + date = new SimpleDateFormat("yyyy-MM-dd", Locale.getDefault(Locale.Category.FORMAT)).parse(time); + } catch (ParseException e) { + logger.error("parse time to timestamp error!"); + return 0L; + } + return date.getTime(); + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN + " or hasPermission(#project, 'ADMINISTRATION')") + private void checkAuthorization(ProjectInstance project) throws AccessDeniedException { + //for selected project + } + + @PreAuthorize(Constant.ACCESS_HAS_ROLE_ADMIN) + private void checkAuthorization() throws AccessDeniedException { + //for no selected project + } + + public void checkAuthorization(String projectName) { + if (projectName != null && !projectName.isEmpty()) { + ProjectInstance project = getProjectManager().getProject(projectName); + try { + checkAuthorization(project); + } catch (AccessDeniedException e) { + List<NDataModelResponse> models = modelService.getCubes0(null, projectName); + if (models.isEmpty()) { + throw new AccessDeniedException("Access is denied"); + } + } + } else { + checkAuthorization(); + } + } +} diff --git a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java index c6dc63bba1..339b742aa4 100644 --- a/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java +++ b/src/query-service/src/main/java/org/apache/kylin/rest/service/QueryHistoryService.java @@ -86,6 +86,9 @@ public class QueryHistoryService extends BasicService implements AsyncTaskQueryH // public static final String DELETED_MODEL = "Deleted Model"; // public static final byte[] CSV_UTF8_BOM = new byte[]{(byte)0xEF, (byte)0xBB, (byte)0xBF}; public static final String DAY = "day"; + public static final String MODEL = "model"; + public static final String COUNT = "count"; + public static final String MEAN_DURATION = "meanDuration"; private static final Logger logger = LoggerFactory.getLogger("query"); @Autowired private AclEvaluate aclEvaluate; @@ -299,6 +302,16 @@ public class QueryHistoryService extends BasicService implements AsyncTaskQueryH return new QueryStatisticsResponse(queryStatistics.getCount(), queryStatistics.getMeanDuration()); } + public QueryStatisticsResponse getQueryStatisticsByRealization(String project, long startTime, long endTime) { + Preconditions.checkArgument(StringUtils.isNotEmpty(project)); + aclEvaluate.checkProjectReadPermission(project); + + QueryHistoryDAO queryHistoryDao = getQueryHistoryDao(); + QueryStatistics queryStatistics = queryHistoryDao.getQueryCountAndAvgDurationRealization(startTime, endTime, + project); + return new QueryStatisticsResponse(queryStatistics.getCount(), queryStatistics.getMeanDuration()); + } + public long getLastWeekQueryCount(String project) { Preconditions.checkArgument(StringUtils.isNotEmpty(project)); aclEvaluate.checkProjectReadPermission(project); @@ -325,14 +338,14 @@ public class QueryHistoryService extends BasicService implements AsyncTaskQueryH QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao(); List<QueryStatistics> queryStatistics; - if (dimension.equals("model")) { + if (dimension.equals(MODEL)) { queryStatistics = queryHistoryDAO.getQueryCountByModel(startTime, endTime, project); - return transformQueryStatisticsByModel(project, queryStatistics, "count"); + return transformQueryStatisticsByModel(project, queryStatistics, COUNT); } queryStatistics = queryHistoryDAO.getQueryCountByTime(startTime, endTime, dimension, project); fillZeroForQueryStatistics(queryStatistics, startTime, endTime, dimension); - return transformQueryStatisticsByTime(queryStatistics, "count", dimension); + return transformQueryStatisticsByTime(queryStatistics, COUNT, dimension); } public Map<String, Object> getAvgDuration(String project, long startTime, long endTime, String dimension) { @@ -341,14 +354,48 @@ public class QueryHistoryService extends BasicService implements AsyncTaskQueryH QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao(); List<QueryStatistics> queryStatistics; - if (dimension.equals("model")) { + if (dimension.equals(MODEL)) { queryStatistics = queryHistoryDAO.getAvgDurationByModel(startTime, endTime, project); - return transformQueryStatisticsByModel(project, queryStatistics, "meanDuration"); + return transformQueryStatisticsByModel(project, queryStatistics, MEAN_DURATION); } queryStatistics = queryHistoryDAO.getAvgDurationByTime(startTime, endTime, dimension, project); fillZeroForQueryStatistics(queryStatistics, startTime, endTime, dimension); - return transformQueryStatisticsByTime(queryStatistics, "meanDuration", dimension); + return transformQueryStatisticsByTime(queryStatistics, MEAN_DURATION, dimension); + } + + public Map<String, Object> getAvgDurationByRealization(String project, long startTime, long endTime, + String dimension) { + Preconditions.checkArgument(StringUtils.isNotEmpty(project)); + aclEvaluate.checkProjectReadPermission(project); + QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao(); + List<QueryStatistics> queryStatistics; + + if (dimension.equals(MODEL)) { + queryStatistics = queryHistoryDAO.getAvgDurationByModel(startTime, endTime, project); + return transformQueryStatisticsByModel(project, queryStatistics, MEAN_DURATION); + } + + queryStatistics = queryHistoryDAO.getAvgDurationRealizationByTime(startTime, endTime, dimension, project); + fillZeroForQueryStatistics(queryStatistics, startTime, endTime, dimension); + return transformQueryStatisticsByTime(queryStatistics, MEAN_DURATION, dimension); + } + + public Map<String, Object> getQueryCountByRealization(String project, long startTime, long endTime, + String dimension) { + Preconditions.checkArgument(StringUtils.isNotEmpty(project)); + aclEvaluate.checkProjectReadPermission(project); + QueryHistoryDAO queryHistoryDAO = getQueryHistoryDao(); + List<QueryStatistics> queryStatistics; + + if (dimension.equals(MODEL)) { + queryStatistics = queryHistoryDAO.getQueryCountByModel(startTime, endTime, project); + return transformQueryStatisticsByModel(project, queryStatistics, COUNT); + } + + queryStatistics = queryHistoryDAO.getQueryCountRealizationByTime(startTime, endTime, dimension, project); + fillZeroForQueryStatistics(queryStatistics, startTime, endTime, dimension); + return transformQueryStatisticsByTime(queryStatistics, COUNT, dimension); } private Map<String, Object> transformQueryStatisticsByModel(String project, List<QueryStatistics> statistics, diff --git a/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java b/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java new file mode 100644 index 0000000000..3710ec21ae --- /dev/null +++ b/src/query-service/src/test/java/org/apache/kylin/rest/service/DashboardServiceTest.java @@ -0,0 +1,394 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.kylin.rest.service; + +import com.google.common.collect.Lists; +import lombok.extern.slf4j.Slf4j; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.response.MetricsResponse; +import org.apache.kylin.engine.spark.utils.ComputedColumnEvalUtil; +import org.apache.kylin.metadata.model.util.ExpandableMeasureUtil; +import org.apache.kylin.metadata.query.QueryStatistics; +import org.apache.kylin.metadata.query.RDBMSQueryHistoryDAO; +import org.apache.kylin.query.util.QueryUtil; +import org.apache.kylin.rest.constant.Constant; +import org.apache.kylin.rest.response.JobStatisticsResponse; +import org.apache.kylin.rest.util.AclEvaluate; +import org.apache.kylin.rest.util.AclPermissionUtil; +import org.apache.kylin.rest.util.AclUtil; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.InjectMocks; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.springframework.security.authentication.TestingAuthenticationToken; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.test.util.ReflectionTestUtils; + +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.time.Instant; +import java.util.List; +import java.util.Locale; +import java.util.TimeZone; + +@Slf4j +public class DashboardServiceTest extends SourceTestCase{ + + public static final String MODEL = "model"; + public static final String DAY = "day"; + public static final String AVG_QUERY_LATENCY = "AVG_QUERY_LATENCY"; + public static final String JOB = "JOB"; + public static final String AVG_JOB_BUILD_TIME = "AVG_JOB_BUILD_TIME"; + private static final String WEEK = "week"; + private static final String MONTH = "month"; + private static final String QUERY = "QUERY"; + private static final String QUERY_COUNT = "QUERY_COUNT"; + private static final String JOB_COUNT = "JOB_COUNT"; + @InjectMocks + private final DashboardService dashboardService = Mockito.spy(new DashboardService()); + @InjectMocks + private final ModelService modelService = Mockito.spy(new ModelService()); + @InjectMocks + private final JobService jobService = Mockito.spy(new JobService()); + @InjectMocks + private final QueryHistoryService queryHistoryService = Mockito.spy(new QueryHistoryService()); + @InjectMocks + private final ModelBuildService modelBuildService = Mockito.spy(new ModelBuildService()); + @InjectMocks + private final ModelSemanticHelper semanticService = Mockito.spy(new ModelSemanticHelper()); + @InjectMocks + private final ProjectService projectService = Mockito.spy(new ProjectService()); + @InjectMocks + private final MockModelQueryService modelQueryService = Mockito.spy(new MockModelQueryService()); + @InjectMocks + private final SegmentHelper segmentHelper = new SegmentHelper(); + + + @Mock + private final AclEvaluate aclEvaluate = Mockito.spy(AclEvaluate.class); + @Mock + protected IUserGroupService userGroupService = Mockito.spy(NUserGroupService.class); + @Mock + private final AccessService accessService = Mockito.spy(AccessService.class); + @Mock + private final AclUtil aclUtil = Mockito.spy(AclUtil.class); + + + protected String getProject() { + return "default"; + } + + @Before + public void setup() { + super.setup(); + ReflectionTestUtils.setField(aclEvaluate, "aclUtil", aclUtil); + ReflectionTestUtils.setField(modelService, "aclEvaluate", aclEvaluate); + ReflectionTestUtils.setField(modelService, "accessService", accessService); + ReflectionTestUtils.setField(modelService, "userGroupService", userGroupService); + ReflectionTestUtils.setField(semanticService, "userGroupService", userGroupService); + ReflectionTestUtils.setField(modelBuildService, "userGroupService", userGroupService); + ReflectionTestUtils.setField(semanticService, "expandableMeasureUtil", + new ExpandableMeasureUtil((model, ccDesc) -> { + String ccExpression = QueryUtil.massageComputedColumn(model, model.getProject(), ccDesc, + AclPermissionUtil.createAclInfo(model.getProject(), + semanticService.getCurrentUserGroups())); + ccDesc.setInnerExpression(ccExpression); + ComputedColumnEvalUtil.evaluateExprAndType(model, ccDesc); + })); + ReflectionTestUtils.setField(modelService, "projectService", projectService); + ReflectionTestUtils.setField(modelService, "modelQuerySupporter", modelQueryService); + ReflectionTestUtils.setField(modelService, "modelBuildService", modelBuildService); + + ReflectionTestUtils.setField(modelBuildService, "modelService", modelService); + ReflectionTestUtils.setField(modelBuildService, "segmentHelper", segmentHelper); + ReflectionTestUtils.setField(modelBuildService, "aclEvaluate", aclEvaluate); + modelService.setSemanticUpdater(semanticService); + modelService.setSegmentHelper(segmentHelper); + + ReflectionTestUtils.setField(jobService, "aclEvaluate", aclEvaluate); + ReflectionTestUtils.setField(jobService, "projectService", projectService); + ReflectionTestUtils.setField(jobService, "modelService", modelService); + + + ReflectionTestUtils.setField(queryHistoryService, "aclEvaluate", aclEvaluate); + ReflectionTestUtils.setField(queryHistoryService, "modelService", modelService); + ReflectionTestUtils.setField(queryHistoryService, "userGroupService", userGroupService); + ReflectionTestUtils.setField(queryHistoryService, "asyncTaskService", new AsyncTaskService()); + + ReflectionTestUtils.setField(dashboardService, "jobService", jobService); + ReflectionTestUtils.setField(dashboardService, "modelService", modelService); + ReflectionTestUtils.setField(dashboardService, "queryHistoryService", queryHistoryService); + + SecurityContextHolder.getContext() + .setAuthentication(new TestingAuthenticationToken("ADMIN", "ADMIN", Constant.ROLE_ADMIN)); + } + + @Test + public void testGetModelMetrics() { + MetricsResponse modelMetrics = dashboardService.getModelMetrics(getProject(), null); + Assert.assertEquals(4, modelMetrics.size()); + } + + @Test + public void testGetQueryMetrics() { + QueryStatistics queryStatistics = new QueryStatistics(); + queryStatistics.setCount(777); + queryStatistics.setMeanDuration(7070); + + RDBMSQueryHistoryDAO queryHistoryDAO = Mockito.mock(RDBMSQueryHistoryDAO.class); + Mockito.doReturn(queryStatistics).when(queryHistoryDAO).getQueryCountAndAvgDurationRealization(1262275200000L, + 1640966400000L, "default"); + Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao(); + + MetricsResponse queryMetrics = dashboardService.getQueryMetrics(getProject(), "2010-01-01", + "2022-01-01"); + Assert.assertEquals(2, queryMetrics.size()); + Assert.assertEquals(777, (double) queryMetrics.get("queryCount"), 0.1); + Assert.assertEquals(7070, (double) queryMetrics.get("avgQueryLatency"), 0.1); + } + + @Test + public void testGetJobMetrics() { + JobStatisticsResponse jobStats = jobService.getJobStats("default", Long.MIN_VALUE, Long.MAX_VALUE); + Assert.assertEquals(0, jobStats.getCount()); + Assert.assertEquals(0, jobStats.getTotalByteSize()); + Assert.assertEquals(0, jobStats.getTotalDuration()); + + String startTime = "2018-01-01"; + String endTime = "2018-02-01"; + + MetricsResponse metricsResponse = dashboardService.getJobMetrics(getProject(), startTime, endTime); + Assert.assertEquals(3, metricsResponse.size()); + Assert.assertEquals((float) 0, metricsResponse.get("jobCount"), 0.1); + } + + @Test + public void testGetChartDataOfQuery() throws ParseException { + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.getDefault(Locale.Category.FORMAT)); + String _startTime = "2018-01-01"; + String _endTime = "2018-01-03"; + + long startTime = format.parse("2018-01-01").getTime(); + long endTime = format.parse("2018-01-03").getTime(); + + + RDBMSQueryHistoryDAO queryHistoryDAO = Mockito.mock(RDBMSQueryHistoryDAO.class); + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getQueryCountByModel(startTime, endTime, "default"); + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getQueryCountRealizationByTime(Mockito.anyLong(), + Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByModel(startTime, endTime, + "default"); + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationRealizationByTime(Mockito.anyLong(), + Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao(); + + // QUERY_COUNT + // query count by model + MetricsResponse metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, MODEL, QUERY_COUNT); + Assert.assertEquals(3, metricsResponse.size()); + Assert.assertEquals(10, (double)metricsResponse.get("nmodel_basic"), 0.1); + Assert.assertEquals(11, (double)metricsResponse.get("all_fixed_length"), 0.1); + + // query count by day + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, DAY, QUERY_COUNT); + Assert.assertEquals(4, metricsResponse.size()); + Assert.assertEquals(10, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(11, (double)metricsResponse.get("2018-01-02"), 0.1); + + // query count by week + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, WEEK, QUERY_COUNT); + Assert.assertEquals(5, metricsResponse.size()); + Assert.assertEquals(10, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(11, (double)metricsResponse.get("2018-01-02"), 0.1); + + // query count by month + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, MONTH, QUERY_COUNT); + Assert.assertEquals(2, metricsResponse.size()); + Assert.assertEquals(11, (double)metricsResponse.get("2018-01"), 0.1); + + + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByModel(startTime, endTime, + "default"); + Mockito.doReturn(getTestStatistics()).when(queryHistoryDAO).getAvgDurationByTime(Mockito.anyLong(), + Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); + Mockito.doReturn(queryHistoryDAO).when(queryHistoryService).getQueryHistoryDao(); + + // AVG_QUERY_LATENCY + // avg duration by model + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, MODEL, AVG_QUERY_LATENCY); + Assert.assertEquals(3, metricsResponse.size()); + Assert.assertEquals(500, (double) metricsResponse.get("nmodel_basic"), 0.1); + Assert.assertEquals(600, (double) metricsResponse.get("all_fixed_length"), 0.1); + + // avg duration by day + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, DAY, AVG_QUERY_LATENCY); + Assert.assertEquals(4, metricsResponse.size()); + Assert.assertEquals(500, (double) metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(600, (double) metricsResponse.get("2018-01-02"), 0.1); + + // avg duration by week + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, WEEK, AVG_QUERY_LATENCY); + Assert.assertEquals(5, metricsResponse.size()); + Assert.assertEquals(500, (double) metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(600, (double) metricsResponse.get("2018-01-02"), 0.1); + + // avg duration by month + metricsResponse = dashboardService.getChartData(QUERY, getProject(), _startTime, _endTime, MONTH, AVG_QUERY_LATENCY); + Assert.assertEquals(2, metricsResponse.size()); + Assert.assertEquals(600, (double) metricsResponse.get("2018-01"), 0.1); + } + + @Test + public void testGetChartDataOfJob() { + JobStatisticsResponse jobStats = jobService.getJobStats("default", Long.MIN_VALUE, Long.MAX_VALUE); + Assert.assertEquals(0, jobStats.getCount()); + Assert.assertEquals(0, jobStats.getTotalByteSize()); + Assert.assertEquals(0, jobStats.getTotalDuration()); + + String startTime = "2018-01-01"; + String endTime = "2018-02-01"; + + //JOB_COUNT + //model + MetricsResponse metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, MODEL, JOB_COUNT); + Assert.assertEquals(0, metricsResponse.size()); + + //day + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, DAY, JOB_COUNT); + Assert.assertEquals(32, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1); + + //week + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, WEEK, JOB_COUNT); + Assert.assertEquals(5, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-08"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-29"), 0.1); + + //month + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, MONTH, JOB_COUNT); + Assert.assertEquals(2, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1); + + //AVG_BUILD_TIME + //model + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, MODEL, AVG_JOB_BUILD_TIME); + Assert.assertEquals(0, metricsResponse.size()); + + //day + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, DAY, AVG_JOB_BUILD_TIME); + Assert.assertEquals(32, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1); + + //week + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, WEEK, AVG_JOB_BUILD_TIME); + Assert.assertEquals(5, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-08"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-29"), 0.1); + + //month + metricsResponse = dashboardService.getChartData(JOB, getProject(), startTime, endTime, MONTH, AVG_JOB_BUILD_TIME); + Assert.assertEquals(2, metricsResponse.size()); + Assert.assertEquals(0, (double)metricsResponse.get("2018-01-01"), 0.1); + Assert.assertEquals(0, (double)metricsResponse.get("2018-02-01"), 0.1); + } + + @Test + public void testErrorCase() { + String errorMsg = ""; + try { + dashboardService.getChartData("error", getProject(), "2018-01-01", "2018-02-01", null, null); + } catch (Exception e) { + errorMsg = e.getMessage(); + } + Assert.assertNotNull(errorMsg); + + errorMsg = ""; + try { + dashboardService.getChartData(QUERY, getProject(), "2018-01-01", "2018-02-01", null, "error"); + } catch (Exception e) { + errorMsg = e.getMessage(); + } + Assert.assertNotNull(errorMsg); + + errorMsg = ""; + try { + dashboardService.getChartData(JOB, getProject(), "2018-01-01", "2018-02-01", null, "error"); + } catch (Exception e) { + errorMsg = e.getMessage(); + } + Assert.assertNotNull(errorMsg); + + } + + @Test + public void testCheckAuthorization() { + dashboardService.checkAuthorization(null); + dashboardService.checkAuthorization(getProject()); + } + + private List<QueryStatistics> getTestStatistics() throws ParseException { + int rawOffsetTime = TimeZone.getTimeZone(KylinConfig.getInstanceFromEnv().getTimeZone()).getRawOffset(); + String date = "2018-01-01"; + SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd", Locale.getDefault(Locale.Category.FORMAT)); + long time = format.parse(date).getTime(); + + QueryStatistics queryStatistics1 = new QueryStatistics(); + queryStatistics1.setCount(10); + queryStatistics1.setMeanDuration(500); + queryStatistics1.setModel("89af4ee2-2cdb-4b07-b39e-4c29856309aa"); + queryStatistics1.setTime(Instant.ofEpochMilli(time + rawOffsetTime)); + queryStatistics1.setMonth(date); + + date = "2018-01-02"; + time = format.parse(date).getTime(); + + QueryStatistics queryStatistics2 = new QueryStatistics(); + queryStatistics2.setCount(11); + queryStatistics2.setMeanDuration(600); + queryStatistics2.setModel("abe3bf1a-c4bc-458d-8278-7ea8b00f5e96"); + queryStatistics2.setTime(Instant.ofEpochMilli(time + rawOffsetTime)); + queryStatistics2.setMonth(date); + + date = "2018-01-03"; + time = format.parse(date).getTime(); + + QueryStatistics queryStatistics3 = new QueryStatistics(); + queryStatistics3.setCount(12); + queryStatistics3.setMeanDuration(700); + queryStatistics3.setModel("a8ba3ff1-83bd-4066-ad54-d2fb3d1f0e94"); + queryStatistics3.setTime(Instant.ofEpochMilli(time + rawOffsetTime)); + queryStatistics3.setMonth(date); + + date = "2018-01-04"; + time = format.parse(date).getTime(); + QueryStatistics queryStatistics4 = new QueryStatistics(); + queryStatistics4.setCount(11); + queryStatistics4.setMeanDuration(600); + queryStatistics4.setModel("not_existing_model"); + queryStatistics4.setTime(Instant.ofEpochMilli(time + rawOffsetTime)); + queryStatistics4.setMonth(date); + + return Lists.newArrayList(queryStatistics1, queryStatistics2, queryStatistics3, queryStatistics4); + } +} \ No newline at end of file