APACHE-KYLIN-2822 Introduce sunburst chart to show cuboid tree
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/659c3861 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/659c3861 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/659c3861 Branch: refs/heads/master Commit: 659c3861633f094634c9f7902d10341e9e698906 Parents: 7a54e58 Author: liapan <lia...@ebay.com> Authored: Mon Nov 20 10:26:44 2017 +0800 Committer: Zhong <nju_y...@apache.org> Committed: Sat Dec 2 23:43:43 2017 +0800 ---------------------------------------------------------------------- .../org/apache/kylin/cube/CubeInstance.java | 8 + .../job/execution/CheckpointExecutable.java | 37 ++++ webapp/app/js/controllers/cube.js | 179 ++++++++++++++++++- webapp/app/js/model/cubeConfig.js | 78 +++++++- webapp/app/js/services/cubes.js | 45 ++++- webapp/app/less/app.less | 25 +++ webapp/app/partials/cubes/cube_detail.html | 46 ++++- 7 files changed, 413 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java ---------------------------------------------------------------------- diff --git a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java index 462223a..70477eb 100644 --- a/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java +++ b/core-cube/src/main/java/org/apache/kylin/cube/CubeInstance.java @@ -435,6 +435,14 @@ public class CubeInstance extends RootPersistentEntity implements IRealization, } } + public long getCuboidLastOptimized() { + return cuboidLastOptimized; + } + + public void setCuboidLastOptimized(long lastOptimized) { + this.cuboidLastOptimized = lastOptimized; + } + /** * Get cuboid level count except base cuboid * @return http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java ---------------------------------------------------------------------- diff --git a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java index db477cb..c5f1c0a 100644 --- a/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java +++ b/core-job/src/main/java/org/apache/kylin/job/execution/CheckpointExecutable.java @@ -18,8 +18,12 @@ package org.apache.kylin.job.execution; +import java.io.IOException; import java.util.List; +import org.apache.kylin.cube.CubeInstance; +import org.apache.kylin.cube.CubeManager; +import org.apache.kylin.cube.CubeUpdate; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +37,7 @@ public class CheckpointExecutable extends DefaultChainedExecutable { private static final String DEPLOY_ENV_NAME = "envName"; private static final String PROJECT_INSTANCE_NAME = "projectName"; + private static final String CUBE_NAME = "cubeName"; private final List<AbstractExecutable> subTasksForCheck = Lists.newArrayList(); @@ -62,6 +67,33 @@ public class CheckpointExecutable extends DefaultChainedExecutable { return true; } + @Override + protected void onExecuteFinished(ExecuteResult result, ExecutableContext executableContext) { + super.onExecuteFinished(result, executableContext); + if (!isDiscarded() && result.succeed()) { + List<? extends Executable> jobs = getTasks(); + boolean allSucceed = true; + for (Executable task : jobs) { + final ExecutableState status = task.getStatus(); + if (status != ExecutableState.SUCCEED) { + allSucceed = false; + } + } + if (allSucceed) { + // Add last optimization time + CubeManager cubeManager = CubeManager.getInstance(executableContext.getConfig()); + CubeInstance cube = cubeManager.getCube(getCubeName()); + try{ + cube.setCuboidLastOptimized(getEndTime()); + CubeUpdate cubeUpdate = new CubeUpdate(cube); + cubeManager.updateCube(cubeUpdate); + } catch (IOException e) { + logger.error("Failed to update last optimized for " + getCubeName(), e); + } + } + } + } + public String getDeployEnvName() { return getParam(DEPLOY_ENV_NAME); } @@ -78,8 +110,13 @@ public class CheckpointExecutable extends DefaultChainedExecutable { setParam(PROJECT_INSTANCE_NAME, name); } + public String getCubeName() { + return getParam(CUBE_NAME); + } + @Override public int getDefaultPriority() { return DEFAULT_PRIORITY; } + } http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/webapp/app/js/controllers/cube.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/controllers/cube.js b/webapp/app/js/controllers/cube.js index b573b24..26dddea 100755 --- a/webapp/app/js/controllers/cube.js +++ b/webapp/app/js/controllers/cube.js @@ -18,7 +18,7 @@ 'use strict'; -KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, TableService, ModelGraphService, UserService,SweetAlert,loadingRequest,modelsManager,$modal,cubesManager) { +KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, CubeService, cubeConfig, TableService, ModelGraphService, UserService,SweetAlert,loadingRequest,modelsManager,$modal,cubesManager, $location) { $scope.newAccess = null; $scope.state = {jsonEdit: false}; @@ -111,5 +111,182 @@ KylinApp.controller('CubeCtrl', function ($scope, AccessService, MessageService, } }; + // cube api to refresh current chart after get recommend data. + $scope.currentChart = {}; + + // click planner tab to get current cuboid chart + $scope.getCubePlanner = function(cube) { + $scope.enableRecommend = cube.segments.length > 0 && _.some(cube.segments, function(segment){ return segment.status === 'READY'; }); + if (!cube.currentCuboids) { + CubeService.getCurrentCuboids({cubeId: cube.name}, function(data) { + if (data && data.nodeInfos) { + $scope.createChart(data, 'current'); + cube.currentCuboids = data; + } else { + $scope.currentOptions = angular.copy(cubeConfig.chartOptions); + $scope.currentData = []; + } + }, function(e) { + SweetAlert.swal('Oops...', 'Failed to get current cuboid.', 'error'); + console.error('current cuboid error', e.data); + }); + } else { + $scope.createChart(cube.currentCuboids, 'current'); + } + }; + + // get recommend cuboid chart + $scope.getRecommendCuboids = function(cube) { + if (!cube.recommendCuboids) { + loadingRequest.show(); + CubeService.getRecommendCuboids({cubeId: cube.name}, function(data) { + loadingRequest.hide(); + if (data && data.nodeInfos) { + // recommending + if (data.nodeInfos.length === 1 && !data.nodeInfos[0].cuboid_id) { + SweetAlert.swal('Loading', 'Please wait a minute, servers are recommending for you', 'success'); + } else { + $scope.createChart(data, 'recommend'); + cube.recommendCuboids = data; + // update current chart mark delete node gray. + angular.forEach(cube.currentCuboids.nodeInfos, function(nodeInfo) { + var tempNode = _.find(data.nodeInfos, function(o) { return o.cuboid_id == nodeInfo.cuboid_id; }); + if (!tempNode) { + nodeInfo.deleted = true; + } + }); + $scope.createChart(cube.currentCuboids, 'current'); + $scope.currentChart.api.refresh(); + } + } else { + $scope.currentOptions = angular.copy(cubeConfig.chartOptions); + $scope.recommendData = []; + } + }, function(e) { + loadingRequest.hide(); + SweetAlert.swal('Oops...', 'Failed to get recommend cuboid.', 'error'); + console.error('recommend cuboid error', e.data); + }); + } else { + $scope.createChart(cube.recommendCuboids, 'recommend'); + } + }; + + // optimize cuboid + $scope.optimizeCuboids = function(cube){ + SweetAlert.swal({ + title: '', + text: 'Are you sure to optimize the cube?', + type: '', + showCancelButton: true, + confirmButtonColor: '#DD6B55', + confirmButtonText: "Yes", + closeOnConfirm: true + }, function(isConfirm) { + if(isConfirm) { + var cuboidsRecommendArr = []; + angular.forEach(cube.recommendCuboids.nodeInfos, function(node) { + cuboidsRecommendArr.push(node.cuboid_id); + }); + loadingRequest.show(); + CubeService.optimize({cubeId: cube.name}, {cuboidsRecommend: cuboidsRecommendArr}, + function(job){ + loadingRequest.hide(); + SweetAlert.swal({ + title: 'Success!', + text: 'Optimize cube job has been started!', + type: 'success'}, + function() { + $location.path("/jobs"); + } + ); + }, function(e) { + loadingRequest.hide(); + if (e.status === 400) { + SweetAlert.swal('Oops...', e.data.exception, 'error'); + } else { + SweetAlert.swal('Oops...', "Failed to create optimize cube job.", 'error'); + console.error('optimize cube error', e.data); + } + }); + } + }); + }; + + // transform chart data and customized options. + $scope.createChart = function(data, type) { + var chartData = data.treeNode; + if ('current' === type) { + $scope.currentData = [chartData]; + $scope.currentOptions = angular.copy(cubeConfig.baseChartOptions); + $scope.currentOptions.caption = angular.copy(cubeConfig.currentCaption); + if ($scope.cube.recommendCuboids){ + $scope.currentOptions.caption.css['text-align'] = 'right'; + $scope.currentOptions.caption.css['right'] = '-12px'; + } + $scope.currentOptions.chart.color = function(d) { + var cuboid = _.find(data.nodeInfos, function(o) { return o.name == d; }); + if (cuboid.deleted) { + return d3.scale.category20c().range()[17]; + } else { + return getColorByQuery(0, 1/data.nodeInfos.length, cuboid.query_rate); + } + }; + $scope.currentOptions.chart.sunburst = getSunburstDispatch(); + $scope.currentOptions.title.text = 'Current Cuboid Distribution'; + $scope.currentOptions.subtitle.text = '[Cuboid Count: ' + data.nodeInfos.length + '] [Row Count: ' + data.totalRowCount + ']'; + } else if ('recommend' === type) { + $scope.recommendData = [chartData]; + $scope.recommendOptions = angular.copy(cubeConfig.baseChartOptions); + $scope.recommendOptions.caption = angular.copy(cubeConfig.recommendCaption); + $scope.recommendOptions.chart.color = function(d) { + var cuboid = _.find(data.nodeInfos, function(o) { return o.name == d; }); + if (cuboid.row_count < 0) { + return d3.scale.category20c().range()[5]; + } else { + var colorIndex = 0; + if (!cuboid.existed) { + colorIndex = 8; + } + return getColorByQuery(colorIndex, 1/data.nodeInfos.length, cuboid.query_rate); + } + }; + $scope.recommendOptions.chart.sunburst = getSunburstDispatch(); + $scope.recommendOptions.title.text = 'Recommend Cuboid Distribution'; + $scope.recommendOptions.subtitle.text = '[Cuboid Count: ' + data.nodeInfos.length + '] [Row Count: ' + data.totalRowCount + ']'; + } + }; + + // Hover behavior for highlight dimensions + function getSunburstDispatch() { + return { + dispatch: { + elementMouseover: function(t, u) { + $scope.selectCuboid = t.data.name; + $scope.$apply(); + }, + renderEnd: function(t, u) { + var chartElements = document.getElementsByClassName('nv-sunburst'); + angular.element(chartElements).on('mouseleave', function() { + $scope.selectCuboid = '0'; + $scope.$apply(); + }); + } + } + }; + }; + + // Different color for chart element by query count + function getColorByQuery(colorIndex, baseRate, queryRate) { + if (queryRate > (3 * baseRate)) { + return d3.scale.category20c().range()[colorIndex]; + } else if (queryRate > (2 * baseRate)) { + return d3.scale.category20c().range()[colorIndex+1]; + } else if (queryRate > baseRate) { + return d3.scale.category20c().range()[colorIndex+2]; + } else { + return d3.scale.category20c().range()[colorIndex+3]; + } + } }); http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/webapp/app/js/model/cubeConfig.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/model/cubeConfig.js b/webapp/app/js/model/cubeConfig.js index d04af76..e163d75 100644 --- a/webapp/app/js/model/cubeConfig.js +++ b/webapp/app/js/model/cubeConfig.js @@ -113,5 +113,79 @@ KylinApp.constant('cubeConfig', { {name:"Global Dictionary", value:"org.apache.kylin.dict.GlobalDictionaryBuilder"}, {name:"Segment Dictionary", value:"org.apache.kylin.dict.global.SegmentAppendTrieDictBuilder"} ], - needSetLengthEncodingList:['fixed_length','fixed_length_hex','int','integer'] -}); + needSetLengthEncodingList:['fixed_length','fixed_length_hex','int','integer'], + baseChartOptions: { + chart: { + type: 'sunburstChart', + height: 500, + duration: 250, + groupColorByParent: false, + tooltip: { + contentGenerator: function(obj) { + var preCalculatedStr = ''; + if (typeof obj.data.existed !== 'undefined' && obj.data.existed !== null) { + preCalculatedStr = '<tr><td align="right"><b>Existed:</b></td><td>' + obj.data.existed + '</td></tr>'; + } + var rowCountRateStr = ''; + if (obj.data.row_count) { + rowCountRateStr = '<tr><td align="right"><b>Row Count:</b></td><td>' + obj.data.row_count + '</td></tr><tr><td align="right"><b>Rollup Rate:</b></td><td>' + (obj.data.row_count * 100 / obj.data.parent_row_count).toFixed(2) + '%</td></tr>'; + } + return '<table><tbody>' + + '<tr><td align="right"><i class="fa fa-square" style="color: ' + obj.color + '; margin-right: 15px;" aria-hidden="true"></i><b>Name:</b></td><td class="key"><b>' + obj.data.name +'</b></td></tr>' + + '<tr><td align="right"><b>ID:</b></td><td>' + obj.data.cuboid_id + '</td></tr>' + + '<tr><td align="right"><b>Query Count:</b></td><td>' + obj.data.query_count + ' [' + (obj.data.query_rate * 100).toFixed(2) + '%]</td></tr>' + + '<tr><td align="right"><b>Exactly Match Count:</b></td><td>' + obj.data.exactly_match_count + '</td></tr>' + + rowCountRateStr + + preCalculatedStr + + '</tbody></table>'; + } + } + }, + title: { + enable: true, + text: '', + className: 'h4', + css: { + position: 'relative', + top: '30px' + } + }, + subtitle: { + enable: true, + text: '', + className: 'h5', + css: { + position: 'relative', + top: '40px' + } + } + }, + currentCaption: { + enable: true, + html: '<div>Existed: <i class="fa fa-square" style="color:#38c;"></i> Hottest ' + + '<i class="fa fa-square" style="color:#7bd;"></i> Hot ' + + '<i class="fa fa-square" style="color:#ade;"></i> Warm ' + + '<i class="fa fa-square" style="color:#cef;"></i> Cold ' + + '<i class="fa fa-square" style="color:#999;"></i> Retire</div>', + css: { + position: 'relative', + top: '-35px', + height: 0 + } + }, + recommendCaption: { + enable: true, + html: '<div>New: <i class="fa fa-square" style="color:#3a5;"></i> Hottest ' + + '<i class="fa fa-square" style="color:#7c7;"></i> Hot ' + + '<i class="fa fa-square" style="color:#aea;"></i> Warm ' + + '<i class="fa fa-square" style="color:#cfc;"></i> Cold ' + + '<i class="fa fa-square" style="color:#f94;"></i> Mandatory</div>', + css: { + position: 'relative', + top: '-35px', + height: 0, + 'text-align': 'left', + 'left': '-12px' + } + } +}); \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/webapp/app/js/services/cubes.js ---------------------------------------------------------------------- diff --git a/webapp/app/js/services/cubes.js b/webapp/app/js/services/cubes.js index 7e2ee00..150d932 100644 --- a/webapp/app/js/services/cubes.js +++ b/webapp/app/js/services/cubes.js @@ -17,6 +17,25 @@ */ KylinApp.factory('CubeService', ['$resource', function ($resource, config) { + function transformCuboidsResponse(data) { + var cuboids = { + nodeInfos: [], + treeNode: data.root, + totalRowCount: 0 + }; + function iterator(node, parentRowCount) { + node.parent_row_count = parentRowCount; + cuboids.nodeInfos.push(node); + cuboids.totalRowCount += node.row_count; + if (node.children.length) { + angular.forEach(node.children, function(child) { + iterator(child, node.row_count); + }); + } + }; + iterator(data.root, data.root.row_count); + return cuboids; + }; return $resource(Config.service.url + 'cubes/:cubeId/:propName/:propValue/:action', {}, { list: {method: 'GET', params: {}, isArray: true}, getValidEncodings: {method: 'GET', params: {action:"validEncodings"}, isArray: false}, @@ -34,6 +53,30 @@ KylinApp.factory('CubeService', ['$resource', function ($resource, config) { drop: {method: 'DELETE', params: {}, isArray: false}, save: {method: 'POST', params: {}, isArray: false}, update: {method: 'PUT', params: {}, isArray: false}, - getHbaseInfo: {method: 'GET', params: {propName: 'hbase'}, isArray: true} + getHbaseInfo: {method: 'GET', params: {propName: 'hbase'}, isArray: true}, + getCurrentCuboids: { + method: 'GET', + params: { + propName: 'cuboids', + propValue: 'current' + }, + isArray: false, + interceptor: { + response: function(response) { + return transformCuboidsResponse(response.data); + } + } + }, + getRecommendCuboids: { + method: 'GET', + params: {propName: 'cuboids', propValue: 'recommend'}, + isArray: false, + interceptor: { + response: function(response) { + return transformCuboidsResponse(response.data); + } + } + }, + optimize: {method: 'PUT', params: {action: 'optimize'}, isArray: false} }); }]); http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/webapp/app/less/app.less ---------------------------------------------------------------------- diff --git a/webapp/app/less/app.less b/webapp/app/less/app.less index fcba436..7a23acc 100644 --- a/webapp/app/less/app.less +++ b/webapp/app/less/app.less @@ -899,4 +899,29 @@ pre { font-size: 18px; color: #6a6a6a; } +} +/* cube planner*/ +.cube-planner-column { + margin: 0 60px; + table { + border: 0; + tr { + font-weight: bolder; + color: #EEEEEE; + th { + text-align: center; + vertical-align: middle; + width: 20%; + padding: 2px; + } + .column-in-cuobid { + color: #9E9E9E; + font-weight: bolder; + } + .column-not-in-cuboid { + color: #EEEEEE; + font-weight: bolder; + } + } + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/659c3861/webapp/app/partials/cubes/cube_detail.html ---------------------------------------------------------------------- diff --git a/webapp/app/partials/cubes/cube_detail.html b/webapp/app/partials/cubes/cube_detail.html index 674e3f0..e80bb09 100755 --- a/webapp/app/partials/cubes/cube_detail.html +++ b/webapp/app/partials/cubes/cube_detail.html @@ -41,6 +41,9 @@ ng-if="userService.hasRole('ROLE_ADMIN')"> <a href="" ng-click="cube.visiblePage='hbase';getHbaseInfo(cube)">Storage</a> </li> + <li class="{{cube.visiblePage=='planner'? 'active':''}}" ng-if="userService.hasRole('ROLE_ADMIN') || hasPermission(cube, permissions.ADMINISTRATION.mask)"> + <a href="" ng-click="cube.visiblePage='planner';getCubePlanner(cube);">Planner</a> + </li> </ul> <div class="cube-detail" ng-if="!cube.visiblePage || cube.visiblePage=='metadata'"> @@ -117,5 +120,46 @@ </div> </div> </div> - </div> + <div class="cube-detail" ng-if="cube.visiblePage=='planner'"> + <div style="padding: 15px;"> + <div class="row"> + <div class="col-sm-12"> + <h4 style="display: inline;">Cuboid Distribution</h4> + <button ng-if="enableRecommend" class="btn btn-success btn-sm pull-right" ng-click="getRecommendCuboids(cube)" ng-if="currentData"> + Recommend + </button> + <div ng-if="cube.cuboid_last_optimized" class="pull-right" style="padding: 5px;">Last Optimized Time: {{cube.cuboid_last_optimized | utcToConfigTimeZone}}</div> + </div> + </div> + <div class="row"> + <div class="col-md-6 col-sm-12"> + <nvd3 options="currentOptions" data="currentData" api="currentChart.api"></nvd3> + </div> + <div class="col-md-6 col-sm-12" ng-if="recommendData"> + <nvd3 options="recommendOptions" data="recommendData"></nvd3> + </div> + </div> + <div class="row cube-planner-column" ng-if="currentData || recommendData"> + <table class="table table-bordered"> + <tbody> + <tr ng-repeat="row in cube.detail.rowkey.rowkey_columns track by $index" ng-if="$index % 5 == 0" class="row"> + <th ng-class="{'column-in-cuobid': selectCuboid.charAt($index) == 1, 'column-not-in-cuboid': selectCuboid.charAt($index) == 0}">{{cube.detail.rowkey.rowkey_columns[$index].column}}</th> + <th ng-class="{'column-in-cuobid': selectCuboid.charAt($index + 1) == 1, 'column-not-in-cuboid': selectCuboid.charAt($index + 1) == 0}">{{cube.detail.rowkey.rowkey_columns[$index + 1].column}}</th> + <th ng-class="{'column-in-cuobid': selectCuboid.charAt($index + 2) == 1, 'column-not-in-cuboid': selectCuboid.charAt($index + 2) == 0}">{{cube.detail.rowkey.rowkey_columns[$index + 2].column}}</th> + <th ng-class="{'column-in-cuobid': selectCuboid.charAt($index + 3) == 1, 'column-not-in-cuboid': selectCuboid.charAt($index + 3) == 0}">{{cube.detail.rowkey.rowkey_columns[$index + 3].column}}</th> + <th ng-class="{'column-in-cuobid': selectCuboid.charAt($index + 4) == 1, 'column-not-in-cuboid': selectCuboid.charAt($index + 4) == 0}">{{cube.detail.rowkey.rowkey_columns[$index + 4].column}}</th> + </tr> + </tbody> + </table> + </div> + <div class="row"> + <div class="col-sm-12"> + <button class="btn btn-success btn-next pull-right" ng-click="optimizeCuboids(cube)" ng-if="recommendData"> + Optimize + </button> + </div> + </div> + </div> + </div> +</div>