This is an automated email from the ASF dual-hosted git repository. sushuang pushed a commit to branch fix/aggregate in repository https://gitbox.apache.org/repos/asf/incubator-echarts.git
commit 4eee8b87fd4c5bb922338a565dc00545ec9522bb Author: 100pah <sushuang0...@gmail.com> AuthorDate: Thu Nov 19 17:50:59 2020 +0800 test: update myTransform, support average, Q1, median, Q3. --- build/config.js | 3 +- test/data-transform-aggregate.html | 292 +++++++++++++++++ test/lib/myTransform/dist/myTransform.js | 347 +++++++++++++++++--- test/lib/myTransform/dist/myTransform.js.map | 2 +- test/lib/myTransform/src/aggregate.ts | 463 +++++++++++++++++++++------ 5 files changed, 954 insertions(+), 153 deletions(-) diff --git a/build/config.js b/build/config.js index 53cec29..c1c31b7 100644 --- a/build/config.js +++ b/build/config.js @@ -221,7 +221,8 @@ exports.createMyTransform = function () { clean: true }, { include: [ - nodePath.resolve(ecDir, 'test/lib/myTransform/src/**/*.ts') + nodePath.resolve(ecDir, 'test/lib/myTransform/src/**/*.ts'), + nodePath.resolve(ecDir, 'src/**/*.ts') ] }), input: input, diff --git a/test/data-transform-aggregate.html b/test/data-transform-aggregate.html new file mode 100644 index 0000000..cfce0ab --- /dev/null +++ b/test/data-transform-aggregate.html @@ -0,0 +1,292 @@ +<!DOCTYPE html> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + + +<html> + <head> + <meta charset="utf-8"> + <meta name="viewport" content="width=device-width, initial-scale=1" /> + <script src="lib/esl.js"></script> + <script src="lib/config.js"></script> + <script src="lib/jquery.min.js"></script> + <script src="lib/facePrint.js"></script> + <script src="lib/testHelper.js"></script> + <!-- <script src="ut/lib/canteen.js"></script> --> + <link rel="stylesheet" href="lib/reset.css" /> + </head> + <body> + <style> + </style> + + + + <div id="main_sum"></div> + <div id="main_boxplot"></div> + <div id="main_average"></div> + + + + + <script> + require(['echarts', 'myTransform'], function (echarts, myTransform) { + + $.get('data/life-expectancy-table.json', function (_rawData) { + + echarts.registerTransform(myTransform.aggregate); + + option = { + dataset: [{ + id: 'raw', + source: _rawData + }, { + id: 'income_aggregate', + fromDatasetId: 'raw', + transform: [{ + type: 'filter', + config: { + dimension: 'Year', gte: 1950 + } + }, { + type: 'myTransform:aggregate', + config: { + resultDimensions: [ + { from: 'Income', method: 'sum' }, + { from: 'Country' } + ], + groupBy: 'Country' + } + }] + }], + title: { + text: 'Income sum since 1950' + }, + tooltip: { + trigger: 'axis' + }, + xAxis: { + name: 'Income', + nameLocation: 'middle', + nameGap: 30 + }, + yAxis: { + type: 'category' + }, + series: { + type: 'bar', + datasetId: 'income_aggregate', + label: { + show: true, + position: 'right' + }, + encode: { + x: 'Income', + y: 'Country', + itemName: ['Country'], + tooltip: ['Income'] + } + } + }; + + var chart = testHelper.create(echarts, 'main_sum', { + title: [ + 'Aggregate sum' + ], + height: 400, + option: option + }); + + }); + }); + </script> + + + + + + <script> + require(['echarts', 'myTransform'], function (echarts, myTransform) { + + $.get('data/life-expectancy-table.json', function (_rawData) { + + echarts.registerTransform(myTransform.aggregate); + + option = { + dataset: [{ + id: 'raw', + source: _rawData + }, { + id: 'income_aggregate', + fromDatasetId: 'raw', + transform: [{ + type: 'filter', + config: { + and: [{ + dimension: 'Year', gte: 1950 + // }, { + // dimension: 'Country', eq: 'Japan' + }] + } + }, { + type: 'myTransform:aggregate', + config: { + resultDimensions: [ + { name: 'min', from: 'Income', method: 'min' }, + { name: 'Q1', from: 'Income', method: 'Q1' }, + { name: 'Q2', from: 'Income', method: 'Q2' }, + { name: 'Q3', from: 'Income', method: 'Q3' }, + { name: 'max', from: 'Income', method: 'max' }, + { name: 'Country', from: 'Country' } + ], + groupBy: 'Country' + } + }] + }], + title: { + text: 'Income sum since 1950' + }, + tooltip: { + trigger: 'axis' + }, + xAxis: { + name: 'Income', + nameLocation: 'middle', + nameGap: 30 + }, + yAxis: { + type: 'category' + }, + series: { + type: 'boxplot', + datasetId: 'income_aggregate', + label: { + show: true, + position: 'right' + }, + encode: { + x: ['min', 'Q1', 'Q2', 'Q3', 'max'], + y: 'Country', + itemName: ['Country'], + tooltip: ['min', 'Q1', 'Q2', 'Q3', 'max'] + } + } + }; + + var chart = testHelper.create(echarts, 'main_boxplot', { + title: [ + 'Aggregate Q1 Q2 Q3 min max' + ], + height: 600, + option: option + }); + + }); + }); + </script> + + + + + + + + + + <script> + require(['echarts', 'myTransform'], function (echarts, myTransform) { + + $.get('data/life-expectancy-table.json', function (_rawData) { + + echarts.registerTransform(myTransform.aggregate); + + option = { + dataset: [{ + id: 'raw', + source: _rawData + }, { + id: 'income_aggregate', + fromDatasetId: 'raw', + transform: [{ + type: 'filter', + print: true, + config: { + and: [{ + dimension: 'Year', gte: 1950 + // }, { + // dimension: 'Country', eq: 'Japan' + }] + } + }, { + type: 'myTransform:aggregate', + config: { + resultDimensions: [ + { from: 'Income', method: 'average' }, + { from: 'Country' } + ], + groupBy: 'Country' + } + }] + }], + title: { + text: 'Income sum since 1950' + }, + tooltip: { + trigger: 'axis' + }, + xAxis: { + name: 'Income', + nameLocation: 'middle', + nameGap: 30 + }, + yAxis: { + type: 'category' + }, + series: { + type: 'bar', + datasetId: 'income_aggregate', + label: { + show: true, + position: 'right' + }, + encode: { + x: 'Income', + y: 'Country', + itemName: ['Country'], + tooltip: ['Income'] + } + } + }; + + var chart = testHelper.create(echarts, 'main_average', { + title: [ + 'Aggregate average' + ], + height: 600, + option: option + }); + + }); + }); + </script> + + + + </body> +</html> + diff --git a/test/lib/myTransform/dist/myTransform.js b/test/lib/myTransform/dist/myTransform.js index 9edeb7f..a2f97b5 100644 --- a/test/lib/myTransform/dist/myTransform.js +++ b/test/lib/myTransform/dist/myTransform.js @@ -26,12 +26,56 @@ } }; var arrayProto = Array.prototype; + var nativeForEach = arrayProto.forEach; var nativeSlice = arrayProto.slice; + var nativeMap = arrayProto.map; var ctorFunction = function () {}.constructor; var protoFunction = ctorFunction ? ctorFunction.prototype : null; + function each(arr, cb, context) { + if (!(arr && cb)) { + return; + } + + if (arr.forEach && arr.forEach === nativeForEach) { + arr.forEach(cb, context); + } else if (arr.length === +arr.length) { + for (var i = 0, len = arr.length; i < len; i++) { + cb.call(context, arr[i], i, arr); + } + } else { + for (var key in arr) { + if (arr.hasOwnProperty(key)) { + cb.call(context, arr[key], key, arr); + } + } + } + } + + function map(arr, cb, context) { + if (!arr) { + return []; + } + + if (!cb) { + return slice(arr); + } + + if (arr.map && arr.map === nativeMap) { + return arr.map(cb, context); + } else { + var result = []; + + for (var i = 0, len = arr.length; i < len; i++) { + result.push(cb.call(context, arr[i], i, arr)); + } + + return result; + } + } + function bindPolyfill(func, context) { var args = []; @@ -50,6 +94,16 @@ return typeof value === 'function'; } + function slice(arr) { + var args = []; + + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + + return nativeSlice.apply(arr, args); + } + function assert(condition, message) { if (!condition) { throw new Error(message); @@ -60,6 +114,14 @@ return own.hasOwnProperty(prop); } + function quantile(ascArr, p) { + var H = (ascArr.length - 1) * p + 1; + var h = Math.floor(H); + var v = +ascArr[h - 1]; + var e = H - h; + return e ? v + e * (ascArr[h] - v) : v; + } + var METHOD_INTERNAL = { 'SUM': true, 'COUNT': true, @@ -71,49 +133,135 @@ 'MIN': true, 'MAX': true }; + var METHOD_NEEDS_COLLECT = { + AVERAGE: ['COUNT'] + }; + var METHOD_NEEDS_GATHER_VALUES = { + Q1: true, + Q2: true, + Q3: true + }; var METHOD_ALIAS = { MEDIAN: 'Q2' }; + + var ResultDimInfoInternal = function () { + function ResultDimInfoInternal(index, indexInUpstream, method, name, needGatherValues) { + this.collectionInfoList = []; + this.gatheredValuesByGroup = {}; + this.gatheredValuesNoGroup = []; + this.needGatherValues = false; + this._collectionInfoMap = {}; + this.method = method; + this.name = name; + this.index = index; + this.indexInUpstream = indexInUpstream; + this.needGatherValues = needGatherValues; + } + + ResultDimInfoInternal.prototype.addCollectionInfo = function (item) { + this._collectionInfoMap[item.method] = this.collectionInfoList.length; + this.collectionInfoList.push(item); + }; + + ResultDimInfoInternal.prototype.getCollectionInfo = function (method) { + return this.collectionInfoList[this._collectionInfoMap[method]]; + }; + + ResultDimInfoInternal.prototype.gatherValue = function (groupByDimInfo, groupVal, value) { + value = +value; + + if (groupByDimInfo) { + if (groupVal != null) { + var groupValStr = groupVal + ''; + var values = this.gatheredValuesByGroup[groupValStr] || (this.gatheredValuesByGroup[groupValStr] = []); + values.push(value); + } + } else { + this.gatheredValuesNoGroup.push(value); + } + }; + + return ResultDimInfoInternal; + }(); + var transform$1 = { type: 'myTransform:aggregate', transform: function (params) { var upstream = params.upstream; var config = params.config; - var dimWrap = prepareDimensions(config, upstream); - var resultDimInfoList = dimWrap.resultDimInfoList; - var resultDimensions = dimWrap.resultDimensions; var groupByDimInfo = prepareGroupByDimInfo(config, upstream); - var finalResult = travel(groupByDimInfo, upstream, resultDimInfoList, createResultLine, aggregateResultLine); + + var _a = prepareDimensions(config, upstream, groupByDimInfo), + finalResultDimInfoList = _a.finalResultDimInfoList, + collectionDimInfoList = _a.collectionDimInfoList; + + var collectionResult; + + if (collectionDimInfoList.length) { + collectionResult = travel(groupByDimInfo, upstream, collectionDimInfoList, createCollectionResultLine, updateCollectionResultLine); + } + + each(collectionDimInfoList, function (dimInfo) { + dimInfo.__collectionResult = collectionResult; + asc(dimInfo.gatheredValuesNoGroup); + each(dimInfo.gatheredValuesByGroup, function (values) { + asc(values); + }); + }); + var finalResult = travel(groupByDimInfo, upstream, finalResultDimInfoList, createFinalResultLine, updateFinalResultLine); return { - dimensions: resultDimensions, + dimensions: map(finalResultDimInfoList, function (item) { + return item.name; + }), data: finalResult.outList }; } }; - function prepareDimensions(config, upstream) { + function prepareDimensions(config, upstream, groupByDimInfo) { var resultDimensionsConfig = config.resultDimensions; - var resultDimInfoList = []; - var resultDimensions = []; + var finalResultDimInfoList = []; + var collectionDimInfoList = []; + var gIndexInLine = 0; for (var i = 0; i < resultDimensionsConfig.length; i++) { var resultDimInfoConfig = resultDimensionsConfig[i]; - var resultDimInfo = upstream.getDimensionInfo(resultDimInfoConfig.from); - assert(resultDimInfo, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); - resultDimInfo.method = normalizeMethod(resultDimInfoConfig.method); - assert(resultDimInfo.method, 'method is required'); - resultDimInfoList.push(resultDimInfo); + var dimInfoInUpstream = upstream.getDimensionInfo(resultDimInfoConfig.from); + assert(dimInfoInUpstream, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); + var rawMethod = resultDimInfoConfig.method; + assert(groupByDimInfo.index !== dimInfoInUpstream.index || rawMethod == null, "Dimension " + dimInfoInUpstream.name + " is the \"groupBy\" dimension, must not have any \"method\"."); + var method = normalizeMethod(rawMethod); + assert(method, 'method is required'); + var name_1 = resultDimInfoConfig.name != null ? resultDimInfoConfig.name : dimInfoInUpstream.name; + var finalResultDimInfo = new ResultDimInfoInternal(finalResultDimInfoList.length, dimInfoInUpstream.index, method, name_1, hasOwn(METHOD_NEEDS_GATHER_VALUES, method)); + finalResultDimInfoList.push(finalResultDimInfo); + var needCollect = false; + + if (hasOwn(METHOD_NEEDS_COLLECT, method)) { + needCollect = true; + var collectionTargetMethods = METHOD_NEEDS_COLLECT[method]; + + for (var j = 0; j < collectionTargetMethods.length; j++) { + finalResultDimInfo.addCollectionInfo({ + method: collectionTargetMethods[j], + indexInLine: gIndexInLine++ + }); + } + } - if (resultDimInfoConfig.name != null) { - resultDimInfo.name = resultDimInfoConfig.name; + if (hasOwn(METHOD_NEEDS_GATHER_VALUES, method)) { + needCollect = true; } - resultDimensions.push(resultDimInfo.name); + if (needCollect) { + collectionDimInfoList.push(finalResultDimInfo); + } } return { - resultDimensions: resultDimensions, - resultDimInfoList: resultDimInfoList + collectionDimInfoList: collectionDimInfoList, + finalResultDimInfoList: finalResultDimInfoList }; } @@ -129,12 +277,12 @@ return groupByDimInfo; } - function travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doAggregate) { + function travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doUpdate) { var outList = []; - var groupMap; + var mapByGroup; if (groupByDimInfo) { - groupMap = {}; + mapByGroup = {}; for (var dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { var groupByVal = upstream.retrieveValue(dataIndex, groupByDimInfo.index); @@ -145,26 +293,26 @@ var groupByValStr = groupByVal + ''; - if (!hasOwn(groupMap, groupByValStr)) { + if (!hasOwn(mapByGroup, groupByValStr)) { var newLine = doCreate(upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal); outList.push(newLine); - groupMap[groupByValStr] = newLine; + mapByGroup[groupByValStr] = newLine; } else { - var targetLine = groupMap[groupByValStr]; - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo); + var targetLine = mapByGroup[groupByValStr]; + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo, groupByVal); } } } else { var targetLine = doCreate(upstream, 0, resultDimInfoList); outList.push(targetLine); - for (var dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList); + for (var dataIndex = 1, len = upstream.count(); dataIndex < len; dataIndex++) { + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList); } } return { - groupMap: groupMap, + mapByGroup: mapByGroup, outList: outList }; } @@ -180,37 +328,146 @@ return methodInternal; } - var createResultLine = function (upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal) { + var createCollectionResultLine = function (upstream, dataIndex, collectionDimInfoList, groupByDimInfo, groupByVal) { var newLine = []; - for (var j = 0; j < resultDimInfoList.length; j++) { - var resultDimInfo = resultDimInfoList[j]; - var method = resultDimInfo.method; - newLine[j] = groupByDimInfo && resultDimInfo.index === groupByDimInfo.index ? groupByVal : method === 'SUM' || method === 'COUNT' ? 0 : upstream.retrieveValue(dataIndex, resultDimInfo.index); + for (var i = 0; i < collectionDimInfoList.length; i++) { + var dimInfo = collectionDimInfoList[i]; + var collectionInfoList = dimInfo.collectionInfoList; + + for (var j = 0; j < collectionInfoList.length; j++) { + var collectionInfo = collectionInfoList[j]; + newLine[collectionInfo.indexInLine] = +lineCreator[collectionInfo.method](upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + + if (dimInfo.needGatherValues) { + var val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } } return newLine; }; - var aggregateResultLine = function (upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo) { - for (var j = 0; j < resultDimInfoList.length; j++) { - var resultDimInfo = resultDimInfoList[j]; - var method = resultDimInfo.method; + var updateCollectionResultLine = function (upstream, dataIndex, targetLine, collectionDimInfoList, groupByDimInfo, groupByVal) { + for (var i = 0; i < collectionDimInfoList.length; i++) { + var dimInfo = collectionDimInfoList[i]; + var collectionInfoList = dimInfo.collectionInfoList; - if (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) { - continue; + for (var j = 0; j < collectionInfoList.length; j++) { + var collectionInfo = collectionInfoList[j]; + var indexInLine = collectionInfo.indexInLine; + targetLine[indexInLine] = +lineUpdater[collectionInfo.method](targetLine[indexInLine], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); } - if (method === 'SUM') { - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index); - } else if (method === 'COUNT') { - targetLine[j] += 1; - } else if (method === 'AVERAGE') { - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) / 1; + if (dimInfo.needGatherValues) { + var val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); } } }; + var createFinalResultLine = function (upstream, dataIndex, finalResultDimInfoList, groupByDimInfo, groupByVal) { + var newLine = []; + + for (var i = 0; i < finalResultDimInfoList.length; i++) { + var dimInfo = finalResultDimInfoList[i]; + var method = dimInfo.method; + newLine[i] = isGroupByDimension(groupByDimInfo, dimInfo) ? groupByVal : lineCreator[method](upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + + return newLine; + }; + + var updateFinalResultLine = function (upstream, dataIndex, targetLine, finalResultDimInfoList, groupByDimInfo, groupByVal) { + for (var i = 0; i < finalResultDimInfoList.length; i++) { + var dimInfo = finalResultDimInfoList[i]; + + if (isGroupByDimension(groupByDimInfo, dimInfo)) { + continue; + } + + var method = dimInfo.method; + targetLine[i] = lineUpdater[method](targetLine[i], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + }; + + function isGroupByDimension(groupByDimInfo, targetDimInfo) { + return groupByDimInfo && targetDimInfo.indexInUpstream === groupByDimInfo.index; + } + + function asc(list) { + list.sort(function (a, b) { + return a - b; + }); + } + + var lineCreator = { + 'SUM': function () { + return 0; + }, + 'COUNT': function () { + return 1; + }, + 'FIRST': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MIN': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MAX': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'AVERAGE': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + var collectLine = groupByDimInfo ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] : dimInfo.__collectionResult.outList[0]; + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.25, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q2': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.5, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q3': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.75, dimInfo, groupByDimInfo, groupByVal); + } + }; + var lineUpdater = { + 'SUM': function (val, upstream, dataIndex, dimInfo) { + return val + upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'COUNT': function (val) { + return val + 1; + }, + 'FIRST': function (val) { + return val; + }, + 'MIN': function (val, upstream, dataIndex, dimInfo) { + return Math.min(val, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream)); + }, + 'MAX': function (val, upstream, dataIndex, dimInfo) { + return Math.max(val, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream)); + }, + 'AVERAGE': function (val, upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + var collectLine = groupByDimInfo ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] : dimInfo.__collectionResult.outList[0]; + return val + upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1': function (val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q2': function (val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q3': function (val, upstream, dataIndex, dimInfo) { + return val; + } + }; + + function lineCreatorForQ(percent, dimInfo, groupByDimInfo, groupByVal) { + var gatheredValues = groupByDimInfo ? dimInfo.gatheredValuesByGroup[groupByVal + ''] : dimInfo.gatheredValuesNoGroup; + return quantile(gatheredValues, percent); + } + exports.aggregate = transform$1; exports.id = transform; Object.defineProperty(exports, '__esModule', { diff --git a/test/lib/myTransform/dist/myTransform.js.map b/test/lib/myTransform/dist/myTransform.js.map index 4b03361..7628c23 100644 --- a/test/lib/myTransform/dist/myTransform.js.map +++ b/test/lib/myTransform/dist/myTransform.js.map @@ -1 +1 @@ -{"version":3,"file":"myTransform.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"} \ No newline at end of file +{"version":3,"file":"myTransform.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; [...] \ No newline at end of file diff --git a/test/lib/myTransform/src/aggregate.ts b/test/lib/myTransform/src/aggregate.ts index 68a4d04..4b6e02a 100644 --- a/test/lib/myTransform/src/aggregate.ts +++ b/test/lib/myTransform/src/aggregate.ts @@ -21,9 +21,10 @@ import { DataTransformOption, ExternalDataTransform, ExternalSource, ExternalDimensionDefinition } from '../../../../src/data/helper/transform'; import { - DimensionName, DimensionLoose, DimensionDefinitionLoose, OptionDataValue + DimensionName, DimensionLoose, OptionDataValue } from '../../../../src/util/types'; -import { hasOwn, assert } from 'zrender/src/core/util'; +import { hasOwn, assert, map, each } from 'zrender/src/core/util'; +import { quantile } from '../../../../src/util/number'; /** @@ -48,7 +49,7 @@ import { hasOwn, assert } from 'zrender/src/core/util'; * { from: 'aa', method: 'sum' }, * { from: 'bb', method: 'count' }, * { from: 'cc' }, // method by default: use the first value. - * { from: 'dd', method: 'Q1', boundIQR: 1 }, + * { from: 'dd', method: 'Q1' }, * { from: 'tag' } * ], * groupBy: 'tag' @@ -73,13 +74,6 @@ import { hasOwn, assert } from 'zrender/src/core/util'; * 'Q2' or 'median' * 'min' * 'max' - * - * Current supported other arguments: - * boundIQR: - * Data less than min bound is outlier. - * default 1.5, means Q1 - 1.5 * (Q3 - Q1). - * If 'none'/0 passed, min bound will not be used. - * */ export interface AggregateTransformOption extends DataTransformOption { @@ -113,7 +107,9 @@ const METHOD_INTERNAL = { 'MAX': true } as const; const METHOD_NEEDS_COLLECT = { - AVERAGE: true, + AVERAGE: ['COUNT'] +} as const; +const METHOD_NEEDS_GATHER_VALUES = { Q1: true, Q2: true, Q3: true @@ -127,23 +123,86 @@ type AggregateMethodLoose = | 'sum' | 'count' | 'first' | 'average' | 'Q1' | 'Q2' | 'Q3' | 'median' | 'min' | 'max'; type AggregateMethodInternal = keyof typeof METHOD_INTERNAL; -interface ResultDimInfoInternal extends ExternalDimensionDefinition { - method: AggregateMethodInternal; + +class ResultDimInfoInternal { + + readonly method: AggregateMethodInternal; + readonly name: DimensionName; + readonly index: number; + readonly indexInUpstream: number; + + readonly collectionInfoList = [] as { + method: AggregateMethodInternal; + indexInLine: number; + }[]; + + // FIXME: refactor + readonly gatheredValuesByGroup: { [groupVal: string]: number[] } = {}; + readonly gatheredValuesNoGroup = [] as number[]; + readonly needGatherValues: boolean = false; + + __collectionResult: TravelResult<CollectionResultLine>; + + private _collectionInfoMap = {} as { + // number is the index of `list` + [method in AggregateMethodInternal]: number + }; + + constructor( + index: number, + indexInUpstream: number, + method: AggregateMethodInternal, + name: DimensionName, + needGatherValues: boolean + ) { + this.method = method; + this.name = name; + this.index = index; + this.indexInUpstream = indexInUpstream; + this.needGatherValues = needGatherValues; + } + + addCollectionInfo(item: ResultDimInfoInternal['collectionInfoList'][number]) { + this._collectionInfoMap[item.method] = this.collectionInfoList.length; + this.collectionInfoList.push(item); + } + + getCollectionInfo(method: AggregateMethodInternal) { + return this.collectionInfoList[this._collectionInfoMap[method]]; + } + + // FIXME: temp implementation. Need refactor. + gatherValue(groupByDimInfo: ExternalDimensionDefinition, groupVal: OptionDataValue, value: OptionDataValue) { + // FIXME: convert to number compulsorily temporarily. + value = +value; + if (groupByDimInfo) { + if (groupVal != null) { + const groupValStr = groupVal + ''; + const values = this.gatheredValuesByGroup[groupValStr] + || (this.gatheredValuesByGroup[groupValStr] = []); + values.push(value); + } + } + else { + this.gatheredValuesNoGroup.push(value); + } + } } -type CreateInTravel = ( +type CreateInTravel<LINE> = ( upstream: ExternalSource, dataIndex: number, - resultDimInfoList: ResultDimInfoInternal[], + dimInfoList: ResultDimInfoInternal[], groupByDimInfo?: ExternalDimensionDefinition, groupByVal?: OptionDataValue -) => void; -type AggregateInTravel = ( +) => LINE; +type UpdateInTravel<LINE> = ( upstream: ExternalSource, dataIndex: number, - targetLine: unknown, - resultDimInfoList: ResultDimInfoInternal[], - groupByDimInfo?: ExternalDimensionDefinition + targetLine: LINE, + dimInfoList: ResultDimInfoInternal[], + groupByDimInfo?: ExternalDimensionDefinition, + groupByVal?: OptionDataValue ) => void; export const transform: ExternalDataTransform<AggregateTransformOption> = { @@ -154,26 +213,43 @@ export const transform: ExternalDataTransform<AggregateTransformOption> = { const upstream = params.upstream; const config = params.config; - const dimWrap = prepareDimensions(config, upstream); - const resultDimInfoList = dimWrap.resultDimInfoList; - const resultDimensions = dimWrap.resultDimensions; - const groupByDimInfo = prepareGroupByDimInfo(config, upstream); + const { finalResultDimInfoList, collectionDimInfoList } = prepareDimensions( + config, upstream, groupByDimInfo + ); // Collect - // const collectResult; - // const dimInfoListForCollect = makeDimInfoListForCollect(resultDimInfoList); - // if (dimInfoListForCollect.length) { - // collectResult = travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doAggregate); - // } + let collectionResult: TravelResult<CollectionResultLine>; + if (collectionDimInfoList.length) { + collectionResult = travel( + groupByDimInfo, + upstream, + collectionDimInfoList, + createCollectionResultLine, + updateCollectionResultLine + ); + } + + each(collectionDimInfoList, dimInfo => { + dimInfo.__collectionResult = collectionResult; + // FIXME: just for Q1, Q2, Q3: need asc. + asc(dimInfo.gatheredValuesNoGroup); + each(dimInfo.gatheredValuesByGroup, values => { + asc(values); + }); + }); // Calculate const finalResult = travel( - groupByDimInfo, upstream, resultDimInfoList, createResultLine, aggregateResultLine + groupByDimInfo, + upstream, + finalResultDimInfoList, + createFinalResultLine, + updateFinalResultLine ); return { - dimensions: resultDimensions, + dimensions: map(finalResultDimInfoList, item => item.name), data: finalResult.outList }; } @@ -181,34 +257,65 @@ export const transform: ExternalDataTransform<AggregateTransformOption> = { function prepareDimensions( config: AggregateTransformOption['config'], - upstream: ExternalSource + upstream: ExternalSource, + groupByDimInfo: ExternalDimensionDefinition ): { - resultDimInfoList: ResultDimInfoInternal[]; - resultDimensions: DimensionDefinitionLoose[]; + finalResultDimInfoList: ResultDimInfoInternal[]; + collectionDimInfoList: ResultDimInfoInternal[]; } { const resultDimensionsConfig = config.resultDimensions; - const resultDimInfoList: ResultDimInfoInternal[] = []; - const resultDimensions: DimensionDefinitionLoose[] = []; + const finalResultDimInfoList: ResultDimInfoInternal[] = []; + const collectionDimInfoList: ResultDimInfoInternal[] = []; + let gIndexInLine = 0; for (let i = 0; i < resultDimensionsConfig.length; i++) { const resultDimInfoConfig = resultDimensionsConfig[i]; - const resultDimInfo = upstream.getDimensionInfo(resultDimInfoConfig.from) as ResultDimInfoInternal; - assert(resultDimInfo, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); + const dimInfoInUpstream = upstream.getDimensionInfo(resultDimInfoConfig.from); + assert(dimInfoInUpstream, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); - resultDimInfo.method = normalizeMethod(resultDimInfoConfig.method); - assert(resultDimInfo.method, 'method is required'); + const rawMethod = resultDimInfoConfig.method; - resultDimInfoList.push(resultDimInfo); + assert( + groupByDimInfo.index !== dimInfoInUpstream.index || rawMethod == null, + `Dimension ${dimInfoInUpstream.name} is the "groupBy" dimension, must not have any "method".` + ); - if (resultDimInfoConfig.name != null) { - resultDimInfo.name = resultDimInfoConfig.name; - } + const method = normalizeMethod(rawMethod); + assert(method, 'method is required'); + + const name = resultDimInfoConfig.name != null ? resultDimInfoConfig.name : dimInfoInUpstream.name; - resultDimensions.push(resultDimInfo.name); + const finalResultDimInfo = new ResultDimInfoInternal( + finalResultDimInfoList.length, + dimInfoInUpstream.index, + method, + name, + hasOwn(METHOD_NEEDS_GATHER_VALUES, method) + ); + finalResultDimInfoList.push(finalResultDimInfo); + + // For collection. + let needCollect = false; + if (hasOwn(METHOD_NEEDS_COLLECT, method)) { + needCollect = true; + const collectionTargetMethods = METHOD_NEEDS_COLLECT[method as keyof typeof METHOD_NEEDS_COLLECT]; + for (let j = 0; j < collectionTargetMethods.length; j++) { + finalResultDimInfo.addCollectionInfo({ + method: collectionTargetMethods[j], + indexInLine: gIndexInLine++ + }); + } + } + if (hasOwn(METHOD_NEEDS_GATHER_VALUES, method)) { + needCollect = true; + } + if (needCollect) { + collectionDimInfoList.push(finalResultDimInfo); + } } - return { resultDimensions, resultDimInfoList }; + return { collectionDimInfoList, finalResultDimInfoList }; } function prepareGroupByDimInfo( @@ -224,21 +331,23 @@ function prepareGroupByDimInfo( return groupByDimInfo; } -function travel( +interface TravelResult<LINE> { + mapByGroup: { [groupVal: string]: LINE }; + outList: LINE[]; +} + +function travel<LINE>( groupByDimInfo: ExternalDimensionDefinition, upstream: ExternalSource, resultDimInfoList: ResultDimInfoInternal[], - doCreate: CreateInTravel, - doAggregate: AggregateInTravel -): { - groupMap: { [groupVal in string]: unknown }; - outList: unknown[]; -} { - const outList: unknown[] = []; - let groupMap: { [groupVal in string]: unknown }; + doCreate: CreateInTravel<LINE>, + doUpdate: UpdateInTravel<LINE> +): TravelResult<LINE> { + const outList: TravelResult<LINE>['outList'] = []; + let mapByGroup: TravelResult<LINE>['mapByGroup']; if (groupByDimInfo) { - groupMap = {}; + mapByGroup = {}; for (let dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { const groupByVal = upstream.retrieveValue(dataIndex, groupByDimInfo.index); @@ -250,43 +359,28 @@ function travel( const groupByValStr = groupByVal + ''; - if (!hasOwn(groupMap, groupByValStr)) { + if (!hasOwn(mapByGroup, groupByValStr)) { const newLine = doCreate(upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal); outList.push(newLine); - groupMap[groupByValStr] = newLine; + mapByGroup[groupByValStr] = newLine; } else { - const targetLine = groupMap[groupByValStr]; - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo); + const targetLine = mapByGroup[groupByValStr]; + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo, groupByVal); } } } else { const targetLine = doCreate(upstream, 0, resultDimInfoList); outList.push(targetLine); - for (let dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList); + for (let dataIndex = 1, len = upstream.count(); dataIndex < len; dataIndex++) { + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList); } } - return { - groupMap: groupMap, - outList: outList - }; + return { mapByGroup, outList }; } -// function makeDimInfoListForCollect(resultDimInfoList) { -// const dimInfoListForCollect = []; -// for (const j = 0; j < resultDimInfoList.length; j++) { -// const resultDimInfo = resultDimInfoList[j]; -// const method = resultDimInfo.method; -// if (hasOwn(METHOD_NEEDS_COLLECT, method)) { -// dimInfoListForCollect.push(resultDimInfo); -// } -// } -// return dimInfoListForCollect; -// } - function normalizeMethod(method: AggregateMethodLoose): AggregateMethodInternal { if (method == null) { return 'FIRST'; @@ -299,44 +393,201 @@ function normalizeMethod(method: AggregateMethodLoose): AggregateMethodInternal return methodInternal; } -const createResultLine: CreateInTravel = ( - upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal + + +type CollectionResultLine = number[]; + +const createCollectionResultLine: CreateInTravel<CollectionResultLine> = ( + upstream, dataIndex, collectionDimInfoList, groupByDimInfo, groupByVal +) => { + const newLine = [] as number[]; + for (let i = 0; i < collectionDimInfoList.length; i++) { + const dimInfo = collectionDimInfoList[i]; + const collectionInfoList = dimInfo.collectionInfoList; + for (let j = 0; j < collectionInfoList.length; j++) { + const collectionInfo = collectionInfoList[j]; + // FIXME: convert to number compulsorily temporarily. + newLine[collectionInfo.indexInLine] = +lineCreator[collectionInfo.method]( + upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } + // FIXME: refactor + if (dimInfo.needGatherValues) { + const val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } + } + return newLine; +}; + +const updateCollectionResultLine: UpdateInTravel<CollectionResultLine> = ( + upstream, dataIndex, targetLine: number[], collectionDimInfoList, groupByDimInfo, groupByVal +) => { + for (let i = 0; i < collectionDimInfoList.length; i++) { + const dimInfo = collectionDimInfoList[i]; + const collectionInfoList = dimInfo.collectionInfoList; + for (let j = 0; j < collectionInfoList.length; j++) { + const collectionInfo = collectionInfoList[j]; + const indexInLine = collectionInfo.indexInLine; + // FIXME: convert to number compulsorily temporarily. + targetLine[indexInLine] = +lineUpdater[collectionInfo.method]( + targetLine[indexInLine], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } + // FIXME: refactor + if (dimInfo.needGatherValues) { + const val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } + } +}; + + + +type FinalResultLine = OptionDataValue[]; + +const createFinalResultLine: CreateInTravel<FinalResultLine> = ( + upstream, dataIndex, finalResultDimInfoList, groupByDimInfo, groupByVal ) => { const newLine = []; - for (let j = 0; j < resultDimInfoList.length; j++) { - const resultDimInfo = resultDimInfoList[j]; - const method = resultDimInfo.method; - newLine[j] = (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) + for (let i = 0; i < finalResultDimInfoList.length; i++) { + const dimInfo = finalResultDimInfoList[i]; + const method = dimInfo.method; + newLine[i] = isGroupByDimension(groupByDimInfo, dimInfo) ? groupByVal - : (method === 'SUM' || method === 'COUNT') - ? 0 - // By default, method: 'first' - : upstream.retrieveValue(dataIndex, resultDimInfo.index); + : lineCreator[method]( + upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); } return newLine; }; -const aggregateResultLine: AggregateInTravel = ( - upstream, dataIndex, targetLine: number[], resultDimInfoList, groupByDimInfo +const updateFinalResultLine: UpdateInTravel<FinalResultLine> = ( + upstream, dataIndex, targetLine, finalResultDimInfoList, groupByDimInfo, groupByVal ) => { - for (let j = 0; j < resultDimInfoList.length; j++) { - const resultDimInfo = resultDimInfoList[j]; - const method = resultDimInfo.method; - - if (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) { + for (let i = 0; i < finalResultDimInfoList.length; i++) { + const dimInfo = finalResultDimInfoList[i]; + if (isGroupByDimension(groupByDimInfo, dimInfo)) { continue; } + const method = dimInfo.method; + targetLine[i] = lineUpdater[method]( + targetLine[i], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } +}; - if (method === 'SUM') { - // FIXME: handle other types - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) as number; - } - else if (method === 'COUNT') { - targetLine[j] += 1; - } - else if (method === 'AVERAGE') { - // FIXME: handle other types - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) as number / 1; - } +function isGroupByDimension( + groupByDimInfo: ExternalDimensionDefinition, + targetDimInfo: ResultDimInfoInternal +): boolean { + return groupByDimInfo && targetDimInfo.indexInUpstream === groupByDimInfo.index; +} + +function asc(list: number[]) { + list.sort((a, b) => { + return a - b; + }); +} + +const lineCreator: { + [key in AggregateMethodInternal]: ( + upstream: ExternalSource, + dataIndex: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue + ) => OptionDataValue +} = { + 'SUM'() { + return 0; + }, + 'COUNT'() { + return 1; + }, + 'FIRST'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MIN'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MAX'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'AVERAGE'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + // FIXME: refactor, bad implementation. + const collectLine = groupByDimInfo + ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] + : dimInfo.__collectionResult.outList[0]; + return (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number) + / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + // FIXME: refactor + 'Q1'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.25, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q2'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.5, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q3'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.75, dimInfo, groupByDimInfo, groupByVal); + } +}; + +const lineUpdater: { + [key in AggregateMethodInternal]: ( + val: OptionDataValue, + upstream: ExternalSource, + dataIndex: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue + ) => OptionDataValue +} = { + 'SUM'(val, upstream, dataIndex, dimInfo) { + // FIXME: handle other types + return (val as number) + (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'COUNT'(val) { + return (val as number) + 1; + }, + 'FIRST'(val) { + return val; + }, + 'MIN'(val, upstream, dataIndex, dimInfo) { + return Math.min(val as number, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'MAX'(val, upstream, dataIndex, dimInfo) { + return Math.max(val as number, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'AVERAGE'(val, upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + // FIXME: refactor, bad implementation. + const collectLine = groupByDimInfo + ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] + : dimInfo.__collectionResult.outList[0]; + return (val as number) + + (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number) + / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1'(val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q2'(val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q3'(val, upstream, dataIndex, dimInfo) { + return val; } }; + +function lineCreatorForQ( + percent: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue +) { + const gatheredValues = groupByDimInfo + ? dimInfo.gatheredValuesByGroup[groupByVal + ''] + : dimInfo.gatheredValuesNoGroup; + return quantile(gatheredValues, percent); +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@echarts.apache.org For additional commands, e-mail: commits-h...@echarts.apache.org