From be70a3979a48ed250f683e8b4d7a7d657526ee56 Mon Sep 17 00:00:00 2001 From: 100pah Date: Thu, 19 Nov 2020 01:25:11 +0800 Subject: [PATCH 1/3] test: migrate test lib myTransform to ts. --- build/build.js | 6 + build/config.js | 24 ++ build/release.js | 2 +- src/data/helper/transform.ts | 2 +- test/custom-shape-morphing2.html | 5 +- test/custom-shape-morphing3.html | 8 +- test/lib/config.js | 1 + test/lib/myTransform/aggregate.js | 174 ---------- test/lib/myTransform/dist/myTransform.js | 219 ++++++++++++ test/lib/myTransform/dist/myTransform.js.map | 1 + test/lib/myTransform/id.js | 88 ----- test/lib/myTransform/src/.eslintrc.yaml | 47 +++ test/lib/myTransform/src/aggregate.ts | 342 +++++++++++++++++++ test/lib/myTransform/src/id.ts | 90 +++++ test/lib/myTransform/src/index.ts | 3 + tsconfig.json | 3 +- 16 files changed, 742 insertions(+), 273 deletions(-) delete mode 100644 test/lib/myTransform/aggregate.js create mode 100644 test/lib/myTransform/dist/myTransform.js create mode 100644 test/lib/myTransform/dist/myTransform.js.map delete mode 100644 test/lib/myTransform/id.js create mode 100644 test/lib/myTransform/src/.eslintrc.yaml create mode 100644 test/lib/myTransform/src/aggregate.ts create mode 100644 test/lib/myTransform/src/id.ts create mode 100644 test/lib/myTransform/src/index.ts diff --git a/build/build.js b/build/build.js index 24da784dc3..42bb096f0b 100755 --- a/build/build.js +++ b/build/build.js @@ -138,6 +138,12 @@ async function run() { ]; await build(cfgs, opt.min, opt.sourcemap); } + else if (opt.type === 'myTransform') { + const cfgs = [ + config.createMyTransform() + ]; + await build(cfgs, opt.min, opt.sourcemap); + } else { const cfg = config.createECharts(opt); await build([cfg], opt.min, opt.sourcemap); diff --git a/build/config.js b/build/config.js index 9cc6e82b7c..53cec294fb 100644 --- a/build/config.js +++ b/build/config.js @@ -212,3 +212,27 @@ exports.createDataTool = function () { } }; }; + +exports.createMyTransform = function () { + let input = nodePath.resolve(ecDir, `test/lib/myTransform/src/index.ts`); + + return { + plugins: preparePlugins({ + clean: true + }, { + include: [ + nodePath.resolve(ecDir, 'test/lib/myTransform/src/**/*.ts') + ] + }), + input: input, + output: { + name: 'myTransform', + format: 'umd', + sourcemap: true, + file: nodePath.resolve(ecDir, `test/lib/myTransform/dist/myTransform.js`) + }, + watch: { + include: [nodePath.resolve(ecDir, 'test/lib/myTransform/src/**')] + } + }; +}; diff --git a/build/release.js b/build/release.js index 2a4e070d3d..d9d6762607 100644 --- a/build/release.js +++ b/build/release.js @@ -42,7 +42,7 @@ function release() { } } - const argsList = ['', 'simple', 'common', 'extension'].map((type) => { + const argsList = ['', 'simple', 'common', 'extension', 'myTransform'].map((type) => { return [ '--type', type, diff --git a/src/data/helper/transform.ts b/src/data/helper/transform.ts index 287b55dafa..349e9857d8 100644 --- a/src/data/helper/transform.ts +++ b/src/data/helper/transform.ts @@ -83,7 +83,7 @@ export interface ExternalDataTransformResultItem { */ dimensions?: DimensionDefinitionLoose[]; } -interface ExternalDimensionDefinition extends Partial { +export interface ExternalDimensionDefinition extends Partial { // Mandatory index: DimensionIndex; } diff --git a/test/custom-shape-morphing2.html b/test/custom-shape-morphing2.html index ac1273861c..f55d36774a 100644 --- a/test/custom-shape-morphing2.html +++ b/test/custom-shape-morphing2.html @@ -27,7 +27,6 @@ - @@ -76,9 +75,9 @@ - - @@ -42,11 +40,11 @@ + + + + + + + + + + + + +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/lib/myTransform/dist/myTransform.js b/test/lib/myTransform/dist/myTransform.js index 9edeb7fd79..a2f97b57d1 100644 --- a/test/lib/myTransform/dist/myTransform.js +++ b/test/lib/myTransform/dist/myTransform.js @@ -26,12 +26,56 @@ } }; var arrayProto = Array.prototype; + var nativeForEach = arrayProto.forEach; var nativeSlice = arrayProto.slice; + var nativeMap = arrayProto.map; var ctorFunction = function () {}.constructor; var protoFunction = ctorFunction ? ctorFunction.prototype : null; + function each(arr, cb, context) { + if (!(arr && cb)) { + return; + } + + if (arr.forEach && arr.forEach === nativeForEach) { + arr.forEach(cb, context); + } else if (arr.length === +arr.length) { + for (var i = 0, len = arr.length; i < len; i++) { + cb.call(context, arr[i], i, arr); + } + } else { + for (var key in arr) { + if (arr.hasOwnProperty(key)) { + cb.call(context, arr[key], key, arr); + } + } + } + } + + function map(arr, cb, context) { + if (!arr) { + return []; + } + + if (!cb) { + return slice(arr); + } + + if (arr.map && arr.map === nativeMap) { + return arr.map(cb, context); + } else { + var result = []; + + for (var i = 0, len = arr.length; i < len; i++) { + result.push(cb.call(context, arr[i], i, arr)); + } + + return result; + } + } + function bindPolyfill(func, context) { var args = []; @@ -50,6 +94,16 @@ return typeof value === 'function'; } + function slice(arr) { + var args = []; + + for (var _i = 1; _i < arguments.length; _i++) { + args[_i - 1] = arguments[_i]; + } + + return nativeSlice.apply(arr, args); + } + function assert(condition, message) { if (!condition) { throw new Error(message); @@ -60,6 +114,14 @@ return own.hasOwnProperty(prop); } + function quantile(ascArr, p) { + var H = (ascArr.length - 1) * p + 1; + var h = Math.floor(H); + var v = +ascArr[h - 1]; + var e = H - h; + return e ? v + e * (ascArr[h] - v) : v; + } + var METHOD_INTERNAL = { 'SUM': true, 'COUNT': true, @@ -71,49 +133,135 @@ 'MIN': true, 'MAX': true }; + var METHOD_NEEDS_COLLECT = { + AVERAGE: ['COUNT'] + }; + var METHOD_NEEDS_GATHER_VALUES = { + Q1: true, + Q2: true, + Q3: true + }; var METHOD_ALIAS = { MEDIAN: 'Q2' }; + + var ResultDimInfoInternal = function () { + function ResultDimInfoInternal(index, indexInUpstream, method, name, needGatherValues) { + this.collectionInfoList = []; + this.gatheredValuesByGroup = {}; + this.gatheredValuesNoGroup = []; + this.needGatherValues = false; + this._collectionInfoMap = {}; + this.method = method; + this.name = name; + this.index = index; + this.indexInUpstream = indexInUpstream; + this.needGatherValues = needGatherValues; + } + + ResultDimInfoInternal.prototype.addCollectionInfo = function (item) { + this._collectionInfoMap[item.method] = this.collectionInfoList.length; + this.collectionInfoList.push(item); + }; + + ResultDimInfoInternal.prototype.getCollectionInfo = function (method) { + return this.collectionInfoList[this._collectionInfoMap[method]]; + }; + + ResultDimInfoInternal.prototype.gatherValue = function (groupByDimInfo, groupVal, value) { + value = +value; + + if (groupByDimInfo) { + if (groupVal != null) { + var groupValStr = groupVal + ''; + var values = this.gatheredValuesByGroup[groupValStr] || (this.gatheredValuesByGroup[groupValStr] = []); + values.push(value); + } + } else { + this.gatheredValuesNoGroup.push(value); + } + }; + + return ResultDimInfoInternal; + }(); + var transform$1 = { type: 'myTransform:aggregate', transform: function (params) { var upstream = params.upstream; var config = params.config; - var dimWrap = prepareDimensions(config, upstream); - var resultDimInfoList = dimWrap.resultDimInfoList; - var resultDimensions = dimWrap.resultDimensions; var groupByDimInfo = prepareGroupByDimInfo(config, upstream); - var finalResult = travel(groupByDimInfo, upstream, resultDimInfoList, createResultLine, aggregateResultLine); + + var _a = prepareDimensions(config, upstream, groupByDimInfo), + finalResultDimInfoList = _a.finalResultDimInfoList, + collectionDimInfoList = _a.collectionDimInfoList; + + var collectionResult; + + if (collectionDimInfoList.length) { + collectionResult = travel(groupByDimInfo, upstream, collectionDimInfoList, createCollectionResultLine, updateCollectionResultLine); + } + + each(collectionDimInfoList, function (dimInfo) { + dimInfo.__collectionResult = collectionResult; + asc(dimInfo.gatheredValuesNoGroup); + each(dimInfo.gatheredValuesByGroup, function (values) { + asc(values); + }); + }); + var finalResult = travel(groupByDimInfo, upstream, finalResultDimInfoList, createFinalResultLine, updateFinalResultLine); return { - dimensions: resultDimensions, + dimensions: map(finalResultDimInfoList, function (item) { + return item.name; + }), data: finalResult.outList }; } }; - function prepareDimensions(config, upstream) { + function prepareDimensions(config, upstream, groupByDimInfo) { var resultDimensionsConfig = config.resultDimensions; - var resultDimInfoList = []; - var resultDimensions = []; + var finalResultDimInfoList = []; + var collectionDimInfoList = []; + var gIndexInLine = 0; for (var i = 0; i < resultDimensionsConfig.length; i++) { var resultDimInfoConfig = resultDimensionsConfig[i]; - var resultDimInfo = upstream.getDimensionInfo(resultDimInfoConfig.from); - assert(resultDimInfo, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); - resultDimInfo.method = normalizeMethod(resultDimInfoConfig.method); - assert(resultDimInfo.method, 'method is required'); - resultDimInfoList.push(resultDimInfo); + var dimInfoInUpstream = upstream.getDimensionInfo(resultDimInfoConfig.from); + assert(dimInfoInUpstream, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); + var rawMethod = resultDimInfoConfig.method; + assert(groupByDimInfo.index !== dimInfoInUpstream.index || rawMethod == null, "Dimension " + dimInfoInUpstream.name + " is the \"groupBy\" dimension, must not have any \"method\"."); + var method = normalizeMethod(rawMethod); + assert(method, 'method is required'); + var name_1 = resultDimInfoConfig.name != null ? resultDimInfoConfig.name : dimInfoInUpstream.name; + var finalResultDimInfo = new ResultDimInfoInternal(finalResultDimInfoList.length, dimInfoInUpstream.index, method, name_1, hasOwn(METHOD_NEEDS_GATHER_VALUES, method)); + finalResultDimInfoList.push(finalResultDimInfo); + var needCollect = false; + + if (hasOwn(METHOD_NEEDS_COLLECT, method)) { + needCollect = true; + var collectionTargetMethods = METHOD_NEEDS_COLLECT[method]; + + for (var j = 0; j < collectionTargetMethods.length; j++) { + finalResultDimInfo.addCollectionInfo({ + method: collectionTargetMethods[j], + indexInLine: gIndexInLine++ + }); + } + } - if (resultDimInfoConfig.name != null) { - resultDimInfo.name = resultDimInfoConfig.name; + if (hasOwn(METHOD_NEEDS_GATHER_VALUES, method)) { + needCollect = true; } - resultDimensions.push(resultDimInfo.name); + if (needCollect) { + collectionDimInfoList.push(finalResultDimInfo); + } } return { - resultDimensions: resultDimensions, - resultDimInfoList: resultDimInfoList + collectionDimInfoList: collectionDimInfoList, + finalResultDimInfoList: finalResultDimInfoList }; } @@ -129,12 +277,12 @@ return groupByDimInfo; } - function travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doAggregate) { + function travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doUpdate) { var outList = []; - var groupMap; + var mapByGroup; if (groupByDimInfo) { - groupMap = {}; + mapByGroup = {}; for (var dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { var groupByVal = upstream.retrieveValue(dataIndex, groupByDimInfo.index); @@ -145,26 +293,26 @@ var groupByValStr = groupByVal + ''; - if (!hasOwn(groupMap, groupByValStr)) { + if (!hasOwn(mapByGroup, groupByValStr)) { var newLine = doCreate(upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal); outList.push(newLine); - groupMap[groupByValStr] = newLine; + mapByGroup[groupByValStr] = newLine; } else { - var targetLine = groupMap[groupByValStr]; - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo); + var targetLine = mapByGroup[groupByValStr]; + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo, groupByVal); } } } else { var targetLine = doCreate(upstream, 0, resultDimInfoList); outList.push(targetLine); - for (var dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList); + for (var dataIndex = 1, len = upstream.count(); dataIndex < len; dataIndex++) { + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList); } } return { - groupMap: groupMap, + mapByGroup: mapByGroup, outList: outList }; } @@ -180,37 +328,146 @@ return methodInternal; } - var createResultLine = function (upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal) { + var createCollectionResultLine = function (upstream, dataIndex, collectionDimInfoList, groupByDimInfo, groupByVal) { var newLine = []; - for (var j = 0; j < resultDimInfoList.length; j++) { - var resultDimInfo = resultDimInfoList[j]; - var method = resultDimInfo.method; - newLine[j] = groupByDimInfo && resultDimInfo.index === groupByDimInfo.index ? groupByVal : method === 'SUM' || method === 'COUNT' ? 0 : upstream.retrieveValue(dataIndex, resultDimInfo.index); + for (var i = 0; i < collectionDimInfoList.length; i++) { + var dimInfo = collectionDimInfoList[i]; + var collectionInfoList = dimInfo.collectionInfoList; + + for (var j = 0; j < collectionInfoList.length; j++) { + var collectionInfo = collectionInfoList[j]; + newLine[collectionInfo.indexInLine] = +lineCreator[collectionInfo.method](upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + + if (dimInfo.needGatherValues) { + var val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } } return newLine; }; - var aggregateResultLine = function (upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo) { - for (var j = 0; j < resultDimInfoList.length; j++) { - var resultDimInfo = resultDimInfoList[j]; - var method = resultDimInfo.method; + var updateCollectionResultLine = function (upstream, dataIndex, targetLine, collectionDimInfoList, groupByDimInfo, groupByVal) { + for (var i = 0; i < collectionDimInfoList.length; i++) { + var dimInfo = collectionDimInfoList[i]; + var collectionInfoList = dimInfo.collectionInfoList; - if (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) { - continue; + for (var j = 0; j < collectionInfoList.length; j++) { + var collectionInfo = collectionInfoList[j]; + var indexInLine = collectionInfo.indexInLine; + targetLine[indexInLine] = +lineUpdater[collectionInfo.method](targetLine[indexInLine], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); } - if (method === 'SUM') { - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index); - } else if (method === 'COUNT') { - targetLine[j] += 1; - } else if (method === 'AVERAGE') { - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) / 1; + if (dimInfo.needGatherValues) { + var val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); } } }; + var createFinalResultLine = function (upstream, dataIndex, finalResultDimInfoList, groupByDimInfo, groupByVal) { + var newLine = []; + + for (var i = 0; i < finalResultDimInfoList.length; i++) { + var dimInfo = finalResultDimInfoList[i]; + var method = dimInfo.method; + newLine[i] = isGroupByDimension(groupByDimInfo, dimInfo) ? groupByVal : lineCreator[method](upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + + return newLine; + }; + + var updateFinalResultLine = function (upstream, dataIndex, targetLine, finalResultDimInfoList, groupByDimInfo, groupByVal) { + for (var i = 0; i < finalResultDimInfoList.length; i++) { + var dimInfo = finalResultDimInfoList[i]; + + if (isGroupByDimension(groupByDimInfo, dimInfo)) { + continue; + } + + var method = dimInfo.method; + targetLine[i] = lineUpdater[method](targetLine[i], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal); + } + }; + + function isGroupByDimension(groupByDimInfo, targetDimInfo) { + return groupByDimInfo && targetDimInfo.indexInUpstream === groupByDimInfo.index; + } + + function asc(list) { + list.sort(function (a, b) { + return a - b; + }); + } + + var lineCreator = { + 'SUM': function () { + return 0; + }, + 'COUNT': function () { + return 1; + }, + 'FIRST': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MIN': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MAX': function (upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'AVERAGE': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + var collectLine = groupByDimInfo ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] : dimInfo.__collectionResult.outList[0]; + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.25, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q2': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.5, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q3': function (upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.75, dimInfo, groupByDimInfo, groupByVal); + } + }; + var lineUpdater = { + 'SUM': function (val, upstream, dataIndex, dimInfo) { + return val + upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'COUNT': function (val) { + return val + 1; + }, + 'FIRST': function (val) { + return val; + }, + 'MIN': function (val, upstream, dataIndex, dimInfo) { + return Math.min(val, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream)); + }, + 'MAX': function (val, upstream, dataIndex, dimInfo) { + return Math.max(val, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream)); + }, + 'AVERAGE': function (val, upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + var collectLine = groupByDimInfo ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] : dimInfo.__collectionResult.outList[0]; + return val + upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1': function (val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q2': function (val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q3': function (val, upstream, dataIndex, dimInfo) { + return val; + } + }; + + function lineCreatorForQ(percent, dimInfo, groupByDimInfo, groupByVal) { + var gatheredValues = groupByDimInfo ? dimInfo.gatheredValuesByGroup[groupByVal + ''] : dimInfo.gatheredValuesNoGroup; + return quantile(gatheredValues, percent); + } + exports.aggregate = transform$1; exports.id = transform; Object.defineProperty(exports, '__esModule', { diff --git a/test/lib/myTransform/dist/myTransform.js.map b/test/lib/myTransform/dist/myTransform.js.map index 4b03361adc..7628c230f9 100644 --- a/test/lib/myTransform/dist/myTransform.js.map +++ b/test/lib/myTransform/dist/myTransform.js.map @@ -1 +1 @@ -{"version":3,"file":"myTransform.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"} \ No newline at end of file +{"version":3,"file":"myTransform.js","sources":[],"sourcesContent":[],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;"} \ No newline at end of file diff --git a/test/lib/myTransform/src/aggregate.ts b/test/lib/myTransform/src/aggregate.ts index 68a4d04a4e..4b6e02a6ee 100644 --- a/test/lib/myTransform/src/aggregate.ts +++ b/test/lib/myTransform/src/aggregate.ts @@ -21,9 +21,10 @@ import { DataTransformOption, ExternalDataTransform, ExternalSource, ExternalDimensionDefinition } from '../../../../src/data/helper/transform'; import { - DimensionName, DimensionLoose, DimensionDefinitionLoose, OptionDataValue + DimensionName, DimensionLoose, OptionDataValue } from '../../../../src/util/types'; -import { hasOwn, assert } from 'zrender/src/core/util'; +import { hasOwn, assert, map, each } from 'zrender/src/core/util'; +import { quantile } from '../../../../src/util/number'; /** @@ -48,7 +49,7 @@ import { hasOwn, assert } from 'zrender/src/core/util'; * { from: 'aa', method: 'sum' }, * { from: 'bb', method: 'count' }, * { from: 'cc' }, // method by default: use the first value. - * { from: 'dd', method: 'Q1', boundIQR: 1 }, + * { from: 'dd', method: 'Q1' }, * { from: 'tag' } * ], * groupBy: 'tag' @@ -73,13 +74,6 @@ import { hasOwn, assert } from 'zrender/src/core/util'; * 'Q2' or 'median' * 'min' * 'max' - * - * Current supported other arguments: - * boundIQR: - * Data less than min bound is outlier. - * default 1.5, means Q1 - 1.5 * (Q3 - Q1). - * If 'none'/0 passed, min bound will not be used. - * */ export interface AggregateTransformOption extends DataTransformOption { @@ -113,7 +107,9 @@ const METHOD_INTERNAL = { 'MAX': true } as const; const METHOD_NEEDS_COLLECT = { - AVERAGE: true, + AVERAGE: ['COUNT'] +} as const; +const METHOD_NEEDS_GATHER_VALUES = { Q1: true, Q2: true, Q3: true @@ -127,23 +123,86 @@ type AggregateMethodLoose = | 'sum' | 'count' | 'first' | 'average' | 'Q1' | 'Q2' | 'Q3' | 'median' | 'min' | 'max'; type AggregateMethodInternal = keyof typeof METHOD_INTERNAL; -interface ResultDimInfoInternal extends ExternalDimensionDefinition { - method: AggregateMethodInternal; + +class ResultDimInfoInternal { + + readonly method: AggregateMethodInternal; + readonly name: DimensionName; + readonly index: number; + readonly indexInUpstream: number; + + readonly collectionInfoList = [] as { + method: AggregateMethodInternal; + indexInLine: number; + }[]; + + // FIXME: refactor + readonly gatheredValuesByGroup: { [groupVal: string]: number[] } = {}; + readonly gatheredValuesNoGroup = [] as number[]; + readonly needGatherValues: boolean = false; + + __collectionResult: TravelResult; + + private _collectionInfoMap = {} as { + // number is the index of `list` + [method in AggregateMethodInternal]: number + }; + + constructor( + index: number, + indexInUpstream: number, + method: AggregateMethodInternal, + name: DimensionName, + needGatherValues: boolean + ) { + this.method = method; + this.name = name; + this.index = index; + this.indexInUpstream = indexInUpstream; + this.needGatherValues = needGatherValues; + } + + addCollectionInfo(item: ResultDimInfoInternal['collectionInfoList'][number]) { + this._collectionInfoMap[item.method] = this.collectionInfoList.length; + this.collectionInfoList.push(item); + } + + getCollectionInfo(method: AggregateMethodInternal) { + return this.collectionInfoList[this._collectionInfoMap[method]]; + } + + // FIXME: temp implementation. Need refactor. + gatherValue(groupByDimInfo: ExternalDimensionDefinition, groupVal: OptionDataValue, value: OptionDataValue) { + // FIXME: convert to number compulsorily temporarily. + value = +value; + if (groupByDimInfo) { + if (groupVal != null) { + const groupValStr = groupVal + ''; + const values = this.gatheredValuesByGroup[groupValStr] + || (this.gatheredValuesByGroup[groupValStr] = []); + values.push(value); + } + } + else { + this.gatheredValuesNoGroup.push(value); + } + } } -type CreateInTravel = ( +type CreateInTravel = ( upstream: ExternalSource, dataIndex: number, - resultDimInfoList: ResultDimInfoInternal[], + dimInfoList: ResultDimInfoInternal[], groupByDimInfo?: ExternalDimensionDefinition, groupByVal?: OptionDataValue -) => void; -type AggregateInTravel = ( +) => LINE; +type UpdateInTravel = ( upstream: ExternalSource, dataIndex: number, - targetLine: unknown, - resultDimInfoList: ResultDimInfoInternal[], - groupByDimInfo?: ExternalDimensionDefinition + targetLine: LINE, + dimInfoList: ResultDimInfoInternal[], + groupByDimInfo?: ExternalDimensionDefinition, + groupByVal?: OptionDataValue ) => void; export const transform: ExternalDataTransform = { @@ -154,26 +213,43 @@ export const transform: ExternalDataTransform = { const upstream = params.upstream; const config = params.config; - const dimWrap = prepareDimensions(config, upstream); - const resultDimInfoList = dimWrap.resultDimInfoList; - const resultDimensions = dimWrap.resultDimensions; - const groupByDimInfo = prepareGroupByDimInfo(config, upstream); + const { finalResultDimInfoList, collectionDimInfoList } = prepareDimensions( + config, upstream, groupByDimInfo + ); // Collect - // const collectResult; - // const dimInfoListForCollect = makeDimInfoListForCollect(resultDimInfoList); - // if (dimInfoListForCollect.length) { - // collectResult = travel(groupByDimInfo, upstream, resultDimInfoList, doCreate, doAggregate); - // } + let collectionResult: TravelResult; + if (collectionDimInfoList.length) { + collectionResult = travel( + groupByDimInfo, + upstream, + collectionDimInfoList, + createCollectionResultLine, + updateCollectionResultLine + ); + } + + each(collectionDimInfoList, dimInfo => { + dimInfo.__collectionResult = collectionResult; + // FIXME: just for Q1, Q2, Q3: need asc. + asc(dimInfo.gatheredValuesNoGroup); + each(dimInfo.gatheredValuesByGroup, values => { + asc(values); + }); + }); // Calculate const finalResult = travel( - groupByDimInfo, upstream, resultDimInfoList, createResultLine, aggregateResultLine + groupByDimInfo, + upstream, + finalResultDimInfoList, + createFinalResultLine, + updateFinalResultLine ); return { - dimensions: resultDimensions, + dimensions: map(finalResultDimInfoList, item => item.name), data: finalResult.outList }; } @@ -181,34 +257,65 @@ export const transform: ExternalDataTransform = { function prepareDimensions( config: AggregateTransformOption['config'], - upstream: ExternalSource + upstream: ExternalSource, + groupByDimInfo: ExternalDimensionDefinition ): { - resultDimInfoList: ResultDimInfoInternal[]; - resultDimensions: DimensionDefinitionLoose[]; + finalResultDimInfoList: ResultDimInfoInternal[]; + collectionDimInfoList: ResultDimInfoInternal[]; } { const resultDimensionsConfig = config.resultDimensions; - const resultDimInfoList: ResultDimInfoInternal[] = []; - const resultDimensions: DimensionDefinitionLoose[] = []; + const finalResultDimInfoList: ResultDimInfoInternal[] = []; + const collectionDimInfoList: ResultDimInfoInternal[] = []; + let gIndexInLine = 0; for (let i = 0; i < resultDimensionsConfig.length; i++) { const resultDimInfoConfig = resultDimensionsConfig[i]; - const resultDimInfo = upstream.getDimensionInfo(resultDimInfoConfig.from) as ResultDimInfoInternal; - assert(resultDimInfo, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); + const dimInfoInUpstream = upstream.getDimensionInfo(resultDimInfoConfig.from); + assert(dimInfoInUpstream, 'Can not find dimension by `from`: ' + resultDimInfoConfig.from); - resultDimInfo.method = normalizeMethod(resultDimInfoConfig.method); - assert(resultDimInfo.method, 'method is required'); + const rawMethod = resultDimInfoConfig.method; - resultDimInfoList.push(resultDimInfo); + assert( + groupByDimInfo.index !== dimInfoInUpstream.index || rawMethod == null, + `Dimension ${dimInfoInUpstream.name} is the "groupBy" dimension, must not have any "method".` + ); - if (resultDimInfoConfig.name != null) { - resultDimInfo.name = resultDimInfoConfig.name; - } + const method = normalizeMethod(rawMethod); + assert(method, 'method is required'); + + const name = resultDimInfoConfig.name != null ? resultDimInfoConfig.name : dimInfoInUpstream.name; - resultDimensions.push(resultDimInfo.name); + const finalResultDimInfo = new ResultDimInfoInternal( + finalResultDimInfoList.length, + dimInfoInUpstream.index, + method, + name, + hasOwn(METHOD_NEEDS_GATHER_VALUES, method) + ); + finalResultDimInfoList.push(finalResultDimInfo); + + // For collection. + let needCollect = false; + if (hasOwn(METHOD_NEEDS_COLLECT, method)) { + needCollect = true; + const collectionTargetMethods = METHOD_NEEDS_COLLECT[method as keyof typeof METHOD_NEEDS_COLLECT]; + for (let j = 0; j < collectionTargetMethods.length; j++) { + finalResultDimInfo.addCollectionInfo({ + method: collectionTargetMethods[j], + indexInLine: gIndexInLine++ + }); + } + } + if (hasOwn(METHOD_NEEDS_GATHER_VALUES, method)) { + needCollect = true; + } + if (needCollect) { + collectionDimInfoList.push(finalResultDimInfo); + } } - return { resultDimensions, resultDimInfoList }; + return { collectionDimInfoList, finalResultDimInfoList }; } function prepareGroupByDimInfo( @@ -224,21 +331,23 @@ function prepareGroupByDimInfo( return groupByDimInfo; } -function travel( +interface TravelResult { + mapByGroup: { [groupVal: string]: LINE }; + outList: LINE[]; +} + +function travel( groupByDimInfo: ExternalDimensionDefinition, upstream: ExternalSource, resultDimInfoList: ResultDimInfoInternal[], - doCreate: CreateInTravel, - doAggregate: AggregateInTravel -): { - groupMap: { [groupVal in string]: unknown }; - outList: unknown[]; -} { - const outList: unknown[] = []; - let groupMap: { [groupVal in string]: unknown }; + doCreate: CreateInTravel, + doUpdate: UpdateInTravel +): TravelResult { + const outList: TravelResult['outList'] = []; + let mapByGroup: TravelResult['mapByGroup']; if (groupByDimInfo) { - groupMap = {}; + mapByGroup = {}; for (let dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { const groupByVal = upstream.retrieveValue(dataIndex, groupByDimInfo.index); @@ -250,43 +359,28 @@ function travel( const groupByValStr = groupByVal + ''; - if (!hasOwn(groupMap, groupByValStr)) { + if (!hasOwn(mapByGroup, groupByValStr)) { const newLine = doCreate(upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal); outList.push(newLine); - groupMap[groupByValStr] = newLine; + mapByGroup[groupByValStr] = newLine; } else { - const targetLine = groupMap[groupByValStr]; - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo); + const targetLine = mapByGroup[groupByValStr]; + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList, groupByDimInfo, groupByVal); } } } else { const targetLine = doCreate(upstream, 0, resultDimInfoList); outList.push(targetLine); - for (let dataIndex = 0, len = upstream.count(); dataIndex < len; dataIndex++) { - doAggregate(upstream, dataIndex, targetLine, resultDimInfoList); + for (let dataIndex = 1, len = upstream.count(); dataIndex < len; dataIndex++) { + doUpdate(upstream, dataIndex, targetLine, resultDimInfoList); } } - return { - groupMap: groupMap, - outList: outList - }; + return { mapByGroup, outList }; } -// function makeDimInfoListForCollect(resultDimInfoList) { -// const dimInfoListForCollect = []; -// for (const j = 0; j < resultDimInfoList.length; j++) { -// const resultDimInfo = resultDimInfoList[j]; -// const method = resultDimInfo.method; -// if (hasOwn(METHOD_NEEDS_COLLECT, method)) { -// dimInfoListForCollect.push(resultDimInfo); -// } -// } -// return dimInfoListForCollect; -// } - function normalizeMethod(method: AggregateMethodLoose): AggregateMethodInternal { if (method == null) { return 'FIRST'; @@ -299,44 +393,201 @@ function normalizeMethod(method: AggregateMethodLoose): AggregateMethodInternal return methodInternal; } -const createResultLine: CreateInTravel = ( - upstream, dataIndex, resultDimInfoList, groupByDimInfo, groupByVal + + +type CollectionResultLine = number[]; + +const createCollectionResultLine: CreateInTravel = ( + upstream, dataIndex, collectionDimInfoList, groupByDimInfo, groupByVal +) => { + const newLine = [] as number[]; + for (let i = 0; i < collectionDimInfoList.length; i++) { + const dimInfo = collectionDimInfoList[i]; + const collectionInfoList = dimInfo.collectionInfoList; + for (let j = 0; j < collectionInfoList.length; j++) { + const collectionInfo = collectionInfoList[j]; + // FIXME: convert to number compulsorily temporarily. + newLine[collectionInfo.indexInLine] = +lineCreator[collectionInfo.method]( + upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } + // FIXME: refactor + if (dimInfo.needGatherValues) { + const val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } + } + return newLine; +}; + +const updateCollectionResultLine: UpdateInTravel = ( + upstream, dataIndex, targetLine: number[], collectionDimInfoList, groupByDimInfo, groupByVal +) => { + for (let i = 0; i < collectionDimInfoList.length; i++) { + const dimInfo = collectionDimInfoList[i]; + const collectionInfoList = dimInfo.collectionInfoList; + for (let j = 0; j < collectionInfoList.length; j++) { + const collectionInfo = collectionInfoList[j]; + const indexInLine = collectionInfo.indexInLine; + // FIXME: convert to number compulsorily temporarily. + targetLine[indexInLine] = +lineUpdater[collectionInfo.method]( + targetLine[indexInLine], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } + // FIXME: refactor + if (dimInfo.needGatherValues) { + const val = upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + dimInfo.gatherValue(groupByDimInfo, groupByVal, val); + } + } +}; + + + +type FinalResultLine = OptionDataValue[]; + +const createFinalResultLine: CreateInTravel = ( + upstream, dataIndex, finalResultDimInfoList, groupByDimInfo, groupByVal ) => { const newLine = []; - for (let j = 0; j < resultDimInfoList.length; j++) { - const resultDimInfo = resultDimInfoList[j]; - const method = resultDimInfo.method; - newLine[j] = (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) + for (let i = 0; i < finalResultDimInfoList.length; i++) { + const dimInfo = finalResultDimInfoList[i]; + const method = dimInfo.method; + newLine[i] = isGroupByDimension(groupByDimInfo, dimInfo) ? groupByVal - : (method === 'SUM' || method === 'COUNT') - ? 0 - // By default, method: 'first' - : upstream.retrieveValue(dataIndex, resultDimInfo.index); + : lineCreator[method]( + upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); } return newLine; }; -const aggregateResultLine: AggregateInTravel = ( - upstream, dataIndex, targetLine: number[], resultDimInfoList, groupByDimInfo +const updateFinalResultLine: UpdateInTravel = ( + upstream, dataIndex, targetLine, finalResultDimInfoList, groupByDimInfo, groupByVal ) => { - for (let j = 0; j < resultDimInfoList.length; j++) { - const resultDimInfo = resultDimInfoList[j]; - const method = resultDimInfo.method; - - if (groupByDimInfo && resultDimInfo.index === groupByDimInfo.index) { + for (let i = 0; i < finalResultDimInfoList.length; i++) { + const dimInfo = finalResultDimInfoList[i]; + if (isGroupByDimension(groupByDimInfo, dimInfo)) { continue; } + const method = dimInfo.method; + targetLine[i] = lineUpdater[method]( + targetLine[i], upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal + ); + } +}; - if (method === 'SUM') { - // FIXME: handle other types - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) as number; - } - else if (method === 'COUNT') { - targetLine[j] += 1; - } - else if (method === 'AVERAGE') { - // FIXME: handle other types - targetLine[j] += upstream.retrieveValue(dataIndex, resultDimInfo.index) as number / 1; - } +function isGroupByDimension( + groupByDimInfo: ExternalDimensionDefinition, + targetDimInfo: ResultDimInfoInternal +): boolean { + return groupByDimInfo && targetDimInfo.indexInUpstream === groupByDimInfo.index; +} + +function asc(list: number[]) { + list.sort((a, b) => { + return a - b; + }); +} + +const lineCreator: { + [key in AggregateMethodInternal]: ( + upstream: ExternalSource, + dataIndex: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue + ) => OptionDataValue +} = { + 'SUM'() { + return 0; + }, + 'COUNT'() { + return 1; + }, + 'FIRST'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MIN'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'MAX'(upstream, dataIndex, dimInfo) { + return upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream); + }, + 'AVERAGE'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + // FIXME: refactor, bad implementation. + const collectLine = groupByDimInfo + ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] + : dimInfo.__collectionResult.outList[0]; + return (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number) + / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + // FIXME: refactor + 'Q1'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.25, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q2'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.5, dimInfo, groupByDimInfo, groupByVal); + }, + 'Q3'(upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + return lineCreatorForQ(0.75, dimInfo, groupByDimInfo, groupByVal); + } +}; + +const lineUpdater: { + [key in AggregateMethodInternal]: ( + val: OptionDataValue, + upstream: ExternalSource, + dataIndex: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue + ) => OptionDataValue +} = { + 'SUM'(val, upstream, dataIndex, dimInfo) { + // FIXME: handle other types + return (val as number) + (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'COUNT'(val) { + return (val as number) + 1; + }, + 'FIRST'(val) { + return val; + }, + 'MIN'(val, upstream, dataIndex, dimInfo) { + return Math.min(val as number, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'MAX'(val, upstream, dataIndex, dimInfo) { + return Math.max(val as number, upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number); + }, + 'AVERAGE'(val, upstream, dataIndex, dimInfo, groupByDimInfo, groupByVal) { + // FIXME: refactor, bad implementation. + const collectLine = groupByDimInfo + ? dimInfo.__collectionResult.mapByGroup[groupByVal + ''] + : dimInfo.__collectionResult.outList[0]; + return (val as number) + + (upstream.retrieveValue(dataIndex, dimInfo.indexInUpstream) as number) + / collectLine[dimInfo.getCollectionInfo('COUNT').indexInLine]; + }, + 'Q1'(val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q2'(val, upstream, dataIndex, dimInfo) { + return val; + }, + 'Q3'(val, upstream, dataIndex, dimInfo) { + return val; } }; + +function lineCreatorForQ( + percent: number, + dimInfo: ResultDimInfoInternal, + groupByDimInfo: ExternalDimensionDefinition, + groupByVal: OptionDataValue +) { + const gatheredValues = groupByDimInfo + ? dimInfo.gatheredValuesByGroup[groupByVal + ''] + : dimInfo.gatheredValuesNoGroup; + return quantile(gatheredValues, percent); +} From c3bdb23cf9a37e276d28d442e7b0bdb5c93fbdb4 Mon Sep 17 00:00:00 2001 From: 100pah Date: Thu, 19 Nov 2020 20:23:38 +0800 Subject: [PATCH 3/3] fix: do not build myTransform when release (asf artefacts has no test lib) --- build/release.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/release.js b/build/release.js index d9d6762607..2a4e070d3d 100644 --- a/build/release.js +++ b/build/release.js @@ -42,7 +42,7 @@ function release() { } } - const argsList = ['', 'simple', 'common', 'extension', 'myTransform'].map((type) => { + const argsList = ['', 'simple', 'common', 'extension'].map((type) => { return [ '--type', type,