From commits-return-1754-archive-asf-public=cust-asf.ponee.io@superset.incubator.apache.org Mon Oct 29 19:08:30 2018 Return-Path: X-Original-To: archive-asf-public@cust-asf.ponee.io Delivered-To: archive-asf-public@cust-asf.ponee.io Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by mx-eu-01.ponee.io (Postfix) with SMTP id 01E02180627 for ; Mon, 29 Oct 2018 19:08:29 +0100 (CET) Received: (qmail 89042 invoked by uid 500); 29 Oct 2018 18:08:29 -0000 Mailing-List: contact commits-help@superset.incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: dev@superset.incubator.apache.org Delivered-To: mailing list commits@superset.incubator.apache.org Received: (qmail 89032 invoked by uid 99); 29 Oct 2018 18:08:29 -0000 Received: from ec2-52-202-80-70.compute-1.amazonaws.com (HELO gitbox.apache.org) (52.202.80.70) by apache.org (qpsmtpd/0.29) with ESMTP; Mon, 29 Oct 2018 18:08:29 +0000 Received: by gitbox.apache.org (ASF Mail Server at gitbox.apache.org, from userid 33) id 7F08481E0D; Mon, 29 Oct 2018 18:08:28 +0000 (UTC) Date: Mon, 29 Oct 2018 18:08:28 +0000 To: "commits@superset.apache.org" Subject: [incubator-superset] branch master updated: [deck] allow an array of dynamic of aggregations (#6198) MIME-Version: 1.0 Content-Type: text/plain; charset=utf-8 Content-Transfer-Encoding: 8bit Message-ID: <154083650821.29210.10360138741406756944@gitbox.apache.org> From: beto@apache.org X-Git-Host: gitbox.apache.org X-Git-Repo: incubator-superset X-Git-Refname: refs/heads/master X-Git-Reftype: branch X-Git-Oldrev: 7db11d95b0931bd5d2bd971a4d39dc0382326e44 X-Git-Newrev: acb44165b406e5a35da9c21c1b6127ed8aca0b7a X-Git-Rev: acb44165b406e5a35da9c21c1b6127ed8aca0b7a X-Git-NotificationType: ref_changed_plus_diff X-Git-Multimail-Version: 1.5.dev Auto-Submitted: auto-generated This is an automated email from the ASF dual-hosted git repository. beto pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-superset.git The following commit(s) were added to refs/heads/master by this push: new acb4416 [deck] allow an array of dynamic of aggregations (#6198) acb4416 is described below commit acb44165b406e5a35da9c21c1b6127ed8aca0b7a Author: Maxime Beauchemin AuthorDate: Mon Oct 29 11:08:22 2018 -0700 [deck] allow an array of dynamic of aggregations (#6198) * [deck] allow an array of dynamic of aggregations * Adding quantiles * lint & tests --- superset/assets/package.json | 1 + .../visualizations/deckgl/layers/common_spec.jsx | 31 +++++++++++++++++ superset/assets/src/explore/controls.jsx | 23 +++++++++++++ superset/assets/src/explore/visTypes.jsx | 1 + superset/assets/src/modules/sandbox.js | 2 ++ .../src/visualizations/deckgl/layers/Grid/Grid.jsx | 8 +++-- .../src/visualizations/deckgl/layers/Hex/Hex.jsx | 9 ++--- .../src/visualizations/deckgl/layers/common.jsx | 39 ++++++++++++++++++++-- superset/assets/yarn.lock | 2 +- 9 files changed, 105 insertions(+), 11 deletions(-) diff --git a/superset/assets/package.json b/superset/assets/package.json index f52096d..8293e96 100644 --- a/superset/assets/package.json +++ b/superset/assets/package.json @@ -62,6 +62,7 @@ "brace": "^0.11.1", "classnames": "^2.2.5", "d3": "^3.5.17", + "d3-array": "^1.2.4", "d3-cloud": "^1.2.1", "d3-color": "^1.2.0", "d3-hierarchy": "^1.1.5", diff --git a/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx new file mode 100644 index 0000000..c11b994 --- /dev/null +++ b/superset/assets/spec/javascripts/visualizations/deckgl/layers/common_spec.jsx @@ -0,0 +1,31 @@ +import { max } from 'd3-array'; +import { getAggFunc } from '../../../../../src/visualizations/deckgl/layers/common'; + +describe('deckgl layers common', () => { + it('getAggFunc', () => { + const arr = [10, 0.5, 55, 128, -10]; + expect(getAggFunc('max')(arr)).toEqual(128); + expect(getAggFunc('min')(arr)).toEqual(-10); + expect(getAggFunc('count')(arr)).toEqual(5); + expect(getAggFunc('median')(arr)).toEqual(10); + expect(getAggFunc('mean')(arr)).toEqual(36.7); + expect(getAggFunc('p1')(arr)).toEqual(-9.58); + expect(getAggFunc('p5')(arr)).toEqual(-7.9); + expect(getAggFunc('p95')(arr)).toEqual(113.39999999999998); + expect(getAggFunc('p99')(arr)).toEqual(125.08); + }); + it('getAggFunc with accessor', () => { + const arr = [{ foo: 1 }, { foo: 2 }, { foo: 3 }]; + const accessor = o => o.foo; + expect(getAggFunc('count')(arr, accessor)).toEqual(3); + expect(max(arr, accessor)).toEqual(3); + expect(getAggFunc('max', accessor)(arr)).toEqual(3); + expect(getAggFunc('min', accessor)(arr)).toEqual(1); + expect(getAggFunc('median', accessor)(arr)).toEqual(2); + expect(getAggFunc('mean', accessor)(arr)).toEqual(2); + expect(getAggFunc('p1', accessor)(arr)).toEqual(1.02); + expect(getAggFunc('p5', accessor)(arr)).toEqual(1.1); + expect(getAggFunc('p95', accessor)(arr)).toEqual(2.9); + expect(getAggFunc('p99', accessor)(arr)).toEqual(2.98); + }); +}); diff --git a/superset/assets/src/explore/controls.jsx b/superset/assets/src/explore/controls.jsx index d3a25a5..ad32c52 100644 --- a/superset/assets/src/explore/controls.jsx +++ b/superset/assets/src/explore/controls.jsx @@ -1317,6 +1317,29 @@ export const controls = { 'computing the total rows and columns'), }, + js_agg_function: { + type: 'SelectControl', + label: t('Dynamic Aggregation Function'), + description: t('The function to use when aggregating points into groups'), + default: 'sum', + clearable: false, + renderTrigger: true, + choices: formatSelectOptions([ + 'sum', + 'min', + 'max', + 'mean', + 'median', + 'count', + 'variance', + 'deviation', + 'p1', + 'p5', + 'p95', + 'p99', + ]), + }, + size_from: { type: 'TextControl', isInt: true, diff --git a/superset/assets/src/explore/visTypes.jsx b/superset/assets/src/explore/visTypes.jsx index 97ac501..30aba88 100644 --- a/superset/assets/src/explore/visTypes.jsx +++ b/superset/assets/src/explore/visTypes.jsx @@ -524,6 +524,7 @@ export const visTypes = { ['mapbox_style', 'viewport'], ['color_picker', 'autozoom'], ['grid_size', 'extruded'], + ['js_agg_function', null], ], }, { diff --git a/superset/assets/src/modules/sandbox.js b/superset/assets/src/modules/sandbox.js index a139013..1d50e15 100644 --- a/superset/assets/src/modules/sandbox.js +++ b/superset/assets/src/modules/sandbox.js @@ -1,6 +1,7 @@ // A safe alternative to JS's eval import vm from 'vm'; import _ from 'underscore'; +import * as d3array from 'd3-array'; import * as colors from './colors'; // Objects exposed here should be treated like a public API @@ -10,6 +11,7 @@ const GLOBAL_CONTEXT = { console, _, colors, + d3array, }; // Copied/modified from https://github.com/hacksparrow/safe-eval/blob/master/index.js diff --git a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx index c9140ac..b2ccd5b 100644 --- a/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/Grid/Grid.jsx @@ -1,5 +1,6 @@ import { GridLayer } from 'deck.gl'; -import { commonLayerProps } from '../common'; + +import { commonLayerProps, getAggFunc } from '../common'; import sandboxedEval from '../../../../modules/sandbox'; import { createDeckGLComponent } from '../../factory'; @@ -17,6 +18,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { data = jsFnMutator(data); } + const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight); return new GridLayer({ id: `grid-layer-${fd.slice_id}`, data, @@ -26,8 +28,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { extruded: fd.extruded, maxColor: [c.r, c.g, c.b, 255 * c.a], outline: false, - getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), - getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getElevationValue: aggFunc, + getColorValue: aggFunc, ...commonLayerProps(fd, setTooltip), }); } diff --git a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx index c3362ed..0e83782 100644 --- a/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/Hex/Hex.jsx @@ -1,5 +1,6 @@ import { HexagonLayer } from 'deck.gl'; -import { commonLayerProps } from '../common'; + +import { commonLayerProps, getAggFunc } from '../common'; import sandboxedEval from '../../../../modules/sandbox'; import { createDeckGLComponent } from '../../factory'; @@ -16,7 +17,7 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { const jsFnMutator = sandboxedEval(fd.js_data_mutator); data = jsFnMutator(data); } - + const aggFunc = getAggFunc(fd.js_agg_function, p => p.weight); return new HexagonLayer({ id: `hex-layer-${fd.slice_id}`, data, @@ -26,8 +27,8 @@ export function getLayer(formData, payload, onAddFilter, setTooltip) { extruded: fd.extruded, maxColor: [c.r, c.g, c.b, 255 * c.a], outline: false, - getElevationValue: points => points.reduce((sum, point) => sum + point.weight, 0), - getColorValue: points => points.reduce((sum, point) => sum + point.weight, 0), + getElevationValue: aggFunc, + getColorValue: aggFunc, ...commonLayerProps(fd, setTooltip), }); } diff --git a/superset/assets/src/visualizations/deckgl/layers/common.jsx b/superset/assets/src/visualizations/deckgl/layers/common.jsx index a512526..36b05ce 100644 --- a/superset/assets/src/visualizations/deckgl/layers/common.jsx +++ b/superset/assets/src/visualizations/deckgl/layers/common.jsx @@ -1,11 +1,12 @@ import React from 'react'; + import { fitBounds } from 'viewport-mercator-project'; -import d3 from 'd3'; +import * as d3array from 'd3-array'; import sandboxedEval from '../../../modules/sandbox'; export function getBounds(points) { - const latExt = d3.extent(points, d => d[1]); - const lngExt = d3.extent(points, d => d[0]); + const latExt = d3array.extent(points, d => d[1]); + const lngExt = d3array.extent(points, d => d[0]); return [ [lngExt[0], latExt[0]], [lngExt[1], latExt[1]], @@ -73,3 +74,35 @@ export function commonLayerProps(formData, setTooltip, onSelect) { pickable: Boolean(onHover), }; } + +const percentiles = { + p1: 0.01, + p5: 0.05, + p95: 0.95, + p99: 0.99, +}; + +/* Get an a stat function that operates on arrays, aligns with control=js_agg_function */ +export function getAggFunc(type = 'sum', accessor = null) { + if (type === 'count') { + return arr => arr.length; + } + let d3func; + if (type in percentiles) { + d3func = (arr, acc) => { + let sortedArr; + if (accessor) { + sortedArr = arr.sort((o1, o2) => d3array.ascending(accessor(o1), accessor(o2))); + } else { + sortedArr = arr.sort(d3array.ascending); + } + return d3array.quantile(sortedArr, percentiles[type], acc); + }; + } else { + d3func = d3array[type]; + } + if (!accessor) { + return arr => d3func(arr); + } + return arr => d3func(arr.map(accessor)); +} diff --git a/superset/assets/yarn.lock b/superset/assets/yarn.lock index 532d9a0..a6ffc95 100644 --- a/superset/assets/yarn.lock +++ b/superset/assets/yarn.lock @@ -3394,7 +3394,7 @@ cypress@^3.0.3: url "0.11.0" yauzl "2.8.0" -d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1: +d3-array@1, d3-array@^1.2.0, d3-array@^1.2.1, d3-array@^1.2.4: version "1.2.4" resolved "https://registry.yarnpkg.com/d3-array/-/d3-array-1.2.4.tgz#635ce4d5eea759f6f605863dbcfc30edc737f71f"