This is an automated email from the ASF dual-hosted git repository. johnbodley pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-superset.git
The following commit(s) were added to refs/heads/master by this push: new 7bfa24d [fix] Re-cleanup legacy filters (#8523) 7bfa24d is described below commit 7bfa24d0daa30a0c80e4c545bb641e6968ca0fe5 Author: John Bodley <4567245+john-bod...@users.noreply.github.com> AuthorDate: Tue Nov 12 12:29:00 2019 -0800 [fix] Re-cleanup legacy filters (#8523) --- superset/examples/birth_names.py | 1 - superset/examples/country_map.py | 1 - superset/examples/deck.py | 21 ------ superset/examples/energy.py | 10 +-- superset/examples/long_lat.py | 1 - superset/examples/multiformat_time_series.py | 1 - superset/examples/random_time_series.py | 1 - superset/examples/unicode_test_data.py | 1 - superset/examples/world_bank.py | 12 ++-- ...127d0d1d_reconvert_legacy_filters_into_adhoc.py | 76 ++++++++++++++++++++++ 10 files changed, 85 insertions(+), 40 deletions(-) diff --git a/superset/examples/birth_names.py b/superset/examples/birth_names.py index ec79dd7..b9142db 100644 --- a/superset/examples/birth_names.py +++ b/superset/examples/birth_names.py @@ -126,7 +126,6 @@ def load_birth_names(only_metadata=False, force=False): "since": "100 years ago", "until": "now", "viz_type": "table", - "where": "", "markup_type": "markdown", } diff --git a/superset/examples/country_map.py b/superset/examples/country_map.py index d18b512..7876da1 100644 --- a/superset/examples/country_map.py +++ b/superset/examples/country_map.py @@ -90,7 +90,6 @@ def load_country_map_data(only_metadata=False, force=False): "granularity_sqla": "", "since": "", "until": "", - "where": "", "viz_type": "country_map", "entity": "DEPT_ID", "metric": { diff --git a/superset/examples/deck.py b/superset/examples/deck.py index 7b80e56..2628700 100644 --- a/superset/examples/deck.py +++ b/superset/examples/deck.py @@ -173,10 +173,8 @@ def load_deck_dash(): "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"}, "color_picker": COLOR_RED, "datasource": "5__table", - "filters": [], "granularity_sqla": None, "groupby": [], - "having": "", "mapbox_style": "mapbox://styles/mapbox/light-v9", "multiplier": 10, "point_radius_fixed": {"type": "metric", "value": "count"}, @@ -195,7 +193,6 @@ def load_deck_dash(): "zoom": 12.729132798697304, }, "viz_type": "deck_scatter", - "where": "", } print("Creating Scatterplot slice") @@ -211,7 +208,6 @@ def load_deck_dash(): slice_data = { "point_unit": "square_m", - "filters": [], "row_limit": 5000, "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"}, "mapbox_style": "mapbox://styles/mapbox/dark-v9", @@ -222,8 +218,6 @@ def load_deck_dash(): "point_radius": "Auto", "color_picker": {"a": 1, "r": 14, "b": 0, "g": 255}, "grid_size": 20, - "where": "", - "having": "", "viewport": { "zoom": 14.161641703941438, "longitude": -122.41827069521386, @@ -249,7 +243,6 @@ def load_deck_dash(): slice_data = { "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"}, - "filters": [], "row_limit": 5000, "mapbox_style": "mapbox://styles/mapbox/streets-v9", "granularity_sqla": None, @@ -261,7 +254,6 @@ def load_deck_dash(): "color_picker": {"a": 1, "r": 14, "b": 0, "g": 255}, "grid_size": 40, "extruded": True, - "having": "", "viewport": { "latitude": 37.789795085160335, "pitch": 54.08961642447763, @@ -269,7 +261,6 @@ def load_deck_dash(): "longitude": -122.40632230075536, "bearing": -2.3984797349335167, }, - "where": "", "point_radius_fixed": {"type": "fix", "value": 2000}, "datasource": "5__table", "time_grain_sqla": None, @@ -288,7 +279,6 @@ def load_deck_dash(): slice_data = { "spatial": {"type": "latlong", "lonCol": "LON", "latCol": "LAT"}, - "filters": [], "row_limit": 5000, "mapbox_style": "mapbox://styles/mapbox/satellite-streets-v9", "granularity_sqla": None, @@ -300,7 +290,6 @@ def load_deck_dash(): "color_picker": {"a": 1, "r": 14, "b": 0, "g": 255}, "grid_size": 120, "extruded": True, - "having": "", "viewport": { "longitude": -122.42066918995666, "bearing": 155.80099696026355, @@ -308,7 +297,6 @@ def load_deck_dash(): "latitude": 37.7942314882596, "pitch": 53.470800300695146, }, - "where": "", "point_radius_fixed": {"type": "fix", "value": 2000}, "datasource": "5__table", "time_grain_sqla": None, @@ -403,9 +391,6 @@ def load_deck_dash(): "js_onclick_href": "", "legend_format": ".1s", "legend_position": "tr", - "where": "", - "having": "", - "filters": [], } print("Creating Polygon slice") @@ -456,9 +441,6 @@ def load_deck_dash(): }, "color_picker": {"r": 0, "g": 122, "b": 135, "a": 1}, "stroke_width": 1, - "where": "", - "having": "", - "filters": [], } print("Creating Arc slice") @@ -508,9 +490,6 @@ def load_deck_dash(): "}));", "js_tooltip": "", "js_onclick_href": "", - "where": "", - "having": "", - "filters": [], } print("Creating Path slice") diff --git a/superset/examples/energy.py b/superset/examples/energy.py index 49c15d8..359828b 100644 --- a/superset/examples/energy.py +++ b/superset/examples/energy.py @@ -77,12 +77,10 @@ def load_energy(only_metadata=False, force=False): "source", "target" ], - "having": "", "metric": "sum__value", "row_limit": "5000", "slice_name": "Energy Sankey", - "viz_type": "sankey", - "where": "" + "viz_type": "sankey" } """ ), @@ -104,13 +102,11 @@ def load_energy(only_metadata=False, force=False): "source", "target" ], - "having": "", "link_length": "200", "metric": "sum__value", "row_limit": "5000", "slice_name": "Force", - "viz_type": "directed_force", - "where": "" + "viz_type": "directed_force" } """ ), @@ -130,13 +126,11 @@ def load_energy(only_metadata=False, force=False): "all_columns_y": "target", "canvas_image_rendering": "pixelated", "collapsed_fieldsets": "", - "having": "", "linear_color_scheme": "blue_white_yellow", "metric": "sum__value", "normalize_across": "heatmap", "slice_name": "Heatmap", "viz_type": "heatmap", - "where": "", "xscale_interval": "1", "yscale_interval": "1" } diff --git a/superset/examples/long_lat.py b/superset/examples/long_lat.py index 40ddf35..4e8178c 100644 --- a/superset/examples/long_lat.py +++ b/superset/examples/long_lat.py @@ -96,7 +96,6 @@ def load_long_lat_data(only_metadata=False, force=False): "granularity_sqla": "day", "since": "2014-01-01", "until": "now", - "where": "", "viz_type": "mapbox", "all_columns_x": "LON", "all_columns_y": "LAT", diff --git a/superset/examples/multiformat_time_series.py b/superset/examples/multiformat_time_series.py index 84ac93e..8c62c46 100644 --- a/superset/examples/multiformat_time_series.py +++ b/superset/examples/multiformat_time_series.py @@ -98,7 +98,6 @@ def load_multiformat_time_series(only_metadata=False, force=False): "row_limit": config["ROW_LIMIT"], "since": "2015", "until": "2016", - "where": "", "viz_type": "cal_heatmap", "domain_granularity": "month", "subdomain_granularity": "day", diff --git a/superset/examples/random_time_series.py b/superset/examples/random_time_series.py index eb1e827..b12c44a 100644 --- a/superset/examples/random_time_series.py +++ b/superset/examples/random_time_series.py @@ -62,7 +62,6 @@ def load_random_time_series_data(only_metadata=False, force=False): "since": "1 year ago", "until": "now", "metric": "count", - "where": "", "viz_type": "cal_heatmap", "domain_granularity": "month", "subdomain_granularity": "day", diff --git a/superset/examples/unicode_test_data.py b/superset/examples/unicode_test_data.py index 14e6918..aed109c 100644 --- a/superset/examples/unicode_test_data.py +++ b/superset/examples/unicode_test_data.py @@ -90,7 +90,6 @@ def load_unicode_test_data(only_metadata=False, force=False): "row_limit": config["ROW_LIMIT"], "since": "100 years ago", "until": "now", - "where": "", "viz_type": "word_cloud", "size_from": "10", "series": "short_phrase", diff --git a/superset/examples/world_bank.py b/superset/examples/world_bank.py index 8c1e11a..2a4b607 100644 --- a/superset/examples/world_bank.py +++ b/superset/examples/world_bank.py @@ -108,7 +108,6 @@ def load_world_bank_health_n_pop(only_metadata=False, force=False): "since": "2014-01-01", "until": "2014-01-02", "time_range": "2014-01-01 : 2014-01-02", - "where": "", "markup_type": "markdown", "country_fieldtype": "cca3", "secondary_metric": { @@ -226,10 +225,12 @@ def load_world_bank_health_n_pop(only_metadata=False, force=False): y="sum__SP_DYN_LE00_IN", size="sum__SP_POP_TOTL", max_bubble_size="50", - filters=[ + adhoc_filters=[ { - "col": "country_code", - "val": [ + "clause": "WHERE", + "expressionType": "SIMPLE", + "filterOptionName": "2745eae5", + "comparator": [ "TCA", "MNP", "DMA", @@ -245,7 +246,8 @@ def load_world_bank_health_n_pop(only_metadata=False, force=False): "AMA", "PLW", ], - "op": "not in", + "operator": "not in", + "subject": "country_code", } ], ), diff --git a/superset/migrations/versions/78ee127d0d1d_reconvert_legacy_filters_into_adhoc.py b/superset/migrations/versions/78ee127d0d1d_reconvert_legacy_filters_into_adhoc.py new file mode 100644 index 0000000..1d0690c --- /dev/null +++ b/superset/migrations/versions/78ee127d0d1d_reconvert_legacy_filters_into_adhoc.py @@ -0,0 +1,76 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""reconvert legacy filters into adhoc + +Revision ID: 78ee127d0d1d +Revises: c2acd2cf3df2 +Create Date: 2019-11-06 15:23:26.497876 + +""" + +# revision identifiers, used by Alembic. +revision = "78ee127d0d1d" +down_revision = "c2acd2cf3df2" + +import copy +import json +import logging +import uuid +from collections import defaultdict + +from alembic import op +from sqlalchemy import Column, Integer, Text +from sqlalchemy.ext.declarative import declarative_base + +from superset import db +from superset.utils.core import ( + convert_legacy_filters_into_adhoc, + split_adhoc_filters_into_base_filters, +) + +Base = declarative_base() + + +class Slice(Base): + __tablename__ = "slices" + + id = Column(Integer, primary_key=True) + params = Column(Text) + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + for slc in session.query(Slice).all(): + if slc.params: + try: + source = json.loads(slc.params) + target = copy.deepcopy(source) + convert_legacy_filters_into_adhoc(target) + + if source != target: + slc.params = json.dumps(target, sort_keys=True) + except Exception as ex: + logging.warn(ex) + + session.commit() + session.close() + + +def downgrade(): + pass