From 3bbb7c4a0dc026801e94eb112c13c1dcec475a83 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 17:08:27 -0400 Subject: [PATCH 01/14] add log for values unseen --- ...3-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index f7d6ca5656e8a..793b51bd28242 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -63,9 +63,15 @@ def upgrade(): session = db.Session(bind=bind) with op.batch_alter_table("slices") as batch_op: - for slc in session.query(Slice).filter(Slice.datasource_type == "query").all(): - upgrade_slc(slc) - session.add(slc) + for slc in session.query(Slice).filter(Slice.datasource_type != "table").all(): + if slc.datasource_type == "query": + upgrade_slc(slc) + session.add(slc) + + else: + print( + f"unknown value detected for slc.datasource_type: {slc.datasource_type}" + ) batch_op.create_check_constraint( "ck_chart_datasource", "datasource_type in ('table')" From 9f07e03491db3b3ef90828f828b61f7a415c4b4a Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 17:13:47 -0400 Subject: [PATCH 02/14] user logger --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 793b51bd28242..f9ed718f54f3a 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -27,6 +27,7 @@ down_revision = "07f9a902af1b" import json +import logging import sqlalchemy as sa from alembic import op @@ -36,6 +37,8 @@ Base = declarative_base() +logger = logging.getLogger() + class Slice(Base): # type: ignore __tablename__ = "slices" @@ -69,7 +72,7 @@ def upgrade(): session.add(slc) else: - print( + logger.info( f"unknown value detected for slc.datasource_type: {slc.datasource_type}" ) From 57a268c903914ba06ddc73a1ed70aa80951005b5 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 17:18:08 -0400 Subject: [PATCH 03/14] Update 2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index f9ed718f54f3a..688bb9be5af3c 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -37,7 +37,7 @@ Base = declarative_base() -logger = logging.getLogger() +logger = logging.getLogger(__name__) class Slice(Base): # type: ignore From bc251cf7ea0645335e7c3509ab0ccef30b4943ef Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 17:18:46 -0400 Subject: [PATCH 04/14] Update 2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 688bb9be5af3c..2a89f53de1354 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -72,7 +72,7 @@ def upgrade(): session.add(slc) else: - logger.info( + logger.warn( f"unknown value detected for slc.datasource_type: {slc.datasource_type}" ) From 087c818a626c1aa35032dc79cd79261a83dfa1f9 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 17:22:13 -0400 Subject: [PATCH 05/14] user log 1 more tiem --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 2a89f53de1354..d90f883b01816 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -72,8 +72,9 @@ def upgrade(): session.add(slc) else: - logger.warn( - f"unknown value detected for slc.datasource_type: {slc.datasource_type}" + logger.info( + "unknown value detected for slc.datasource_type: %s", + slc.datasource_type, ) batch_op.create_check_constraint( From 1c02d9c6f36d43af2ba62d426855d5bc5467177d Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 17:29:14 -0400 Subject: [PATCH 06/14] lets try this one --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index d90f883b01816..8f20480f907f8 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -51,6 +51,8 @@ class Slice(Base): # type: ignore def upgrade_slc(slc: Slice) -> None: # clean up all charts with datasource_type not != table slc.datasource_type = "table" + ds_id = None + ds_type = None try: params_dict = json.loads(slc.params) ds_id, ds_type = params_dict["datasource"].split("__") @@ -58,6 +60,7 @@ def upgrade_slc(slc: Slice) -> None: slc.params = json.dumps(params_dict) except Exception: # skip any malformatted params + logger.warning("failed to update slice: %s__%s", ds_id, ds_type) pass @@ -72,7 +75,7 @@ def upgrade(): session.add(slc) else: - logger.info( + logger.warning( "unknown value detected for slc.datasource_type: %s", slc.datasource_type, ) From 3c7509f930f0a4e480c7f8b33cce0347eacaac44 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 17:50:14 -0400 Subject: [PATCH 07/14] Update superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py Co-authored-by: Elizabeth Thompson --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 8f20480f907f8..dfd39ffaffad0 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -56,6 +56,10 @@ def upgrade_slc(slc: Slice) -> None: try: params_dict = json.loads(slc.params) ds_id, ds_type = params_dict["datasource"].split("__") + # the assumption here is that the query was saved as a dataset + # but the type wasn't written properly to the slice + # by updating the type here we expect it will either work + # or it will 404 when the dataset is looked up. params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) except Exception: From 46186beefb807c7a4a79092f263da7cf3de58b76 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 17:53:04 -0400 Subject: [PATCH 08/14] Update superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py Co-authored-by: Elizabeth Thompson --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index dfd39ffaffad0..e9fbf89bddcdf 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -60,6 +60,10 @@ def upgrade_slc(slc: Slice) -> None: # but the type wasn't written properly to the slice # by updating the type here we expect it will either work # or it will 404 when the dataset is looked up. + # the assumption here is that the query was saved as a dataset + # but the type wasn't written properly to the slice + # by updating the type here we expect it will either work + # or it will 404 when the dataset is looked up. params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) except Exception: From f72cde9c508572f50f32e2d521a78af97c8b958e Mon Sep 17 00:00:00 2001 From: Elizabeth Thompson Date: Wed, 3 May 2023 14:55:00 -0700 Subject: [PATCH 09/14] Update superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 1 + 1 file changed, 1 insertion(+) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index e9fbf89bddcdf..a7e9c3ec40a26 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -66,6 +66,7 @@ def upgrade_slc(slc: Slice) -> None: # or it will 404 when the dataset is looked up. params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) + logger.warning("updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id) except Exception: # skip any malformatted params logger.warning("failed to update slice: %s__%s", ds_id, ds_type) From acdb06eca85e24613411ba82f9e04dc22f537a10 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 17:55:21 -0400 Subject: [PATCH 10/14] Update 2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index a7e9c3ec40a26..2726ffba07770 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -59,11 +59,7 @@ def upgrade_slc(slc: Slice) -> None: # the assumption here is that the query was saved as a dataset # but the type wasn't written properly to the slice # by updating the type here we expect it will either work - # or it will 404 when the dataset is looked up. - # the assumption here is that the query was saved as a dataset - # but the type wasn't written properly to the slice - # by updating the type here we expect it will either work - # or it will 404 when the dataset is looked up. + # or it will 404 when the dataset is looked up. params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) logger.warning("updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id) From b8e7ef4d90ccf13b52298b2fc6a8001d56dce799 Mon Sep 17 00:00:00 2001 From: "Hugh A. Miles II" Date: Wed, 3 May 2023 18:01:14 -0400 Subject: [PATCH 11/14] Update 2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 2726ffba07770..6f9f10893d1e9 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -65,7 +65,7 @@ def upgrade_slc(slc: Slice) -> None: logger.warning("updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id) except Exception: # skip any malformatted params - logger.warning("failed to update slice: %s__%s", ds_id, ds_type) + logger.warning("failed to update slice.id = %s w/ datasource = %s__%s to %s__table", slc.id, ds_id, ds_type, ds_id) pass From d0b8ac1f3767836060917019377f745b8b09f625 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 18:11:47 -0400 Subject: [PATCH 12/14] ok --- ...03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 6f9f10893d1e9..68b2f820c5d8d 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -59,13 +59,21 @@ def upgrade_slc(slc: Slice) -> None: # the assumption here is that the query was saved as a dataset # but the type wasn't written properly to the slice # by updating the type here we expect it will either work - # or it will 404 when the dataset is looked up. + # or it will 404 when the dataset is looked up. params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) - logger.warning("updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id) + logger.warning( + "updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id + ) except Exception: # skip any malformatted params - logger.warning("failed to update slice.id = %s w/ datasource = %s__%s to %s__table", slc.id, ds_id, ds_type, ds_id) + logger.warning( + "failed to update slice.id = %s w/ datasource = %s__%s to %s__table", + slc.id, + ds_id, + ds_type, + ds_id, + ) pass From 77f44c3863f2479a5ae1c1089ebc2247282c3793 Mon Sep 17 00:00:00 2001 From: Elizabeth Thompson Date: Wed, 3 May 2023 15:46:42 -0700 Subject: [PATCH 13/14] Update superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py --- .../2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 68b2f820c5d8d..4cefc9b4b3cc4 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -63,7 +63,11 @@ def upgrade_slc(slc: Slice) -> None: params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) logger.warning( - "updated slice datasource from %s__%s to %s__table", ds_id, ds_type, ds_id + "updated slice datasource from %s__%s to %s__table for slice: %s", + ds_id, + ds_type, + ds_id, + slc.id ) except Exception: # skip any malformatted params From 809292e961fc272dc98adfdab92bef1055492ae0 Mon Sep 17 00:00:00 2001 From: hughhhh Date: Wed, 3 May 2023 21:27:35 -0400 Subject: [PATCH 14/14] linting --- ...023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py index 4cefc9b4b3cc4..8d3115acbdb91 100644 --- a/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py +++ b/superset/migrations/versions/2023-03-27_12-30_7e67aecbf3f1_chart_ds_constraint.py @@ -63,11 +63,11 @@ def upgrade_slc(slc: Slice) -> None: params_dict["datasource"] = f"{ds_id}__table" slc.params = json.dumps(params_dict) logger.warning( - "updated slice datasource from %s__%s to %s__table for slice: %s", - ds_id, - ds_type, - ds_id, - slc.id + "updated slice datasource from %s__%s to %s__table for slice: %s", + ds_id, + ds_type, + ds_id, + slc.id, ) except Exception: # skip any malformatted params