Merge pull request #693 from openedx/cag/superset4.0

feat: migrate to superset v4.0.0
openedx · Apr 9, 2024 · 167ad65 · 167ad65
2 parents d6c7434 + 1309008
commit 167ad65
Show file tree

Hide file tree

Showing 46 changed files with 474 additions and 260 deletions.
diff --git a/tutoraspects/plugin.py b/tutoraspects/plugin.py
@@ -372,6 +372,10 @@
         # By default it is on if Caddy is enabled, but it can be set separately in
         # case you are running a different proxy or otherwise have different needs.
         ("SUPERSET_ENABLE_PROXY_FIX", "{{ENABLE_WEB_PROXY}}"),
+        # This setting allows operators to automatically refresh the datasets
+        # in the Superset database. This is useful for keeping the columns up to
+        # date with the latest changes in DBT.
+        ("SUPERSET_REFRESH_DATASETS", False),
         ######################
         # dbt Settings
         # For the most part you shouldn't have to touch these

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py
@@ -15,6 +15,7 @@
 from superset.examples.utils import load_configs_from_directory
 from superset.extensions import db
 from superset.models.dashboard import Dashboard
+from superset.connectors.sqla.models import SqlaTable
 from superset.utils.database import get_or_create_db
 from superset.models.embedded_dashboard import EmbeddedDashboard
 from pythonpath.localization import get_translation
@@ -91,6 +92,7 @@ def create_assets():
     import_assets()
     update_dashboard_roles(roles)
     update_embeddable_uuids()
+    update_datasets()
 
 
 def import_databases():
@@ -258,7 +260,6 @@ def update_dashboard_roles(roles):
         dashboard = db.session.query(Dashboard).filter_by(uuid=dashboard_uuid).one()
         print("Importing dashboard roles", dashboard_uuid, role_ids)
         dashboard.roles = role_ids
-        dashboard.published = True
         if owners:
             dashboard.owners = owners
         db.session.commit()
@@ -282,5 +283,17 @@ def update_embeddable_uuids():
         db.session.commit()
 
 
+def update_datasets():
+    """Update the datasets"""
+    print("Refreshing datasets")
+    if {{SUPERSET_REFRESH_DATASETS}}:
+        datasets = (
+            db.session.query(SqlaTable).all()
+        )
+        for dataset in datasets:
+            print(f"Refreshing dataset {dataset.table_name}")
+            dataset.fetch_metadata(commit=True)
+
+
 if __name__ == "__main__":
     main()
diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/performance_metrics.py
@@ -14,7 +14,7 @@
 import sqlparse
 from flask import g
 from superset import security_manager
-from superset.charts.data.commands.get_data_command import ChartDataCommand
+from superset.commands.chart.data.get_data_command import ChartDataCommand
 from superset.charts.schemas import ChartDataQueryContextSchema
 from superset.extensions import db
 from superset.models.dashboard import Dashboard

diff --git a/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py b/tutoraspects/templates/aspects/apps/superset/pythonpath/superset_config_docker.py
@@ -91,6 +91,7 @@
     "EMBEDDABLE_CHARTS": True,
     "EMBEDDED_SUPERSET": True,
     "ENABLE_TEMPLATE_PROCESSING": True,
+    "TAGGING_SYSTEM": True,
 }
 
 # Add this custom template processor which returns the list of courses the current user can access

diff --git a/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile b/tutoraspects/templates/aspects/build/aspects-superset/Dockerfile
@@ -3,7 +3,7 @@
 # https://github.com/apache/superset/releases
 # https://github.com/apache/superset/blob/master/Dockerfile
 # https://superset.apache.org/docs/databases/installing-database-drivers
-FROM apache/superset:3.0.1
+FROM apache/superset:4.0.0
 
 USER root
 

diff --git a/...spects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml b/...spects/build/aspects-superset/openedx-assets/assets/charts/Course_Grade_Distribution.yaml
@@ -12,6 +12,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...ld/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml b/...ld/aspects-superset/openedx-assets/assets/charts/Currently_Enrolled_Learners_Per_Day.yaml
@@ -24,6 +24,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: enrollment_status_date
+  annotation_layers: []
   color_picker:
     a: 1
     b: 135

diff --git a/...tes/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml b/...tes/aspects/build/aspects-superset/openedx-assets/assets/charts/Distinct_forum_users.yaml
@@ -11,6 +11,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   extra_form_data: {}
   header_font_size: 0.4
   metric:

diff --git a/...aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml b/...aspects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Attempts.yaml
@@ -17,6 +17,7 @@ params:
     operatorId: EQUALS
     sqlExpression: null
     subject: success
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...pects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml b/...pects-superset/openedx-assets/assets/charts/Distribution_Of_Hints_Per_Correct_Answer.yaml
@@ -9,6 +9,7 @@ description: This chart displays counts of the number of times hints (including
   group everyone into "0".
 params:
   adhoc_filters: []
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...s/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml b/...s/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Problem_Grades.yaml
@@ -13,6 +13,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...spects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml b/...spects/build/aspects-superset/openedx-assets/assets/charts/Distribution_Of_Responses.yaml
@@ -13,6 +13,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...s/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml b/...s/build/aspects-superset/openedx-assets/assets/charts/Enrollments_By_Enrollment_Mode.yaml
@@ -23,6 +23,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   color_scheme: supersetColors
   date_format: smart_date
   extra_form_data: {}

diff --git a/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml b/...templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Posts_per_user.yaml
@@ -6,6 +6,7 @@ dataset_uuid: a2823d2e-54c4-4378-98c2-817f000c14ab
 description: null
 params:
   adhoc_filters: []
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...es/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml b/...es/aspects/build/aspects-superset/openedx-assets/assets/charts/Responses_Per_Problem.yaml
@@ -8,6 +8,7 @@ description: The number of students who have responded to a question, and whethe
   but this chart counts each student only once per question.
 params:
   adhoc_filters: []
+  annotation_layers: []
   bar_stacked: true
   bottom_margin: auto
   color_scheme: supersetColors

diff --git a/...s/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml b/...s/build/aspects-superset/openedx-assets/assets/charts/Transcripts_Captions_Per_Video.yaml
@@ -12,6 +12,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...s/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml b/...s/aspects/build/aspects-superset/openedx-assets/assets/charts/Watched_Video_Segments.yaml
@@ -12,6 +12,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: started_at
+  annotation_layers: []
   bar_stacked: true
   bottom_margin: auto
   color_scheme: supersetColors

diff --git a/...plates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml b/...plates/aspects/build/aspects-superset/openedx-assets/assets/charts/Watches_Per_Video.yaml
@@ -13,6 +13,7 @@ params:
     expressionType: SIMPLE
     operator: TEMPORAL_RANGE
     subject: emission_time
+  annotation_layers: []
   bottom_margin: auto
   color_scheme: supersetColors
   columns: []

diff --git a/...aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml b/...aspects/build/aspects-superset/openedx-assets/assets/dashboards/Instructor_Dashboard.yaml
@@ -1,6 +1,8 @@
 _file_name: Instructor_Dashboard.yaml
 _roles:
 - '{{ SUPERSET_ROLES_MAPPING.instructor }}'
+certification_details: null
+certified_by: null
 css: ''
 dashboard_title: Instructor Dashboard
 description: null
@@ -420,7 +422,7 @@ position:
     children: []
     id: CHART-AZZnl_lpMv
     meta:
-      chartId: 229
+      chartId: 22
       height: 50
       sliceName: Watches Per Video
       uuid: 829c1d5b-2844-4115-876a-34ad3b3cad64
@@ -435,7 +437,7 @@ position:
     children: []
     id: CHART-GFCO8s2cxv
     meta:
-      chartId: 625
+      chartId: 20
       height: 50
       sliceName: Distribution Of Responses
       uuid: f1651c44-a8f4-4b44-ad49-962823009319
@@ -450,7 +452,7 @@ position:
     children: []
     id: CHART-GJJ8VYQ03v
     meta:
-      chartId: 783
+      chartId: 6
       height: 50
       sliceName: Posts per user
       uuid: bc191ce7-f39d-48db-86a9-d19949f4211d
@@ -465,7 +467,7 @@ position:
     children: []
     id: CHART-Jr-gNVms2Q
     meta:
-      chartId: 114
+      chartId: 13
       height: 50
       sliceName: Enrollment Events Per Day
       uuid: bb1147cc-b7bc-44b7-b06a-79b0db6626aa
@@ -480,7 +482,7 @@ position:
     children: []
     id: CHART-OMy4wjRBWt
     meta:
-      chartId: 682
+      chartId: 28
       height: 50
       sliceName: Watched Video Segments
       uuid: 2985a9db-c338-4008-af52-2930b81ee2e5
@@ -495,7 +497,7 @@ position:
     children: []
     id: CHART-RTO33WE9FH
     meta:
-      chartId: 597
+      chartId: 16
       height: 50
       sliceName: Responses Per Problem
       uuid: a3e79162-4ace-4349-ab34-89aa60ae75ed
@@ -510,7 +512,7 @@ position:
     children: []
     id: CHART-Tej2oLPBAl
     meta:
-      chartId: 701
+      chartId: 38
       height: 50
       sliceName: Transcripts / Captions Per Video
       uuid: 6b830def-f3ca-4b4c-9455-7a7b7354bce8
@@ -555,7 +557,7 @@ position:
     children: []
     id: CHART-evjVO-ZSSd
     meta:
-      chartId: 156
+      chartId: 39
       height: 50
       sliceName: Enrollments By Enrollment Mode
       uuid: 05ed7102-5464-4e2f-86ae-31700b787cc3
@@ -570,7 +572,7 @@ position:
     children: []
     id: CHART-j3trfNh8_p
     meta:
-      chartId: 325
+      chartId: 30
       height: 50
       sliceName: Pageview engagement by section/subsection
       uuid: 366a8193-30c3-4aaf-a1ac-360609dfa0ed
@@ -585,8 +587,8 @@ position:
     children: []
     id: CHART-jfm-20qPKn
     meta:
-      chartId: 816
-      height: 55
+      chartId: 11
+      height: 50
       sliceName: Video engagement by section/subsection
       uuid: 21861f47-9e8b-4715-b1ff-e7f4aa595c14
       width: 12
@@ -600,7 +602,7 @@ position:
     children: []
     id: CHART-lTr8DL3XuI
     meta:
-      chartId: 695
+      chartId: 4
       height: 50
       sliceName: Distribution Of Hints Per Correct Answer
       uuid: ee94be4c-6fdd-4295-b43c-40890d6c549d
@@ -615,7 +617,7 @@ position:
     children: []
     id: CHART-o56v9yEe2I
     meta:
-      chartId: 770
+      chartId: 32
       height: 50
       sliceName: Distribution Of Problem Grades
       uuid: 4f7e3606-f5de-4643-97c0-bbb6340a3df2
@@ -630,7 +632,7 @@ position:
     children: []
     id: CHART-qG1WaGKl_b
     meta:
-      chartId: 154
+      chartId: 35
       height: 50
       sliceName: Distinct forum users
       uuid: feb323ad-c819-49ca-a336-584bd9ff1a2e
@@ -645,7 +647,7 @@ position:
     children: []
     id: CHART-rnb6PSwCOS
     meta:
-      chartId: 339
+      chartId: 17
       height: 50
       sliceName: Currently Enrolled Learners Per Day
       uuid: ed2fe731-6544-422f-bc55-42f399f48b2c
@@ -660,7 +662,7 @@ position:
     children: []
     id: CHART-tWnaoVNNTH
     meta:
-      chartId: 327
+      chartId: 12
       height: 50
       sliceName: Distribution Of Attempts
       uuid: db90930f-f16e-4c32-8050-0e4abae28f4c
@@ -675,7 +677,7 @@ position:
     children: []
     id: CHART-w-k4N2T_L8
     meta:
-      chartId: 516
+      chartId: 2
       height: 50
       sliceName: Course Grade Distribution
       uuid: f9adbc85-1f50-4c04-ace3-31ba7390de5e
@@ -1130,6 +1132,7 @@ position:
     parents:
     - ROOT_ID
     type: TABS
+published: true
 slug: instructor-dashboard
 uuid: 1d6bf904-f53f-47fd-b1c9-6cd7e284d286
 version: 1.0.0
diff --git a/...templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml b/...templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/course_names.yaml
@@ -1,10 +1,11 @@
 _file_name: course_names.yaml
+always_filter_main_dttm: false
 cache_timeout: null
 columns:
 - advanced_data_type: null
   column_name: course_key
   description: null
-  expression: null
+  expression: ''
   extra: {}
   filterable: true
   groupby: true
@@ -16,7 +17,7 @@ columns:
 - advanced_data_type: null
   column_name: course_name
   description: null
-  expression: null
+  expression: ''
   extra: {}
   filterable: true
   groupby: true
@@ -28,7 +29,7 @@ columns:
 - advanced_data_type: null
   column_name: course_run
   description: null
-  expression: null
+  expression: ''
   extra: {}
   filterable: true
   groupby: true
@@ -40,7 +41,7 @@ columns:
 - advanced_data_type: null
   column_name: org
   description: null
-  expression: null
+  expression: ''
   extra: {}
   filterable: true
   groupby: true

diff --git a/...s/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml b/...s/templates/aspects/build/aspects-superset/openedx-assets/assets/datasets/dashboards.yaml
@@ -227,7 +227,7 @@ normalize_columns: true
 offset: 0
 params: null
 schema: superset
-sql: 'select * from '
+sql: 'select * from dashboards'
 table_name: dashboards
 template_params: null
 uuid: 3dee777a-3dd4-4fcb-a749-4d13651b3444