Merge pull request #1060 from saraburns1/in_context_upadtes_423

feat: Updates to In-Context analytics dashboards

Author: saraburns1

tutoraspects/asset_command_helpers.py

@@ -32,6 +32,9 @@
     "openedx-assets",
     "assets",
 )
+QUERIES_PATH = os.path.join(
+    os.path.dirname(os.path.abspath(__file__)), "templates", "openedx-assets", "queries"
+)
 
 
 def str_presenter(dumper, data):
@@ -595,3 +598,42 @@ def delete_aspects_unused_assets(echo):
             for asset_type, uuids in unused_uuids.items():
                 for uuid, data in uuids.items():
                     echo(f'{asset_type} {data.get("name")} (UUID: {uuid})')
+
+
+def find_unused_queries(echo):
+    """
+    Compare all sql files in openedx-assets/queries to the file names referenced
+    in each dataset's yaml. List any .sql files that are not referenced as
+    possible unused queries. Ignore any sql files listed in `aspects_asset_list`
+    under ignored_files.
+    """
+    dataset_query_list = []
+    for query_file in glob.iglob(QUERIES_PATH + "/*.sql", recursive=True):
+        match = re.search(r"\w+\b.sql", query_file)
+        if match:
+            dataset_query_list.append(match.group())
+
+    for file_name in glob.iglob(ASSETS_PATH + "/datasets/*.yaml", recursive=True):
+        with open(file_name, "r", encoding="utf-8") as file:
+            # We have to remove the jinja for it to parse
+            file_str = file.read()
+            file_str = file_str.replace("{{", "").replace("}}", "")
+            asset = yaml.safe_load(file_str)
+            sql = asset["sql"].strip()
+            match = re.search(r"\w+\b.sql", sql)
+            if match:
+                dataset_query_list.remove(match.group())
+
+    # Remove uuids from 'all' list that are in ignored yaml
+    with open(ASPECT_ASSET_LIST, "r", encoding="utf-8") as file:
+        aspects_assets = yaml.safe_load(file)
+
+    ignored_uuids = aspects_assets.get("ignored_files")
+    if ignored_uuids["queries"]:
+        for file_name in ignored_uuids["queries"]:
+            dataset_query_list.remove(file_name)
+
+    if dataset_query_list:
+        echo(click.style("Potentially unused query files detected:"))
+        for file in dataset_query_list:
+            echo(f"{file}")

tutoraspects/commands_v1.py

@@ -14,6 +14,7 @@
     deduplicate_superset_assets,
     import_superset_assets,
     delete_aspects_unused_assets,
+    find_unused_queries,
 )
 
 
@@ -366,6 +367,7 @@ def serialize_zip(file, base_assets_path):
     click.echo()
     deduplicate_superset_assets(click.echo)
     delete_aspects_unused_assets(click.echo)
+    find_unused_queries(click.echo)
 
     click.echo()
     click.echo("Asset merge complete!")
@@ -385,6 +387,7 @@ def check_superset_assets():
     """
     deduplicate_superset_assets(click.echo)
     delete_aspects_unused_assets(click.echo)
+    find_unused_queries(click.echo)
 
     click.echo()
     click.echo(

tutoraspects/templates/aspects/apps/superset/pythonpath/aspects_asset_list.yaml

@@ -18,3 +18,10 @@ ignored_uuids:
   datasets:
   charts:
     - bb13bb31-c797-4ed3-a7f9-7825cc6dc482 # "Query performance" needed for the performance_metrics script
+  
+ignored_files:
+  queries:
+    # These queries are used in other query files
+    - user_filters.sql 
+    - common_filters.sql
+    - at_risk_learner_filter.sql

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Final_Response_Results_ab43eb.yaml

@@ -5,17 +5,12 @@ certified_by: null
 dataset_uuid: e6134213-e7f4-4081-9861-356c93f376dc
 description: null
 params:
-  adhoc_filters:
-  - clause: WHERE
-    comparator: No filter
-    expressionType: SIMPLE
-    operator: TEMPORAL_RANGE
-    subject: emission_time
+  adhoc_filters: []
   annotation_layers: []
-  color_scheme: supersetColors
+  color_scheme: aspects_default
   extra_form_data: {}
   groupby:
-  - datasourceWarning: false
+  - datasourceWarning: true
     expressionType: SQL
     label: success
     sqlExpression: |-
@@ -55,47 +50,43 @@ params:
   rich_tooltip: true
   row_limit: 10000
   show_empty_columns: true
-  show_legend: true
+  show_legend: false
   sort_series_type: sum
   stack: Stack
   tooltipTimeFormat: smart_date
   truncateXAxis: false
   truncateYAxis: false
   truncate_metric: true
   viz_type: echarts_timeseries_bar
-  x_axis: problem_name
+  x_axis: subsection_number
   x_axis_sort_series: name
   x_axis_sort_series_ascending: true
   x_axis_time_format: smart_date
-  x_axis_title_margin: 15
+  x_axis_title: Subsection
+  x_axis_title_margin: 30
   y_axis_bounds:
   - null
   - null
   y_axis_format: SMART_NUMBER
   y_axis_title: Respondents
-  y_axis_title_margin: 30
-  y_axis_title_position: Left
+  y_axis_title_margin: 15
+  y_axis_title_position: Top
 query_context:
   datasource:
-    id: 725
+    id: 644
     type: table
   force: false
   form_data:
-    adhoc_filters:
-    - clause: WHERE
-      comparator: No filter
-      expressionType: SIMPLE
-      operator: TEMPORAL_RANGE
-      subject: emission_time
+    adhoc_filters: []
     annotation_layers: []
-    color_scheme: supersetColors
+    color_scheme: aspects_default
     dashboards:
-    - 124
-    datasource: 725__table
+    - 4741
+    datasource: 644__table
     extra_form_data: {}
     force: false
     groupby:
-    - datasourceWarning: false
+    - datasourceWarning: true
       expressionType: SQL
       label: success
       sqlExpression: |-
@@ -137,36 +128,37 @@ query_context:
     rich_tooltip: true
     row_limit: 10000
     show_empty_columns: true
-    show_legend: true
-    slice_id: 1670
+    show_legend: false
+    slice_id: 1341
     sort_series_type: sum
     stack: Stack
     tooltipTimeFormat: smart_date
     truncateXAxis: false
     truncateYAxis: false
     truncate_metric: true
     viz_type: echarts_timeseries_bar
-    x_axis: problem_name
+    x_axis: subsection_number
     x_axis_sort_series: name
     x_axis_sort_series_ascending: true
     x_axis_time_format: smart_date
-    x_axis_title_margin: 15
+    x_axis_title: Subsection
+    x_axis_title_margin: 30
     y_axis_bounds:
     - null
     - null
     y_axis_format: SMART_NUMBER
     y_axis_title: Respondents
-    y_axis_title_margin: 30
-    y_axis_title_position: Left
+    y_axis_title_margin: 15
+    y_axis_title_position: Top
   queries:
   - annotation_layers: []
     applied_time_extras: {}
     columns:
     - columnType: BASE_AXIS
       expressionType: SQL
-      label: problem_name
-      sqlExpression: problem_name
-    - datasourceWarning: false
+      label: subsection_number
+      sqlExpression: subsection_number
+    - datasourceWarning: true
       expressionType: SQL
       label: success
       sqlExpression: |-
@@ -176,10 +168,7 @@ query_context:
     extras:
       having: ''
       where: ''
-    filters:
-    - col: emission_time
-      op: TEMPORAL_RANGE
-      val: No filter
+    filters: []
     metrics:
     - aggregate: COUNT_DISTINCT
       column:
@@ -244,7 +233,7 @@ query_context:
         - success
         drop_missing_columns: false
         index:
-        - problem_name
+        - subsection_number
     - operation: rename
       options:
         columns:
@@ -254,7 +243,7 @@ query_context:
     - operation: flatten
     row_limit: 10000
     series_columns:
-    - datasourceWarning: false
+    - datasourceWarning: true
       expressionType: SQL
       label: success
       sqlExpression: |-