Merge pull request #775 from openedx/cag/verbose-name

fix: automatically add verbose name for dataset columns

tutoraspects/asset_command_helpers.py

@@ -149,6 +149,11 @@ def omit_templated_vars(self, content: dict, existing: dict):
                         except IndexError:
                             pass
 
+    def process(self, content: dict, existing: dict):
+        """
+        Process the asset content before writing it to a file.
+        """
+
 
 class ChartAsset(Asset):
     """
@@ -163,6 +168,10 @@ class ChartAsset(Asset):
     ]
     raw_vars = ["sqlExpression", "query_context"]
 
+    def process(self, content: dict, existing: dict):
+        if not content.get("query_context"):
+            content["query_context"] = existing.get("query_context")
+
 
 class DashboardAsset(Asset):
     """
@@ -182,6 +191,18 @@ class DatasetAsset(Asset):
     templated_vars = ["schema", "table_name", "sql"]
     omitted_vars = ["extra.certification"]
 
+    def process(self, content: dict, existing: dict):
+        """
+        Process the content of the chart asset.
+        """
+        for column in content.get("columns", []):
+            if not column.get("verbose_name"):
+                column["verbose_name"] = column["column_name"].replace("_", " ").title()
+
+        for metric in content.get("metrics", []):
+            if not metric.get("verbose_name"):
+                metric["verbose_name"] = metric["metric_name"].replace("_", " ").title()
+
 
 class DatabaseAsset(Asset):
     """
@@ -265,6 +286,7 @@ def validate_asset_file(asset_path, content, echo):
 
             cls.remove_content(content)
             cls.omit_templated_vars(content, existing)
+            cls.process(content, existing)
             # We found the correct class, we can stop looking.
             break
     return out_path, needs_review

tutoraspects/templates/aspects/apps/superset/pythonpath/create_assets.py

@@ -248,31 +248,34 @@ def import_assets():
     # performance_metric script however, so we keep them in the assets.
     # This just blanks them in the database after import, which forces a
     # query to get the assets instead of using the query context.
+    owners = get_owners()
     for o in db.session.query(Slice).all():
         if o.query_context:
             o.query_context = None
+        if owners:
+            o.owners = owners
     db.session.commit()
 
 
 def update_dashboard_roles(roles):
     """Update the roles of the dashboards"""
-    owners_username = {{SUPERSET_OWNERS}}
-
-    owners = []
-
-    for owner in owners_username:
-        user = security_manager.find_user(username=owner)
-        if user:
-            owners.append(user)
+    owners = get_owners()
 
     for dashboard_uuid, role_ids in roles.items():
-        logger.info(f"Importing dashboard roles: {dashboard_uuid} - {role_ids}")
         dashboard = db.session.query(Dashboard).filter_by(uuid=dashboard_uuid).one()
         dashboard.roles = role_ids
         if owners:
             dashboard.owners = owners
         db.session.commit()
 
+def get_owners():
+    owners_username = {{SUPERSET_OWNERS}}
+    owners = []
+    for owner in owners_username:
+        user = security_manager.find_user(username=owner)
+        if user:
+            owners.append(user)
+    return owners
 
 def update_embeddable_uuids():
     """Update the uuids of the embeddable dashboards"""

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Course_Information.yaml

@@ -17,33 +17,33 @@ params:
       d3SmallNumberFormat: ~g
   conditional_formatting:
   - colorScheme: '#ACE1C4'
-    column: Graded Learners
+    column: graded_learners
     operator: '>'
     targetValue: 0
   - colorScheme: '#FDE380'
-    column: Avg Course Grade
+    column: avg_course_grade
     operator: "\u2264 x \u2264"
     targetValueLeft: '50'
     targetValueRight: '70'
   - colorScheme: '#EFA1AA'
-    column: Avg Course Grade
+    column: avg_course_grade
     operator: <
     targetValue: 50
   - colorScheme: '#ACE1C4'
-    column: Avg Course Grade
+    column: avg_course_grade
     operator: "\u2265"
     targetValue: 70
   - colorScheme: '#ACE1C4'
-    column: Median Course Grade
+    column: median_course_grade
     operator: "\u2265"
     targetValue: 70
   - colorScheme: '#FDE380'
-    column: Median Course Grade
+    column: median_course_grade
     operator: "\u2264 x \u2264"
     targetValueLeft: '50'
     targetValueRight: '70'
   - colorScheme: '#EFA1AA'
-    column: Median Course Grade
+    column: median_course_grade
     operator: <
     targetValue: 50
   extra_form_data: {}
@@ -53,46 +53,9 @@ params:
   - course_name
   - course_run
   metrics:
-  - aggregate: COUNT_DISTINCT
-    column:
-      advanced_data_type: null
-      certification_details: null
-      certified_by: null
-      column_name: actor_id
-      description: null
-      expression: ''
-      filterable: true
-      groupby: true
-      id: 538
-      is_certified: false
-      is_dttm: false
-      python_date_format: null
-      type: String
-      type_generic: 1
-      verbose_name: null
-      warning_markdown: null
-    datasourceWarning: false
-    expressionType: SIMPLE
-    hasCustomLabel: true
-    label: Graded Learners
-    optionName: metric_25t7qgbwxu5_a9fbiaboytw
-    sqlExpression: null
-  - aggregate: null
-    column: null
-    datasourceWarning: false
-    expressionType: SQL
-    hasCustomLabel: true
-    label: Avg Course Grade
-    optionName: metric_i5mlkxng7kb_5auir2yzvnx
-    sqlExpression: round(AVG(course_grade),2) * 100
-  - aggregate: null
-    column: null
-    datasourceWarning: false
-    expressionType: SQL
-    hasCustomLabel: true
-    label: Median Course Grade
-    optionName: metric_prd604bf33n_j3zxzos2g5
-    sqlExpression: 'quantile(0.5)(round(course_grade,2) * 100) '
+  - graded_learners
+  - avg_course_grade
+  - median_course_grade
   order_by_cols: []
   order_desc: true
   percent_metrics: []
@@ -104,66 +67,9 @@ params:
   temporal_columns_lookup: {}
   time_grain_sqla: P1D
   viz_type: table
-query_context: "{\"datasource\":{\"id\":54,\"type\":\"table\"},\"force\":false,\"\
-  queries\":[{\"filters\":[],\"extras\":{\"time_grain_sqla\":\"P1D\",\"having\":\"\
-  \",\"where\":\"\"},\"applied_time_extras\":{},\"columns\":[\"org\",\"course_key\"\
-  ,\"course_name\",\"course_run\"],\"metrics\":[{\"aggregate\":\"COUNT_DISTINCT\"\
-  ,\"column\":{\"advanced_data_type\":null,\"certification_details\":null,\"certified_by\"\
-  :null,\"column_name\":\"actor_id\",\"description\":null,\"expression\":\"\",\"filterable\"\
-  :true,\"groupby\":true,\"id\":538,\"is_certified\":false,\"is_dttm\":false,\"python_date_format\"\
-  :null,\"type\":\"String\",\"type_generic\":1,\"verbose_name\":null,\"warning_markdown\"\
-  :null},\"datasourceWarning\":false,\"expressionType\":\"SIMPLE\",\"hasCustomLabel\"\
-  :true,\"label\":\"Graded Learners\",\"optionName\":\"metric_25t7qgbwxu5_a9fbiaboytw\"\
-  ,\"sqlExpression\":null},{\"aggregate\":null,\"column\":null,\"datasourceWarning\"\
-  :false,\"expressionType\":\"SQL\",\"hasCustomLabel\":true,\"label\":\"Avg Course\
-  \ Grade\",\"optionName\":\"metric_i5mlkxng7kb_5auir2yzvnx\",\"sqlExpression\":\"\
-  round(AVG(course_grade),2) * 100\"},{\"aggregate\":null,\"column\":null,\"datasourceWarning\"\
-  :false,\"expressionType\":\"SQL\",\"hasCustomLabel\":true,\"label\":\"Median Course\
-  \ Grade\",\"optionName\":\"metric_prd604bf33n_j3zxzos2g5\",\"sqlExpression\":\"\
-  quantile(0.5)(round(course_grade,2) * 100) \"}],\"orderby\":[[{\"aggregate\":\"\
-  COUNT_DISTINCT\",\"column\":{\"advanced_data_type\":null,\"certification_details\"\
-  :null,\"certified_by\":null,\"column_name\":\"actor_id\",\"description\":null,\"\
-  expression\":\"\",\"filterable\":true,\"groupby\":true,\"id\":538,\"is_certified\"\
-  :false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"String\",\"type_generic\"\
-  :1,\"verbose_name\":null,\"warning_markdown\":null},\"datasourceWarning\":false,\"\
-  expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"label\":\"Graded Learners\"\
-  ,\"optionName\":\"metric_25t7qgbwxu5_a9fbiaboytw\",\"sqlExpression\":null},false]],\"\
-  annotation_layers\":[],\"row_limit\":1000,\"series_limit\":0,\"order_desc\":true,\"\
-  url_params\":{},\"custom_params\":{},\"custom_form_data\":{},\"post_processing\"\
-  :[]}],\"form_data\":{\"datasource\":\"54__table\",\"viz_type\":\"table\",\"slice_id\"\
-  :128,\"query_mode\":\"aggregate\",\"groupby\":[\"org\",\"course_key\",\"course_name\"\
-  ,\"course_run\"],\"time_grain_sqla\":\"P1D\",\"temporal_columns_lookup\":{},\"metrics\"\
-  :[{\"aggregate\":\"COUNT_DISTINCT\",\"column\":{\"advanced_data_type\":null,\"certification_details\"\
-  :null,\"certified_by\":null,\"column_name\":\"actor_id\",\"description\":null,\"\
-  expression\":\"\",\"filterable\":true,\"groupby\":true,\"id\":538,\"is_certified\"\
-  :false,\"is_dttm\":false,\"python_date_format\":null,\"type\":\"String\",\"type_generic\"\
-  :1,\"verbose_name\":null,\"warning_markdown\":null},\"datasourceWarning\":false,\"\
-  expressionType\":\"SIMPLE\",\"hasCustomLabel\":true,\"label\":\"Graded Learners\"\
-  ,\"optionName\":\"metric_25t7qgbwxu5_a9fbiaboytw\",\"sqlExpression\":null},{\"aggregate\"\
-  :null,\"column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"\
-  hasCustomLabel\":true,\"label\":\"Avg Course Grade\",\"optionName\":\"metric_i5mlkxng7kb_5auir2yzvnx\"\
-  ,\"sqlExpression\":\"round(AVG(course_grade),2) * 100\"},{\"aggregate\":null,\"\
-  column\":null,\"datasourceWarning\":false,\"expressionType\":\"SQL\",\"hasCustomLabel\"\
-  :true,\"label\":\"Median Course Grade\",\"optionName\":\"metric_prd604bf33n_j3zxzos2g5\"\
-  ,\"sqlExpression\":\"quantile(0.5)(round(course_grade,2) * 100) \"}],\"all_columns\"\
-  :[],\"percent_metrics\":[],\"adhoc_filters\":[],\"order_by_cols\":[],\"row_limit\"\
-  :1000,\"server_page_length\":10,\"order_desc\":true,\"table_timestamp_format\":\"\
-  smart_date\",\"show_cell_bars\":true,\"color_pn\":true,\"column_config\":{\"Avg\
-  \ Course Grade\":{\"d3NumberFormat\":\"~g\",\"d3SmallNumberFormat\":\"~g\"},\"Median\
-  \ Course Grade\":{\"d3NumberFormat\":\"~g\",\"d3SmallNumberFormat\":\"~g\"}},\"\
-  conditional_formatting\":[{\"colorScheme\":\"#ACE1C4\",\"column\":\"Graded Learners\"\
-  ,\"operator\":\">\",\"targetValue\":0},{\"colorScheme\":\"#FDE380\",\"column\":\"\
-  Avg Course Grade\",\"operator\":\"\u2264 x \u2264\",\"targetValueLeft\":\"50\",\"\
-  targetValueRight\":\"70\"},{\"colorScheme\":\"#EFA1AA\",\"column\":\"Avg Course\
-  \ Grade\",\"operator\":\"<\",\"targetValue\":50},{\"colorScheme\":\"#ACE1C4\",\"\
-  column\":\"Avg Course Grade\",\"operator\":\"\u2265\",\"targetValue\":70},{\"colorScheme\"\
-  :\"#ACE1C4\",\"column\":\"Median Course Grade\",\"operator\":\"\u2265\",\"targetValue\"\
-  :70},{\"colorScheme\":\"#FDE380\",\"column\":\"Median Course Grade\",\"operator\"\
-  :\"\u2264 x \u2264\",\"targetValueLeft\":\"50\",\"targetValueRight\":\"70\"},{\"\
-  colorScheme\":\"#EFA1AA\",\"column\":\"Median Course Grade\",\"operator\":\"<\"\
-  ,\"targetValue\":50}],\"extra_form_data\":{},\"dashboards\":[9],\"force\":false,\"\
-  result_format\":\"json\",\"result_type\":\"full\"},\"result_format\":\"json\",\"\
-  result_type\":\"full\"}"
+query_context: '{"datasource":{"id":282,"type":"table"},"force":false,"queries":[{"filters":[],"extras":{"time_grain_sqla":"P1D","having":"","where":""},"applied_time_extras":{},"columns":["org","course_key","course_name","course_run"],"metrics":["graded_learners","avg_course_grade","median_course_grade"],"orderby":[["graded_learners",false]],"annotation_layers":[],"row_limit":1000,"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"post_processing":[]}],"form_data":{"datasource":"282__table","viz_type":"table","slice_id":528,"query_mode":"aggregate","groupby":["org","course_key","course_name","course_run"],"time_grain_sqla":"P1D","temporal_columns_lookup":{},"metrics":["graded_learners","avg_course_grade","median_course_grade"],"all_columns":[],"percent_metrics":[],"adhoc_filters":[],"order_by_cols":[],"row_limit":1000,"server_page_length":10,"order_desc":true,"table_timestamp_format":"smart_date","show_cell_bars":true,"color_pn":true,"column_config":{"Avg
+  Course Grade":{"d3NumberFormat":"~g","d3SmallNumberFormat":"~g"},"Median Course
+  Grade":{"d3NumberFormat":"~g","d3SmallNumberFormat":"~g"}},"conditional_formatting":[{"column":"graded_learners","colorScheme":"#ACE1C4","operator":">","targetValue":0}],"extra_form_data":{},"dashboards":[629],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
 slice_name: Course Information
 uuid: fa249dda-78da-4ccc-9ef3-39177e6aae0c
 version: 1.0.0

tutoraspects/templates/aspects/build/aspects-superset/openedx-assets/assets/charts/Cumulative_Enrollments_by_Track.yaml

@@ -30,30 +30,7 @@ params:
   markerEnabled: true
   markerSize: 6
   metrics:
-  - aggregate: COUNT_DISTINCT
-    column:
-      advanced_data_type: null
-      certification_details: null
-      certified_by: null
-      column_name: actor_id
-      description: null
-      expression: null
-      filterable: true
-      groupby: true
-      id: 138
-      is_certified: false
-      is_dttm: false
-      python_date_format: null
-      type: String
-      type_generic: 1
-      verbose_name: null
-      warning_markdown: null
-    datasourceWarning: false
-    expressionType: SIMPLE
-    hasCustomLabel: true
-    label: Number of Learners
-    optionName: metric_tpdu0kpjpka_gsgo2so3eld
-    sqlExpression: null
+  - number_of_learners
   only_total: true
   opacity: 0.2
   order_desc: true
@@ -84,15 +61,9 @@ params:
   y_axis_title_margin: 15
   y_axis_title_position: Left
   zoomable: true
-query_context: '{"datasource":{"id":78,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No
-  filter"}],"extras":{"time_grain_sqla":"P1M","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1M","columnType":"BASE_AXIS","sqlExpression":"emission_time","label":"emission_time","expressionType":"SQL"},"enrollment_mode"],"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":138,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Number
-  of Learners","optionName":"metric_tpdu0kpjpka_gsgo2so3eld","sqlExpression":null}],"orderby":[[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":138,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Number
-  of Learners","optionName":"metric_tpdu0kpjpka_gsgo2so3eld","sqlExpression":null},false]],"annotation_layers":[],"row_limit":10000,"series_columns":["enrollment_mode"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_time"],"columns":["enrollment_mode"],"aggregates":{"Number
-  of Learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"cum","options":{"operator":"sum","columns":{"Number
-  of Learners":"Number of Learners"}}},{"operation":"rename","options":{"columns":{"Number
-  of Learners":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"78__table","viz_type":"echarts_area","slice_id":189,"x_axis":"emission_time","time_grain_sqla":"P1M","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":[{"aggregate":"COUNT_DISTINCT","column":{"advanced_data_type":null,"certification_details":null,"certified_by":null,"column_name":"actor_id","description":null,"expression":null,"filterable":true,"groupby":true,"id":138,"is_certified":false,"is_dttm":false,"python_date_format":null,"type":"String","type_generic":1,"verbose_name":null,"warning_markdown":null},"datasourceWarning":false,"expressionType":"SIMPLE","hasCustomLabel":true,"label":"Number
-  of Learners","optionName":"metric_tpdu0kpjpka_gsgo2so3eld","sqlExpression":null}],"groupby":["enrollment_mode"],"adhoc_filters":[{"clause":"WHERE","comparator":"No
-  filter","datasourceWarning":false,"expressionType":"SIMPLE","filterOptionName":"filter_utjjqw9427d_5p5ur7mob2","isExtra":false,"isNew":false,"operator":"TEMPORAL_RANGE","sqlExpression":null,"subject":"emission_time"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"rolling_type":"cumsum","comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","opacity":0.2,"show_value":false,"only_total":true,"show_extra_controls":false,"markerEnabled":true,"markerSize":6,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"~g","truncateXAxis":true,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[16],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
+query_context: '{"datasource":{"id":424,"type":"table"},"force":false,"queries":[{"filters":[{"col":"emission_time","op":"TEMPORAL_RANGE","val":"No
+  filter"}],"extras":{"time_grain_sqla":"P1M","having":"","where":""},"applied_time_extras":{},"columns":[{"timeGrain":"P1M","columnType":"BASE_AXIS","sqlExpression":"emission_time","label":"emission_time","expressionType":"SQL"},"enrollment_mode"],"metrics":["number_of_learners"],"orderby":[["number_of_learners",false]],"annotation_layers":[],"row_limit":10000,"series_columns":["enrollment_mode"],"series_limit":0,"order_desc":true,"url_params":{},"custom_params":{},"custom_form_data":{},"time_offsets":[],"post_processing":[{"operation":"pivot","options":{"index":["emission_time"],"columns":["enrollment_mode"],"aggregates":{"number_of_learners":{"operator":"mean"}},"drop_missing_columns":false}},{"operation":"cum","options":{"operator":"sum","columns":{"number_of_learners":"number_of_learners"}}},{"operation":"rename","options":{"columns":{"number_of_learners":null},"level":0,"inplace":true}},{"operation":"flatten"}]}],"form_data":{"datasource":"424__table","viz_type":"echarts_area","slice_id":439,"x_axis":"emission_time","time_grain_sqla":"P1M","x_axis_sort_asc":true,"x_axis_sort_series":"name","x_axis_sort_series_ascending":true,"metrics":["number_of_learners"],"groupby":["enrollment_mode"],"adhoc_filters":[{"clause":"WHERE","comparator":"No
+  filter","datasourceWarning":false,"expressionType":"SIMPLE","filterOptionName":"filter_utjjqw9427d_5p5ur7mob2","isExtra":false,"isNew":false,"operator":"TEMPORAL_RANGE","sqlExpression":null,"subject":"emission_time"}],"order_desc":true,"row_limit":10000,"truncate_metric":true,"show_empty_columns":true,"rolling_type":"cumsum","comparison_type":"values","annotation_layers":[],"forecastPeriods":10,"forecastInterval":0.8,"x_axis_title_margin":15,"y_axis_title_margin":15,"y_axis_title_position":"Left","sort_series_type":"sum","color_scheme":"supersetColors","seriesType":"line","opacity":0.2,"show_value":false,"only_total":true,"show_extra_controls":false,"markerEnabled":true,"markerSize":6,"zoomable":true,"show_legend":true,"legendType":"scroll","legendOrientation":"top","legendMargin":null,"x_axis_time_format":"smart_date","rich_tooltip":true,"tooltipTimeFormat":"smart_date","y_axis_format":"~g","truncateXAxis":true,"y_axis_bounds":[null,null],"extra_form_data":{},"dashboards":[629],"force":false,"result_format":"json","result_type":"full"},"result_format":"json","result_type":"full"}'
 slice_name: Cumulative Enrollments by Track
 uuid: f207c896-030a-462b-b69f-6416230d50b6
 version: 1.0.0