Updating met repo with all changes on epic engage (#2348)

* Changes to show all survey results to superusers

* removing hard coded values

* fixing linting

* splitting to seperate end points

* Handle no data error for graphs

* adding new nodata component

* adding new email for submission response

* fixing linting and testing

* Upgrades to Issue Tracking Table

* removing try catch

* removing changes for email verification

* updating

* Merging bug fixes from epic engage

* Updating comment service to remove metadata

* fixing met-api linting

* Updating change log

* Updating comment

* fixing linting for analytics

CHANGELOG.MD

@@ -6,6 +6,15 @@ All notable changes to this project will be documented in this file. This projec
 
 > **Feature**: Added the timeline widget. [🎟️DESENG-439](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-439)
 
+## January 9, 2024
+
+- **Task** Improvements from Epic [🎟️DESENG-468](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-468)
+  - Improvements to Survey Result Tracking analytics
+  - New Rejection Email Template for Closed Engagements
+  - Export Format for Proponent updated to be in excel format
+  - Formio Version Update
+  - Enable Survey Editing for Open Engagements
+
 ## December 11, 2023
 
 - **Task** Merge `gdx-sso`, `gdx-dev`, `gdx-main` into `main` [🎟️DESENG-442](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-442)

analytics-api/migrations/versions/812b1f67015a_updating_available_response_options.py

@@ -0,0 +1,30 @@
+"""updating_available_response_options
+
+Revision ID: 812b1f67015a
+Revises: e7fdf769e8ff
+Create Date: 2023-11-30 09:50:55.874798
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '812b1f67015a'
+down_revision = 'e7fdf769e8ff'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('available_response_option', 'request_key', type_=sa.Text())
+    op.alter_column('available_response_option', 'request_id', type_=sa.Text())
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('available_response_option', 'request_key', type_=sa.String(length=100))
+    op.alter_column('available_response_option', 'request_id', type_=sa.String(length=20))
+    # ### end Alembic commands ###

analytics-api/migrations/versions/e7fdf769e8ff_adding_available_response_options.py

@@ -0,0 +1,41 @@
+"""adding_available_response_options
+
+Revision ID: e7fdf769e8ff
+Revises: 3a705c422892
+Create Date: 2023-11-21 12:45:34.871602
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e7fdf769e8ff'
+down_revision = '3a705c422892'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('available_response_option',
+    sa.Column('created_date', sa.DateTime(), nullable=True),
+    sa.Column('updated_date', sa.DateTime(), nullable=True),
+    sa.Column('is_active', sa.Boolean(), nullable=True),
+    sa.Column('id', sa.Integer(), autoincrement=True, nullable=False),
+    sa.Column('participant_id', sa.Integer(), nullable=True),
+    sa.Column('request_key', sa.String(length=100), nullable=False),
+    sa.Column('value', sa.Text(), nullable=True),
+    sa.Column('request_id', sa.String(length=20), nullable=True),
+    sa.Column('survey_id', sa.Integer(), nullable=False),
+    sa.Column('runcycle_id', sa.Integer(), nullable=True),
+    sa.ForeignKeyConstraint(['survey_id'], ['survey.id'], ondelete='CASCADE'),
+    sa.PrimaryKeyConstraint('id', 'request_key')
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('available_response_option')
+    # ### end Alembic commands ###

analytics-api/src/analytics_api/config.py

@@ -22,13 +22,14 @@
 import os
 import sys
 
+from typing import Union
 from dotenv import find_dotenv, load_dotenv
 
 # this will load all the envars from a .env file located in the project root (api)
 load_dotenv(find_dotenv())
 
 
-def get_named_config(environment: 'str | None') -> '_Config':
+def get_named_config(environment: Union[str, None]) -> '_Config':
     """
     Retrieve a configuration object by name. Used by the Flask app factory.
 
@@ -38,16 +39,16 @@ def get_named_config(environment: 'str | None') -> '_Config':
     """
     config_mapping = {
         'development': DevConfig,
-        'default':   ProdConfig,
-        'staging':    ProdConfig,
+        'default': ProdConfig,
+        'staging': ProdConfig,
         'production': ProdConfig,
-        'testing':    TestConfig,
+        'testing': TestConfig,
     }
     try:
         print(f'Loading configuration: {environment}...')
         return config_mapping[environment]()
-    except KeyError:
-        raise KeyError(f'Configuration "{environment}" not found.')
+    except KeyError as e:
+        raise KeyError(f'Configuration "{environment}" not found.') from e
 
 
 class _Config():  # pylint: disable=too-few-public-methods

analytics-api/src/analytics_api/models/available_response_option.py

@@ -0,0 +1,12 @@
+"""available_response_option model class.
+
+Manages the Available Options for a option type questions on a survey
+"""
+from .base_model import BaseModel
+from .response_mixin import ResponseMixin
+
+
+class AvailableResponseOption(BaseModel, ResponseMixin):  # pylint: disable=too-few-public-methods
+    """Definition of the Available Response Options entity."""
+
+    __tablename__ = 'available_response_option'

analytics-api/src/analytics_api/models/request_type_option.py

@@ -4,6 +4,7 @@
 """
 from sqlalchemy import and_, func, or_
 from sqlalchemy.sql.expression import true
+from analytics_api.models.available_response_option import AvailableResponseOption as AvailableResponseOptionModel
 from analytics_api.models.survey import Survey as SurveyModel
 from analytics_api.models.response_type_option import ResponseTypeOption as ResponseTypeOptionModel
 from .base_model import BaseModel
@@ -34,42 +35,76 @@ def get_survey_result(
         if can_view_all_survey_results:
             survey_question = (db.session.query(RequestTypeOption.position.label('position'),
                                                 RequestTypeOption.label.label('label'),
-                                                RequestTypeOption.request_id)
+                                                RequestTypeOption.key)
                                .filter(and_(RequestTypeOption.survey_id.in_(analytics_survey_id),
                                             RequestTypeOption.is_active == true()))
                                .order_by(RequestTypeOption.position)
                                .subquery())
         else:
             survey_question = (db.session.query(RequestTypeOption.position.label('position'),
                                                 RequestTypeOption.label.label('label'),
-                                                RequestTypeOption.request_id)
+                                                RequestTypeOption.key)
                                .filter(and_(RequestTypeOption.survey_id.in_(analytics_survey_id),
                                             RequestTypeOption.is_active == true(),
                                             or_(RequestTypeOption.display == true(),
                                                 RequestTypeOption.display.is_(None))))
                                .order_by(RequestTypeOption.position)
                                .subquery())
 
+        # Get all the available responses for each question within the survey.
+        available_response = (db.session.query(AvailableResponseOptionModel.request_key,
+                                               AvailableResponseOptionModel.value)
+                              .filter(and_(AvailableResponseOptionModel.survey_id.in_(
+                                  analytics_survey_id), AvailableResponseOptionModel.is_active == true()))
+                              .subquery())
         # Get all the survey responses with the counts for each response specific to a survey id which
         # are in active status.
-        survey_response = (db.session.query(ResponseTypeOptionModel.request_id, ResponseTypeOptionModel.value,
-                                            func.count(ResponseTypeOptionModel.request_id).label('response'))
+        survey_response = (db.session.query(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value,
+                                            func.count(ResponseTypeOptionModel.request_key).label('response'))
                            .filter(and_(ResponseTypeOptionModel.survey_id.in_(analytics_survey_id),
                                         ResponseTypeOptionModel.is_active == true()))
-                           .group_by(ResponseTypeOptionModel.request_id, ResponseTypeOptionModel.value)
+                           .group_by(ResponseTypeOptionModel.request_key, ResponseTypeOptionModel.value)
                            .subquery())
 
+        survey_response_exists = db.session.query(survey_response.c.request_key).first()
+        available_response_exists = db.session.query(available_response.c.request_key).first()
+
         # Combine the data fetched above such that the result has a format as below
         # - position: is a unique value for each question which helps to get the order of question on the survey
         # - label: is the the survey question
         # - value: user selected response for each question
         # - count: number of time the same value is selected as a response to each question
-        survey_result = (db.session.query((survey_question.c.position).label('position'),
-                                          (survey_question.c.label).label('question'),
-                                          func.json_agg(func.json_build_object('value', survey_response.c.value,
-                                                                               'count', survey_response.c.response))
-                                          .label('result'))
-                         .join(survey_response, survey_response.c.request_id == survey_question.c.request_id)
-                         .group_by(survey_question.c.position, survey_question.c.label))
 
-        return survey_result.all()
+        # Check if there are records in survey_response and available_response before executing the final query
+        # which fetches all the available responses along with the corresponding responses.
+        if survey_response_exists and available_response_exists:
+            survey_result = (db.session.query((survey_question.c.position).label('position'),
+                                              (survey_question.c.label).label('question'),
+                                              func.json_agg(func.json_build_object(
+                                                  'value', available_response.c.value,
+                                                  'count', func.coalesce(survey_response.c.response, 0)))
+                             .label('result'))
+                             .outerjoin(available_response, survey_question.c.key == available_response.c.request_key)
+                             .outerjoin(survey_response,
+                                        (available_response.c.value == survey_response.c.value) &
+                                        (available_response.c.request_key == survey_response.c.request_key),
+                                        full=True)
+                             .group_by(survey_question.c.position, survey_question.c.label))
+
+            return survey_result.all()
+        # Check if there are records in survey_response before executing the final query which fetches reponses
+        # even if the available_response table is not yet populated.
+        if survey_response_exists:
+            survey_result = (db.session.query((survey_question.c.position).label('position'),
+                                              (survey_question.c.label).label('question'),
+                                              func.json_agg(func.json_build_object('value',
+                                                                                   survey_response.c.value,
+                                                                                   'count',
+                                                                                   survey_response.c.response))
+                             .label('result'))
+                             .join(survey_response, survey_response.c.request_key == survey_question.c.key)
+                             .group_by(survey_question.c.position, survey_question.c.label))
+
+            return survey_result.all()
+
+        return None  # Return None indicating no records

met-api/sample.env

@@ -80,6 +80,7 @@ EMAIL_FROM_ADDRESS="[email protected]"
 SUBSCRIBE_EMAIL_TEMPLATE_ID=
 SUBSCRIBE_EMAIL_SUBJECT=
 REJECTED_EMAIL_TEMPLATE_ID=
+CLOSED_ENGAGEMENT_REJECTED_EMAIL_TEMPLATE_ID=
 REJECTED_EMAIL_SUBJECT=
 VERIFICATION_EMAIL_TEMPLATE_ID=
 VERIFICATION_EMAIL_SUBJECT=

met-api/src/met_api/__init__.py

@@ -130,12 +130,13 @@ def setup_jwt_manager(app_context, jwt_manager):
     def get_roles(token_info):
         """
         Consumes a token_info dictionary and returns a list of roles.
+
         Uses a configurable path to the roles in the token_info dictionary.
         """
         role_access_path = app_context.config['JWT_CONFIG']['ROLE_CLAIM']
         for key in role_access_path.split('.'):
             token_info = token_info.get(key, {})
         return token_info
-        
+
     app_context.config['JWT_ROLE_CALLBACK'] = get_roles
     jwt_manager.init_app(app_context)