diff --git a/.github/workflows/analytics-api-ci.yml b/.github/workflows/analytics-api-ci.yml index daa8491c0..a18e958d9 100644 --- a/.github/workflows/analytics-api-ci.yml +++ b/.github/workflows/analytics-api-ci.yml @@ -67,6 +67,8 @@ jobs: KEYCLOAK_TEST_BASE_URL: "http://localhost:8081" KEYCLOAK_TEST_REALMNAME: "demo" USE_TEST_KEYCLOAK_DOCKER: "YES" + + SQLALCHEMY_DATABASE_URI: "postgresql://postgres:postgres@localhost:5432/postgres" runs-on: ubuntu-20.04 diff --git a/CHANGELOG.MD b/CHANGELOG.MD index a1cdd0b77..361f52f4a 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,3 +1,9 @@ +## February 27, 2024 +- **Task**Enhance analytics api for Improved Readability and Maintainability [DESENG-492](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-492) + - Refactor analytics-api config to harmonize its structure and conventions with met-api. + - Ensure the sample.env file maintains consistent formatting. + - Adjusted the component_id column size in the comment table of the met-api to resolve an error encountered during user submissions. + ## February 26, 2024 - **Task**Models for dynamic engagement pages [DESENG-500](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-500) - Implemented endpoints for dynamic engagement pages, including summary and custom sections. diff --git a/analytics-api/requirements.txt b/analytics-api/requirements.txt index b564550d9..0b7275ba3 100644 --- a/analytics-api/requirements.txt +++ b/analytics-api/requirements.txt @@ -5,12 +5,12 @@ Flask-Migrate==2.7.0 Flask-Moment==1.0.5 Flask-SQLAlchemy==2.5.1 Flask-Script==2.0.5 -Flask==2.2.3 +Flask==2.2.5 Jinja2==3.0.3 Mako==1.2.4 MarkupSafe==2.1.2 SQLAlchemy-Utils==0.40.0 -SQLAlchemy==1.3.24 +SQLAlchemy==1.4.17 Werkzeug==2.2.3 alembic==1.10.3 aniso8601==9.0.1 @@ -23,7 +23,7 @@ charset-normalizer==3.1.0 click==8.1.3 ecdsa==0.18.0 flask-jwt-oidc==0.3.0 -flask-marshmallow==0.11.0 +flask-marshmallow==0.14.0 flask-restx==1.1.0 gunicorn==20.1.0 idna==3.4 diff --git a/analytics-api/sample.env b/analytics-api/sample.env index 1d8c6cea0..e9cf3619c 100644 --- a/analytics-api/sample.env +++ b/analytics-api/sample.env @@ -1,18 +1,46 @@ +# GDX MET Analytics API Configuration +# For more information on these values, please see the documentation +# or analytics-api/src/analytics-api/config.py + +# Changes Flask's run mode and the set of env vars are used to configure the app. You should not need to change this here. FLASK_ENV=development -# local db variables -DATABASE_USERNAME=analytics -DATABASE_PASSWORD=analytics -DATABASE_NAME=met -DATABASE_HOST=localhost -DATABASE_PORT=5432 - -JWT_OIDC_WELL_KNOWN_CONFIG=https://localhost:8080/auth/realms/met/.well-known/openid-configuration -JWT_OIDC_AUDIENCE=account -JWT_OIDC_ISSUER=https://localhost:8080/auth/realms/met -JWT_OIDC_ALGORITHMS=RS256 -JWT_OIDC_JWKS_URI=https://localhost:8080/auth/realms/met/protocol/openid-connect/certs -JWT_OIDC_CACHING_ENABLED=True -JWT_OIDC_JWKS_CACHE_TIMEOUT=3000000 - -CORS_ORIGIN=http://localhost:3000,http://localhost:5000 \ No newline at end of file +USE_DEBUG=True # Enable a dev-friendly debug mode +TESTING= # Handle errors normally (False) or raise exceptions (True) + +# CORS Settings +CORS_ORIGINS=http://localhost:3000,http://localhost:5000 + +# Miscellaneous Settings +SECRET_KEY="" # For Flask sessions. If unset, this value is randomized + +# Database Configuration +DATABASE_HOST="localhost" +DATABASE_PORT="5432" +DATABASE_USERNAME="postgres" +DATABASE_PASSWORD="postgres" +DATABASE_NAME="met" +#Default: set from above settings (this overrides them) +SQLALCHEMY_ECHO= +SQLALCHEMY_TRACK_MODIFICATIONS= + +# Keycloak configuration. +KEYCLOAK_BASE_URL="" # auth-server-url +KEYCLOAK_REALMNAME="" # realm + +# JWT OIDC configuration for authentication +JWT_OIDC_AUDIENCE="" # resource +JWT_OIDC_ISSUER="" # default: constructed from base url and realm name +JWT_OIDC_WELL_KNOWN_CONFIG="" # default: constructed from issuer +JWT_OIDC_JWKS_URI="" # default: constructed from issuer +JWT_OIDC_ROLE_CLAIM=client_roles # Keycloak schema +JWT_OIDC_CACHING_ENABLED=true # Enable caching of JWKS. +JWT_OIDC_JWKS_CACHE_TIMEOUT=300 # Timeout for JWKS cache in seconds. + +# Test database settings +# If unset, uses the same settings as the main database +DATABASE_TEST_USERNAME= +DATABASE_TEST_PASSWORD= +DATABASE_TEST_NAME= +DATABASE_TEST_HOST= +DATABASE_TEST_PORT= \ No newline at end of file diff --git a/analytics-api/src/analytics_api/__init__.py b/analytics-api/src/analytics_api/__init__.py index 5079445e2..27e68c5ff 100644 --- a/analytics-api/src/analytics_api/__init__.py +++ b/analytics-api/src/analytics_api/__init__.py @@ -37,7 +37,7 @@ def create_app(run_mode=os.getenv('FLASK_ENV', 'development')): # All configuration are in config file app.config.from_object(get_named_config(run_mode)) - CORS(app, supports_credentials=True) + CORS(app, origins=app.config['CORS_ORIGINS'], supports_credentials=True) # Register blueprints app.register_blueprint(API_BLUEPRINT) @@ -75,8 +75,21 @@ def set_secure_headers(response): def setup_jwt_manager(app_context, jwt_manager): """Use flask app to configure the JWTManager to work for a particular Realm.""" - def get_roles(a_dict): - return a_dict['realm_access']['roles'] # pragma: no cover + def get_roles(token_info) -> list: + """ + Consumes a token_info dictionary and returns a list of roles. + + Uses a configurable path to the roles in the token_info dictionary. + """ + role_access_path = app_context.config['JWT_CONFIG']['ROLE_CLAIM'] + for key in role_access_path.split('.'): + token_info = token_info.get(key, None) + if token_info is None: + app_context.logger.warning('Unable to find role in token_info. ' + 'Please check your JWT_ROLE_CALLBACK ' + 'configuration.') + return [] + return token_info app_context.config['JWT_ROLE_CALLBACK'] = get_roles jwt_manager.init_app(app_context) diff --git a/analytics-api/src/analytics_api/auth.py b/analytics-api/src/analytics_api/auth.py index 4660aecd4..2182d26a4 100644 --- a/analytics-api/src/analytics_api/auth.py +++ b/analytics-api/src/analytics_api/auth.py @@ -18,19 +18,23 @@ from flask_jwt_oidc import JwtManager from flask_jwt_oidc.exceptions import AuthError -jwt = ( - JwtManager() -) # pylint: disable=invalid-name; lower case name as used by convention in most Flask apps +auth_methods = { # for swagger documentation + 'apikey': { + 'type': 'apiKey', + 'in': 'header', + 'name': 'Authorization' + } +} -class Auth: # pylint: disable=too-few-public-methods +class Auth(JwtManager): # pylint: disable=too-few-public-methods """Extending JwtManager to include additional functionalities.""" @classmethod def require(cls, f): """Validate the Bearer Token.""" - @jwt.requires_auth + @auth.requires_auth @wraps(f) def decorated(*args, **kwargs): g.authorization_header = request.headers.get('Authorization', None) @@ -59,6 +63,6 @@ def decorated(*args, **kwargs): return decorated -auth = ( +jwt = auth = ( Auth() ) diff --git a/analytics-api/src/analytics_api/config.py b/analytics-api/src/analytics_api/config.py index ddebfe23b..a07d0b4bf 100644 --- a/analytics-api/src/analytics_api/config.py +++ b/analytics-api/src/analytics_api/config.py @@ -11,25 +11,31 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""All of the configuration for the service is captured here. +""" +All the configuration for MET's Analytics API. -All items are loaded, -or have Constants defined here that are loaded into the Flask configuration. -All modules and lookups get their configuration from the Flask config, -rather than reading environment variables directly or by accessing this configuration directly. +Wherever possible, the configuration is loaded from the environment. The aim is +to have this be the "single source of truth" for configuration in the API, +wherever feasible. If you are adding a setting or config option that cannot be +configured in a user-facing GUI, please make sure it loads its value from here, +and create an entry for it in the sample .env file. """ import os -import sys from typing import Union from dotenv import find_dotenv, load_dotenv -# this will load all the envars from a .env file located in the project root (api) +from analytics_api.utils.util import is_truthy + +# Search in increasingly higher folders for a .env file, then load it, +# appending any variables we find to the current environment. load_dotenv(find_dotenv()) +# remove all env variables with no text (allows for entries to be unset easily) +os.environ = {k: v for k, v in os.environ.items() if v} -def get_named_config(environment: Union[str, None]) -> '_Config': +def get_named_config(environment: Union[str, None]) -> 'Config': """ Retrieve a configuration object by name. Used by the Flask app factory. @@ -46,77 +52,141 @@ def get_named_config(environment: Union[str, None]) -> '_Config': } try: print(f'Loading configuration: {environment}...') - return config_mapping[environment]() + return config_mapping.get(environment or 'production', ProdConfig)() except KeyError as e: raise KeyError(f'Configuration "{environment}" not found.') from e -class _Config(): # pylint: disable=too-few-public-methods - """Base class configuration that should set reasonable defaults for all the other configurations.""" +def env_truthy(env_var, default: Union[bool, str] = False): + """ + Return True if the environment variable is set to a truthy value. + + Accepts a default value, which is returned if the environment variable is + not set. + """ + return is_truthy(os.getenv(env_var, str(default))) - PROJECT_ROOT = os.path.abspath(os.path.dirname(__file__)) - SECRET_KEY = 'a secret' +class Config(): # pylint: disable=too-few-public-methods + """Base class configuration that should set reasonable defaults for all the other configurations.""" - TESTING = False - DEBUG = False + def __init__(self) -> None: + """ + Initialize the configuration object. + + Performs more advanced configuration logic that is not possible + in the normal class definition. + """ + # If extending this class, call super().__init__() in your constructor. + print(f'SQLAlchemy URL: {self.SQLALCHEMY_DATABASE_URI}') + + # apply configs to _Config in the format that flask_jwt_oidc expects + # this flattens the JWT_CONFIG dict into individual attributes + for key, value in self.JWT_CONFIG.items(): + setattr(self, f'JWT_OIDC_{key}', value) + + # Enable live reload and interactive API debugger for developers + os.environ['FLASK_DEBUG'] = str(self.USE_DEBUG) + + @property + # pylint: disable=invalid-name + def SQLALCHEMY_DATABASE_URI(self) -> str: # noqa + """ + Dynamically fetch the SQLAlchemy Database URI based on the DB config. + + This avoids having to redefine the URI after setting the DB access + credentials in subclasses. Can be overridden by env variables. + """ + return os.environ.get( + 'SQLALCHEMY_DATABASE_URI', + f'postgresql://' + f'{self.DB_CONFIG.get("USER")}:{self.DB_CONFIG.get("PASSWORD")}@' + f'{self.DB_CONFIG.get("HOST")}:{self.DB_CONFIG.get("PORT")}/' + f'{self.DB_CONFIG.get("NAME")}' + ) + + # If enabled, Exceptions are propagated up, instead of being handled + # by the the app’s error handlers. Enable this for tests. + TESTING = env_truthy('FLASK_TESTING', default=False) + + # If enabled, the interactive debugger will be shown for any + # unhandled Exceptions, and the server will be reloaded when code changes. + USE_DEBUG = env_truthy('FLASK_DEBUG', default=False) + + # SQLAlchemy settings + # Echoes the SQL queries generated - useful for debugging + SQLALCHEMY_ECHO = env_truthy('SQLALCHEMY_ECHO') + # Disable modification tracking for performance + SQLALCHEMY_TRACK_MODIFICATIONS = env_truthy('SQLALCHEMY_TRACK_MODIFICATIONS') + + # Used for session management. Randomized by default for security, but + # should be set to a fixed value in production to avoid invalidating sessions. + SECRET_KEY = os.getenv('SECRET_KEY', os.urandom(24)) + + # PostgreSQL configuration + DB_CONFIG = DB = { + 'USER': os.getenv('DATABASE_USERNAME', ''), + 'PASSWORD': os.getenv('DATABASE_PASSWORD', ''), + 'NAME': os.getenv('DATABASE_NAME', ''), + 'HOST': os.getenv('DATABASE_HOST', ''), + 'PORT': os.getenv('DATABASE_PORT', '5432'), + } - # POSTGRESQL - DB_USER = os.getenv('DATABASE_USERNAME', '') - DB_PASSWORD = os.getenv('DATABASE_PASSWORD', '') - DB_NAME = os.getenv('DATABASE_NAME', '') - DB_HOST = os.getenv('DATABASE_HOST', '') - DB_PORT = os.getenv('DATABASE_PORT', '5432') - SQLALCHEMY_DATABASE_URI = f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}' - SQLALCHEMY_ECHO = False - SQLALCHEMY_TRACK_MODIFICATIONS = False + # Keycloak configuration + KEYCLOAK_CONFIG = KC = { + 'BASE_URL': os.getenv('KEYCLOAK_BASE_URL', ''), + 'REALMNAME': os.getenv('KEYCLOAK_REALMNAME', 'standard'), + } - # JWT_OIDC Settings - JWT_OIDC_WELL_KNOWN_CONFIG = os.getenv('JWT_OIDC_WELL_KNOWN_CONFIG') - JWT_OIDC_ALGORITHMS = os.getenv('JWT_OIDC_ALGORITHMS', 'RS256') - JWT_OIDC_JWKS_URI = os.getenv('JWT_OIDC_JWKS_URI') - JWT_OIDC_ISSUER = os.getenv('JWT_OIDC_ISSUER') - JWT_OIDC_AUDIENCE = os.getenv('JWT_OIDC_AUDIENCE', 'account') - JWT_OIDC_CACHING_ENABLED = os.getenv('JWT_OIDC_CACHING_ENABLED', 'True') - JWT_OIDC_JWKS_CACHE_TIMEOUT = 300 + # JWT OIDC Settings (for Keycloak) + JWT_CONFIG = JWT = { + 'ISSUER': ( + _issuer := os.getenv( + 'JWT_OIDC_ISSUER', + f'{KC["BASE_URL"]}/realms/{KC["REALMNAME"]}' + )), + 'WELL_KNOWN_CONFIG': os.getenv( + 'JWT_OIDC_WELL_KNOWN_CONFIG', + f'{_issuer}/.well-known/openid-configuration', + ), + 'JWKS_URI': os.getenv('JWT_OIDC_JWKS_URI', f'{_issuer}/protocol/openid-connect/certs'), + 'ALGORITHMS': os.getenv('JWT_OIDC_ALGORITHMS', 'RS256'), + 'AUDIENCE': os.getenv('JWT_OIDC_AUDIENCE', 'account'), + 'CACHING_ENABLED': str(env_truthy('JWT_OIDC_CACHING_ENABLED', True)), + 'JWKS_CACHE_TIMEOUT': int(os.getenv('JWT_OIDC_JWKS_CACHE_TIMEOUT', '300')), + 'ROLE_CLAIM': os.getenv('JWT_OIDC_ROLE_CLAIM', 'client_roles'), + } - # default tenant configs ; Set to EAO for now.Overwrite using openshift variables - DEFAULT_TENANT_SHORT_NAME = os.getenv('DEFAULT_TENANT_SHORT_NAME', 'GDX') - DEFAULT_TENANT_NAME = os.getenv('DEFAULT_TENANT_NAME', 'Environment Assessment Office') - DEFAULT_TENANT_DESCRIPTION = os.getenv('DEFAULT_TENANT_DESCRIPTION', 'Environment Assessment Office') + # CORS settings + CORS_ORIGINS = os.getenv('CORS_ORIGINS', '').split(',') -class DevConfig(_Config): # pylint: disable=too-few-public-methods +class DevConfig(Config): # pylint: disable=too-few-public-methods """Dev Config.""" - TESTING = False - DEBUG = True - print(f'SQLAlchemy URL (DevConfig): {_Config.SQLALCHEMY_DATABASE_URI}') + # Default to using the debugger for development + USE_DEBUG = env_truthy('USE_DEBUG', True) -class TestConfig(_Config): # pylint: disable=too-few-public-methods +class TestConfig(Config): # pylint: disable=too-few-public-methods """In support of testing only.used by the py.test suite.""" - DEBUG = True - TESTING = True - # POSTGRESQL - DB_USER = os.getenv('DATABASE_TEST_USERNAME', 'postgres') - DB_PASSWORD = os.getenv('DATABASE_TEST_PASSWORD', 'postgres') - DB_NAME = os.getenv('DATABASE_TEST_NAME', 'postgres') - DB_HOST = os.getenv('DATABASE_TEST_HOST', 'localhost') - DB_PORT = os.getenv('DATABASE_TEST_PORT', '5432') - SQLALCHEMY_DATABASE_URI = os.getenv('DATABASE_TEST_URL', - f'postgresql://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{int(DB_PORT)}/{DB_NAME}') + # Propagate exceptions up to the test runner + TESTING = env_truthy('TESTING', default=True) + # explicitly disable the debugger; we want the tests to fail if an + # unhandled exception occurs + USE_DEBUG = False -class ProdConfig(_Config): # pylint: disable=too-few-public-methods - """Production Config.""" - - SECRET_KEY = os.getenv('SECRET_KEY', None) + # Override the DB config to use the test database, if one is configured + DB_CONFIG = { + 'USER': os.getenv('DATABASE_TEST_USERNAME', Config.DB.get('USER')), + 'PASSWORD': os.getenv('DATABASE_TEST_PASSWORD', Config.DB.get('PASSWORD')), + 'NAME': os.getenv('DATABASE_TEST_NAME', Config.DB.get('NAME')), + 'HOST': os.getenv('DATABASE_TEST_HOST', Config.DB.get('HOST')), + 'PORT': os.getenv('DATABASE_TEST_PORT', Config.DB.get('PORT')), + } - if not SECRET_KEY: - SECRET_KEY = os.urandom(24) - print('WARNING: SECRET_KEY being set as a one-shot', file=sys.stderr) - TESTING = False - DEBUG = False +class ProdConfig(Config): # pylint: disable=too-few-public-methods + """Production Config.""" diff --git a/analytics-api/src/analytics_api/models/request_type_option.py b/analytics-api/src/analytics_api/models/request_type_option.py index 499291d11..4b8ee2c56 100644 --- a/analytics-api/src/analytics_api/models/request_type_option.py +++ b/analytics-api/src/analytics_api/models/request_type_option.py @@ -87,8 +87,7 @@ def get_survey_result( .outerjoin(available_response, survey_question.c.key == available_response.c.request_key) .outerjoin(survey_response, (available_response.c.value == survey_response.c.value) & - (available_response.c.request_key == survey_response.c.request_key), - full=True) + (available_response.c.request_key == survey_response.c.request_key)) .group_by(survey_question.c.position, survey_question.c.label)) return survey_result.all() diff --git a/analytics-api/src/analytics_api/utils/user_context.py b/analytics-api/src/analytics_api/utils/user_context.py index a4811cf65..bd731d0de 100644 --- a/analytics-api/src/analytics_api/utils/user_context.py +++ b/analytics-api/src/analytics_api/utils/user_context.py @@ -16,7 +16,7 @@ import functools from typing import Dict -from flask import g, request +from flask import current_app, g, request from analytics_api.utils.roles import Role @@ -37,7 +37,7 @@ def __init__(self): self._first_name: str = token_info.get('firstname', None) self._last_name: str = token_info.get('lastname', None) self._bearer_token: str = _get_token() - self._roles: list = token_info.get('realm_access', None).get('roles', []) if 'realm_access' in token_info \ + self._roles: list = current_app.config['JWT_ROLE_CALLBACK'](token_info) if 'client_roles' in token_info \ else [] self._sub: str = token_info.get('sub', None) self._name: str = f"{token_info.get('firstname', None)} {token_info.get('lastname', None)}" diff --git a/analytics-api/src/analytics_api/utils/util.py b/analytics-api/src/analytics_api/utils/util.py index dba5e3a6a..4114e86a5 100644 --- a/analytics-api/src/analytics_api/utils/util.py +++ b/analytics-api/src/analytics_api/utils/util.py @@ -19,7 +19,6 @@ import base64 import os -import re import urllib from enum import Enum @@ -30,12 +29,18 @@ def cors_preflight(methods): def wrapper(f): def options(self, *args, **kwargs): # pylint: disable=unused-argument - return {'Allow': 'GET, DELETE, PUT, POST'}, 200, \ - { - 'Access-Control-Allow-Origin': '*', - 'Access-Control-Allow-Methods': methods, - 'Access-Control-Allow-Headers': 'Authorization, Content-Type, registries-trace-id, ' - 'invitation_token'} + headers = { + 'Allow': 'GET, DELETE, PUT, POST', + 'Access-Control-Allow-Origin': '*', + 'Access-Control-Allow-Methods': methods, + 'Access-Control-Allow-Headers': 'Authorization, Content-Type, ' + 'registries-trace-id, invitation_token' + } + max_age = os.getenv('CORS_MAX_AGE') + if max_age is not None: + headers['Access-Control-Max-Age'] = str(max_age) + + return headers, 200, {} setattr(f, 'options', options) return f @@ -43,14 +48,19 @@ def options(self, *args, **kwargs): # pylint: disable=unused-argument return wrapper +def is_truthy(value: str) -> bool: + """ + Check if a value is truthy or not. + + :param value: The value to check. + :return: True if the value seems affirmative, False otherwise. + """ + return str(value).lower() in ('1', 'true', 'yes', 'y', 'on') + + def allowedorigins(): - """Return allowed origin.""" - _allowedcors = os.getenv('CORS_ORIGIN') - allowedcors = [] - if _allowedcors and ',' in _allowedcors: - for entry in re.split(',', _allowedcors): - allowedcors.append(entry) - return allowedcors + """Return the allowed origins for CORS.""" + return os.getenv('CORS_ORIGINS', '').split(',') def escape_wam_friendly_url(param): diff --git a/met-api/migrations/versions/cec8d0371f42_update_comment_table.py b/met-api/migrations/versions/cec8d0371f42_update_comment_table.py new file mode 100644 index 000000000..ec1b35199 --- /dev/null +++ b/met-api/migrations/versions/cec8d0371f42_update_comment_table.py @@ -0,0 +1,32 @@ +"""update_comment_table + +Revision ID: cec8d0371f42 +Revises: e2625b0d07ab +Create Date: 2024-02-26 16:18:47.917630 + +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = 'cec8d0371f42' +down_revision = 'e2625b0d07ab' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('comment', 'component_id', + existing_type=sa.String(length=10), + type_=sa.String(length=50)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.alter_column('comment', 'component_id', + existing_type=sa.String(length=50), + type_=sa.String(length=10)) + # ### end Alembic commands ###