Dagster bug fix and python upgrade (#2477)

bcgov · May 1, 2024 · 3f8e58a · 3f8e58a
1 parent 7617137
commit 3f8e58a
Show file tree

Hide file tree

Showing 7 changed files with 29 additions and 19 deletions.
diff --git a/.github/workflows/met-etl-cd.yml b/.github/workflows/met-etl-cd.yml
@@ -32,7 +32,7 @@ env:
   OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT_REPOSITORY }}
 
 jobs:
-  met-cron-cd:
+  met-etl-cd:
     runs-on: ubuntu-20.04
 
     if: github.repository == 'bcgov/met-public'

diff --git a/CHANGELOG.MD b/CHANGELOG.MD
@@ -1,4 +1,12 @@
+## April 30, 2024
+
+- **Task** Dagster ETL error [DESENG-599](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-599)
+  - Upgrading the version of python to 3.12 for dagster user code
+  - Upgrading the dependencies to support the version of python.
+  - Fixing a bug on survey etl service.
+
 ## April 29, 2024
+
 - **Task** Upgrade python to 3.12 [DESENG-466](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-466)
   - Upgrading the version of python to 3.12
   - Upgrading the dependencies to support the version of python.

diff --git a/met-etl/Dockerfile b/met-etl/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 ARG DAGSTER_VERSION=0.19.3
 

diff --git a/met-etl/container/dagster/Dockerfile b/met-etl/container/dagster/Dockerfile
@@ -1,7 +1,7 @@
 # Dagster libraries to run both dagit and the dagster-daemon. Does not
 # need to have access to any pipeline code.
 
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 RUN pip install \
     dagster \

diff --git a/met-etl/container/user_code/Dockerfile b/met-etl/container/user_code/Dockerfile
@@ -1,4 +1,4 @@
-FROM python:3.8.5-buster
+FROM python:3.12.2-bullseye
 
 ARG DAGSTER_VERSION=0.19.3
 

diff --git a/met-etl/requirements.txt b/met-etl/requirements.txt
@@ -4,17 +4,17 @@ dagster-docker
 Flask-Cors==3.0.10
 Flask-Migrate==2.7.0
 Flask-Moment==1.0.5
-Flask-SQLAlchemy==2.5.1
-Flask==2.2.3
-SQLAlchemy-Utils==0.38.3
-SQLAlchemy==1.3.24
+Flask-SQLAlchemy==3.0.5
+Flask==2.2.5
+SQLAlchemy-Utils==0.41.2
+SQLAlchemy==1.4.52
 Werkzeug==2.3.8
-flask-marshmallow==0.11.0
-flask-restx==0.5.1
-marshmallow-sqlalchemy==0.25.0
-marshmallow==3.18.0
-packaging==21.3
-python-dotenv==0.21.0
-pytz==2022.5
+flask-marshmallow==1.0.0
+flask-restx==1.3.0
+marshmallow-sqlalchemy==1.0.0
+marshmallow==3.21.1
+packaging==24.0
+python-dotenv==1.0.1
+pytz==2024.1
 -e git+https://github.com/bcgov/met-public.git#egg=met-api&subdirectory=met-api
 -e git+https://github.com/bcgov/met-public.git#egg=analytics-api&subdirectory=analytics-api
diff --git a/met-etl/src/etl_project/services/ops/survey_etl_service.py b/met-etl/src/etl_project/services/ops/survey_etl_service.py
@@ -142,10 +142,11 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f
         if has_valid_question_type:
             etl_survey = session.query(EtlSurveyModel.id).filter(EtlSurveyModel.source_survey_id == survey.id,
                                                                  EtlSurveyModel.is_active == True)
-            for survey_id in etl_survey:
+            etl_survey_ids = [survey_id[0] for survey_id in etl_survey]
+            for survey_id in etl_survey_ids:
                 position = _do_etl_survey_inputs(session, survey_id, component, component_type,
                                                  survey_new_runcycleid, position)
-                _load_available_response_option(context, session, survey_id, component, component_type,
+                _load_available_response_option(session, survey_id, component, component_type,
                                                 survey_new_runcycleid)
 
     return position
@@ -160,7 +161,8 @@ def _refresh_questions_and_available_option_status(session, source_survey_id):
 
     deactive_flag = {'is_active': False}
 
-    for survey_id in etl_survey_model:
+    etl_survey_ids = [survey_id[0] for survey_id in etl_survey_model]
+    for survey_id in etl_survey_ids:
         session.query(EtlRequestTypeOption).filter(EtlRequestTypeOption.survey_id == survey_id).update(deactive_flag)
         session.query(EtlAvailableResponseOption).filter(
             EtlAvailableResponseOption.survey_id == survey_id).update(deactive_flag)
@@ -221,7 +223,7 @@ def _do_etl_survey_inputs(session, survey_id, component, component_type, survey_
 
 
 # load data to table available response option
-def _load_available_response_option(context, session, survey_id, component, component_type, survey_new_runcycleid):
+def _load_available_response_option(session, survey_id, component, component_type, survey_new_runcycleid):
 
     if component_type == FormIoComponentType.SURVEY.value:
         _load_survey_available_response(session, component, survey_id, survey_new_runcycleid)