From 3f8e58ac2e5201aaa014cee8751773b36b99e140 Mon Sep 17 00:00:00 2001 From: VineetBala-AOT <90332175+VineetBala-AOT@users.noreply.github.com> Date: Wed, 1 May 2024 13:23:07 -0700 Subject: [PATCH] Dagster bug fix and python upgrade (#2477) --- .github/workflows/met-etl-cd.yml | 2 +- CHANGELOG.MD | 8 +++++++ met-etl/Dockerfile | 2 +- met-etl/container/dagster/Dockerfile | 2 +- met-etl/container/user_code/Dockerfile | 2 +- met-etl/requirements.txt | 22 +++++++++---------- .../services/ops/survey_etl_service.py | 10 +++++---- 7 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/met-etl-cd.yml b/.github/workflows/met-etl-cd.yml index b19147e92..8efe264c8 100644 --- a/.github/workflows/met-etl-cd.yml +++ b/.github/workflows/met-etl-cd.yml @@ -32,7 +32,7 @@ env: OPENSHIFT_REPOSITORY: ${{ secrets.OPENSHIFT_REPOSITORY }} jobs: - met-cron-cd: + met-etl-cd: runs-on: ubuntu-20.04 if: github.repository == 'bcgov/met-public' diff --git a/CHANGELOG.MD b/CHANGELOG.MD index d3f88867a..1a38c9fd0 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -1,4 +1,12 @@ +## April 30, 2024 + +- **Task** Dagster ETL error [DESENG-599](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-599) + - Upgrading the version of python to 3.12 for dagster user code + - Upgrading the dependencies to support the version of python. + - Fixing a bug on survey etl service. + ## April 29, 2024 + - **Task** Upgrade python to 3.12 [DESENG-466](https://apps.itsm.gov.bc.ca/jira/browse/DESENG-466) - Upgrading the version of python to 3.12 - Upgrading the dependencies to support the version of python. diff --git a/met-etl/Dockerfile b/met-etl/Dockerfile index 8613bbd64..146466639 100644 --- a/met-etl/Dockerfile +++ b/met-etl/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.5-buster +FROM python:3.12.2-bullseye ARG DAGSTER_VERSION=0.19.3 diff --git a/met-etl/container/dagster/Dockerfile b/met-etl/container/dagster/Dockerfile index 425892634..0d484fea6 100644 --- a/met-etl/container/dagster/Dockerfile +++ b/met-etl/container/dagster/Dockerfile @@ -1,7 +1,7 @@ # Dagster libraries to run both dagit and the dagster-daemon. Does not # need to have access to any pipeline code. -FROM python:3.8.5-buster +FROM python:3.12.2-bullseye RUN pip install \ dagster \ diff --git a/met-etl/container/user_code/Dockerfile b/met-etl/container/user_code/Dockerfile index 2d25f0980..31580745c 100644 --- a/met-etl/container/user_code/Dockerfile +++ b/met-etl/container/user_code/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.8.5-buster +FROM python:3.12.2-bullseye ARG DAGSTER_VERSION=0.19.3 diff --git a/met-etl/requirements.txt b/met-etl/requirements.txt index 52c83b2b2..114087b82 100644 --- a/met-etl/requirements.txt +++ b/met-etl/requirements.txt @@ -4,17 +4,17 @@ dagster-docker Flask-Cors==3.0.10 Flask-Migrate==2.7.0 Flask-Moment==1.0.5 -Flask-SQLAlchemy==2.5.1 -Flask==2.2.3 -SQLAlchemy-Utils==0.38.3 -SQLAlchemy==1.3.24 +Flask-SQLAlchemy==3.0.5 +Flask==2.2.5 +SQLAlchemy-Utils==0.41.2 +SQLAlchemy==1.4.52 Werkzeug==2.3.8 -flask-marshmallow==0.11.0 -flask-restx==0.5.1 -marshmallow-sqlalchemy==0.25.0 -marshmallow==3.18.0 -packaging==21.3 -python-dotenv==0.21.0 -pytz==2022.5 +flask-marshmallow==1.0.0 +flask-restx==1.3.0 +marshmallow-sqlalchemy==1.0.0 +marshmallow==3.21.1 +packaging==24.0 +python-dotenv==1.0.1 +pytz==2024.1 -e git+https://github.com/bcgov/met-public.git#egg=met-api&subdirectory=met-api -e git+https://github.com/bcgov/met-public.git#egg=analytics-api&subdirectory=analytics-api diff --git a/met-etl/src/etl_project/services/ops/survey_etl_service.py b/met-etl/src/etl_project/services/ops/survey_etl_service.py index d0ab4085f..1923026f3 100644 --- a/met-etl/src/etl_project/services/ops/survey_etl_service.py +++ b/met-etl/src/etl_project/services/ops/survey_etl_service.py @@ -142,10 +142,11 @@ def extract_survey_components(context, session, survey, survey_new_runcycleid, f if has_valid_question_type: etl_survey = session.query(EtlSurveyModel.id).filter(EtlSurveyModel.source_survey_id == survey.id, EtlSurveyModel.is_active == True) - for survey_id in etl_survey: + etl_survey_ids = [survey_id[0] for survey_id in etl_survey] + for survey_id in etl_survey_ids: position = _do_etl_survey_inputs(session, survey_id, component, component_type, survey_new_runcycleid, position) - _load_available_response_option(context, session, survey_id, component, component_type, + _load_available_response_option(session, survey_id, component, component_type, survey_new_runcycleid) return position @@ -160,7 +161,8 @@ def _refresh_questions_and_available_option_status(session, source_survey_id): deactive_flag = {'is_active': False} - for survey_id in etl_survey_model: + etl_survey_ids = [survey_id[0] for survey_id in etl_survey_model] + for survey_id in etl_survey_ids: session.query(EtlRequestTypeOption).filter(EtlRequestTypeOption.survey_id == survey_id).update(deactive_flag) session.query(EtlAvailableResponseOption).filter( EtlAvailableResponseOption.survey_id == survey_id).update(deactive_flag) @@ -221,7 +223,7 @@ def _do_etl_survey_inputs(session, survey_id, component, component_type, survey_ # load data to table available response option -def _load_available_response_option(context, session, survey_id, component, component_type, survey_new_runcycleid): +def _load_available_response_option(session, survey_id, component, component_type, survey_new_runcycleid): if component_type == FormIoComponentType.SURVEY.value: _load_survey_available_response(session, component, survey_id, survey_new_runcycleid)