From 086931804910478f795a95733b458ae8a0198ca2 Mon Sep 17 00:00:00 2001 From: maskarb Date: Wed, 18 Oct 2023 11:55:59 -0400 Subject: [PATCH 1/4] Upgrade local and ephemeral to pg15 --- .github/postgres/docker-compose.yaml | 2 +- deploy/clowdapp.yaml | 2 +- deploy/kustomize/base/base.yaml | 2 +- dev/containers/postgresql/Dockerfile | 2 +- docker-compose.yml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/postgres/docker-compose.yaml b/.github/postgres/docker-compose.yaml index d677ba011f..4d85c62136 100644 --- a/.github/postgres/docker-compose.yaml +++ b/.github/postgres/docker-compose.yaml @@ -2,7 +2,7 @@ version: '3' services: db: - image: postgres:14 + image: postgres:15 environment: - DATABASE_NAME=${DATABASE_NAME-postgres} - POSTGRES_USER=${DATABASE_USER-postgres} diff --git a/deploy/clowdapp.yaml b/deploy/clowdapp.yaml index c360041461..7e43263e52 100644 --- a/deploy/clowdapp.yaml +++ b/deploy/clowdapp.yaml @@ -4329,7 +4329,7 @@ parameters: value: koku - name: DATABASE_VERSION required: true - value: "14" + value: "15" - name: CLOWDER_ENABLED required: true value: "True" diff --git a/deploy/kustomize/base/base.yaml b/deploy/kustomize/base/base.yaml index ee76e16d6d..3804329e49 100644 --- a/deploy/kustomize/base/base.yaml +++ b/deploy/kustomize/base/base.yaml @@ -216,7 +216,7 @@ parameters: value: koku - name: DATABASE_VERSION required: true - value: "14" + value: "15" - name: CLOWDER_ENABLED required: true value: "True" diff --git a/dev/containers/postgresql/Dockerfile b/dev/containers/postgresql/Dockerfile index 89ea6a1a11..8ebcd3a29a 100644 --- a/dev/containers/postgresql/Dockerfile +++ b/dev/containers/postgresql/Dockerfile @@ -1,4 +1,4 @@ -from postgres:14 +from postgres:15 ARG USER_ID=999 ARG GROUP_ID=999 diff --git a/docker-compose.yml b/docker-compose.yml index b26f47dfd5..9e636a830e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -444,7 +444,7 @@ services: db: container_name: koku-db - image: koku-db:14 + image: koku-db:15 build: context: dev/containers/postgresql args: From eea9a78fd89fe9a4d03d643d540a445f2b58dc30 Mon Sep 17 00:00:00 2001 From: maskarb Date: Thu, 19 Oct 2023 11:11:34 -0400 Subject: [PATCH 2/4] convert sql string to jinja template --- koku/masu/database/report_db_accessor_base.py | 38 +++++++++++-------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/koku/masu/database/report_db_accessor_base.py b/koku/masu/database/report_db_accessor_base.py index 0055d1528f..2c1a0d96ff 100644 --- a/koku/masu/database/report_db_accessor_base.py +++ b/koku/masu/database/report_db_accessor_base.py @@ -287,23 +287,29 @@ def delete_line_item_daily_summary_entries_for_date_range_raw( msg = f"Deleting records from {table} for source {source_uuid} from {start_date} to {end_date}" LOG.info(msg) - sql = f""" - DELETE FROM {self.schema}.{table} - WHERE usage_start >= %(start_date)s::date - AND usage_start <= %(end_date)s::date + filters = filters or {} # convert None -> dict + null_filters = null_filters or {} # convert None -> dict + + tmp_sql = """ +DELETE FROM {{schema | sqlsafe}}.{{table | sqlsafe}} + WHERE usage_start >= {{start_date}} + AND usage_start <= {{end_date}} + {% for k, v in filters.items() -%} + AND {{ k | sqlsafe }} = {{ v }} + {% endfor -%} + {% for k, v in null_filters.items() -%} + AND {{ k | sqlsafe }} {{ v | sqlsafe }} + {% endfor -%} """ - if filters: - filter_list = [f"AND {k} = %({k})s" for k in filters] - sql += "\n".join(filter_list) - else: - filters = {} - if null_filters: - filter_list = [f"AND {column} {null_filter}" for column, null_filter in null_filters.items()] - sql += "\n".join(filter_list) - filters["start_date"] = start_date - filters["end_date"] = end_date - - self._execute_raw_sql_query(table, sql, start_date, end_date, bind_params=filters, operation="DELETE") + tmp_sql_params = { + "schema": self.schema, + "table": table, + "start_date": start_date, + "end_date": end_date, + "filters": filters, + "null_filters": null_filters, + } + self._prepare_and_execute_raw_sql_query(table, tmp_sql, tmp_sql_params, operation="DELETE") def truncate_partition(self, partition_name): """Issue a TRUNCATE command on a specific partition of a table""" From 0d8e08f3ca6cdaafeb08902ff62df5d1c4077b12 Mon Sep 17 00:00:00 2001 From: maskarb Date: Thu, 19 Oct 2023 11:35:39 -0400 Subject: [PATCH 3/4] compare versions instead of decimal (1.10 is actually greater than 1.8) --- koku/masu/api/db_performance/db_performance.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/koku/masu/api/db_performance/db_performance.py b/koku/masu/api/db_performance/db_performance.py index cc644596b5..cbfbe6573c 100644 --- a/koku/masu/api/db_performance/db_performance.py +++ b/koku/masu/api/db_performance/db_performance.py @@ -7,6 +7,7 @@ from decimal import Decimal import psycopg2 +from packaging import version from psycopg2.errors import ProgrammingError from psycopg2.extras import RealDictCursor from sqlparse import format as sql_format @@ -203,7 +204,7 @@ def get_statement_stats(self, dbname, limit=100, offset=None): limit_clause = self._handle_limit(limit, params) offset_clause = self._handle_offset(offset, params) - col_name_sep = "_" if Decimal(pss_ver) < Decimal("1.8") else "_exec_" + col_name_sep = "_" if version.parse(pss_ver) < version.parse("1.8") else "_exec_" params["dbname"] = dbname sql = f""" -- STATEMENT STATISTICS From 66af57f9437e8112e902164c8855eb71872a5fed Mon Sep 17 00:00:00 2001 From: maskarb Date: Thu, 19 Oct 2023 11:46:49 -0400 Subject: [PATCH 4/4] since importing packaging, explicitly add to pipfile --- Pipfile | 1 + Pipfile.lock | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Pipfile b/Pipfile index c016e0c6a7..44f24e5e71 100644 --- a/Pipfile +++ b/Pipfile @@ -59,6 +59,7 @@ whitenoise = ">=5.0" # Constraints on indirect dependencies jinja2 = "<3" # jinjasql is not compatible with Jinja >=3 https://github.com/sripathikrishnan/jinjasql/pull/53 markupsafe = "<2.1" # jinja2 <3 tries to import markupsafe.soft_unicode which was removed in markupsafe 2.1 +packaging = "*" [dev-packages] argh = ">=0.26.2" diff --git a/Pipfile.lock b/Pipfile.lock index b14e5a1e65..0f1c8f91fb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "12b9a65de36f21d1117234fbeab350c4dfeab1a49ebc679208b0bbae8bd5cd5a" + "sha256": "79dbd2b8e36bca26aadc6bf19edc3220e84e1f22badda598554e7e04da3b8f5d" }, "pipfile-spec": 6, "requires": {