Test chart data endpoint w/ duckdb

pyproject.toml

@@ -52,6 +52,7 @@ test = [
     "responses",
 ]
 dev = [
+    "duckdb",
     "pre-commit",
     "ruff < 0.9",
     "sqlfluff >= 3.2.5"

src/dashboard/get_chart_data/get_chart_data.py

@@ -52,12 +52,14 @@
     "sameYearOrBefore",
     "sameDayOrAfter",
     "sameWeekOrAfter",
+    "sameMonthOrAfter",
     "sameYearOrAfter",
     "beforeDay",
     "beforeWeek",
     "beforeMonth",
     "beforeYear",
     "afterDay",
+    "afterWeek",
     "afterMonth",
     "afterYear",
     # Boolean filters (one param only)
@@ -140,7 +142,6 @@ def _build_query(query_params: dict, filter_groups: list, path_params: dict) ->
     """
     dp_id = path_params["data_package_id"]
     columns = _get_table_cols(dp_id)
-
     inline_configs = []
     none_configs = []
     for filter_group in filter_groups:
@@ -183,6 +184,7 @@ def _build_query(query_params: dict, filter_groups: list, path_params: dict) ->
         columns.remove(query_params["column"])
     if query_params.get("stratifier") in columns:
         columns.remove(query_params["stratifier"])
+
     with open(pathlib.Path(__file__).parent / "templates/get_chart_data.sql.jinja") as file:
         template = file.read()
         loader = jinja2.FileSystemLoader(pathlib.Path(__file__).parent / "templates/")
@@ -196,6 +198,7 @@ def _build_query(query_params: dict, filter_groups: list, path_params: dict) ->
             coalesce_columns=columns,
             inline_configs=inline_configs,
             none_configs=none_configs,
+            extra_filter_configs=[],  # extra_filter_configs
         )
     return query_str, count_col
 

src/dashboard/get_chart_data/templates/filter_inline.sql.jinja

@@ -90,18 +90,18 @@ date_trunc('month',from_iso8601_timestamp("{{ data }}")) > date_trunc('month',fr
 date_trunc('year',from_iso8601_timestamp("{{ data }}")) > date_trunc('year',from_iso8601_timestamp('{{ bound }}'))
 {#- Boolean filters -#}
 {%- elif filter_type == 'isTrue' -%}
-"{{ data }}" IS TRUE
+"{{ data }}" IS TRUE AND "{{ data }}" IS NOT NULL
 {%- elif filter_type == 'isNotTrue' -%}
-"{{ data }}" IS NOT TRUE
+"{{ data }}" IS NOT TRUE AND "{{ data }}" IS NOT NULL
 {%- elif filter_type == 'isFalse' -%}
-"{{ data }}" IS FALSE
+"{{ data }}" IS FALSE AND "{{ data }}" IS NOT NULL
 {%- elif filter_type == 'isNotFalse' -%}
-"{{ data }}" IS NOT FALSE
+"{{ data }}" IS NOT FALSE AND "{{ data }}" IS NOT NULL
 {#- Null filters -#}
 {%- elif filter_type == 'isNull' -%}
 "{{ data }}" IS NULL
 {%- elif filter_type == 'isNotNull' -%}
-"{{ data }}" IS NOT NULL
+"{{ data }}" IS NOT NULL 
 {#- Numeric filters -#}
 {%- elif filter_type == 'eq'-%}
 "{{ data }}" = {{ bound }}

src/shared/functions.py

@@ -112,10 +112,6 @@ def update_metadata(
     if extra_items is None:
         extra_items = {}
     check_meta_type(meta_type)
-    logger.info(f"### Updating metadata {meta_type}")
-    logger.info(f"{study} {data_package} {version}")
-    logger.info(f"Key: {target} Value: {value}")
-    logger.info(f"Pre-update size: {len(metadata.keys())}")
 
     match meta_type:
         case enums.JsonFilename.TRANSACTIONS.value:
@@ -152,8 +148,6 @@ def update_metadata(
         case _:
             raise ValueError(f"{meta_type} does not have a handler for updates.")
     data_version_metadata.update(extra_items)
-    logger.info(f"Post-update size: {len(metadata.keys())}")
-    logger.info(f"### Updated metadata {meta_type}")
     return metadata
 
 

src/site_upload/cache_api/cache_api.py

@@ -38,13 +38,19 @@ def cache_api_data(s3_client, s3_bucket_name: str, db: str, target: str) -> None
             "study": dp.split("__")[0],
             "name": dp.split("__")[1],
         }
-        versions = column_types[dp_detail["study"]][dp_detail["name"]]
+        studies = column_types.get(dp_detail["study"], {"name": None})
+        versions = studies.get(dp_detail["name"], None)
+        if versions is None:
+            print(f"{dp} not found in column_types")
+            continue
         for version in versions:
+            if version != dp:
+                continue
             dp_dict = {
                 **dp_detail,
                 **versions[version],
-                "version": version,
-                "id": f"{dp_detail['study']}__{dp_detail['name']}__{version}",
+                "version": version.split("__")[2],
+                "id": f"{dp_detail['study']}__{dp_detail['name']}__{version.split('__')[2]}",
             }
             if "__flat" in dp:
                 dp_dict["type"] = "flat"

tests/conftest.py

@@ -22,10 +22,13 @@
   aggregator
 """
 
+import datetime
 import os
+import re
 from unittest import mock
 
 import boto3
+import duckdb
 import pytest
 from moto import mock_athena, mock_s3, mock_sns
 
@@ -155,7 +158,7 @@ def mock_notification():
 
 
 @pytest.fixture
-def mock_db():
+def mock_athena_db():
     """Leaving this unused here for now - there are some low level inconsistencies
     between moto and AWS wrangler w.r.t. how workgroups are mocked out, but we might
     be able to use this in the future/mock AWSwranger below the entrypoint if we are
@@ -176,6 +179,36 @@ def mock_db():
     athena.stop()
 
 
+@pytest.fixture
+def mock_db(tmp_path):
+    def _compat_regexp_like(string: str | None, pattern: str | None) -> bool:
+        match = re.search(pattern, string)
+        return match is not None
+
+    def _compat_from_iso8601_timestamp(
+        value: str | datetime.datetime,
+    ) -> datetime.datetime:
+        if type(value) is str:
+            return datetime.datetime.fromisoformat(value)
+        return value
+
+    db = duckdb.connect(tmp_path / "duck.db")
+    db.create_function(
+        # DuckDB's version is regexp_matches.
+        "regexp_like",
+        _compat_regexp_like,
+        None,
+        duckdb.typing.BOOLEAN,
+    )
+    db.create_function(
+        "from_iso8601_timestamp",
+        _compat_from_iso8601_timestamp,
+        None,
+        duckdb.typing.TIMESTAMP,
+    )
+    yield db
+
+
 def test_mock_bucket():
     s3_client = boto3.client("s3", region_name="us-east-1")
     item = s3_client.list_objects_v2(Bucket=os.environ["TEST_BUCKET"])

tests/dashboard/test_filter_inline.py

@@ -194,28 +194,28 @@
             ["col:isTrue"],
             """
         (
-            "col" IS TRUE
+            "col" IS TRUE AND "col" IS NOT NULL
                     )""",
         ),
         (
             ["col:isNotTrue"],
             """
         (
-            "col" IS NOT TRUE
+            "col" IS NOT TRUE AND "col" IS NOT NULL
                     )""",
         ),
         (
             ["col:isFalse"],
             """
         (
-            "col" IS FALSE
+            "col" IS FALSE AND "col" IS NOT NULL
                     )""",
         ),
         (
             ["col:isNotFalse"],
             """
         (
-            "col" IS NOT FALSE
+            "col" IS NOT FALSE AND "col" IS NOT NULL
                     )""",
         ),
         (

tests/dashboard/test_get_chart_data.py

@@ -401,3 +401,118 @@ def test_handler():
         '"rowCount": 2, "totalCount": 20, "data": [{"rows": [["male", 10], '
         '["female", 10]]}]}'
     )
+
+
+def mock_get_table_cols_results(name):
+    return ["cnt", "nato", "greek", "numeric", "timestamp", "bool"]
+
+
+@pytest.mark.parametrize(
+    "query_params,filter_groups,expected",
+    [
+        # flitering on display column
+        ({"column": "nato"}, ["nato:strEq:alfa"], [("alfa", 50)]),
+        # General check on joins with non-included columns
+        ({"column": "nato"}, ["nato:strEq:alfa,greek:strEq:alpha"], [("alfa", 40)]),
+        ({"column": "nato"}, ["nato:strEq:alfa,greek:strEq:beta"], [("alfa", 10)]),
+        # filtering on non-included columns only
+        ({"column": "nato"}, ["greek:strEq:beta"], [("alfa", 10)]),
+        # checking joins on AND/OR
+        (
+            {"column": "nato"},
+            ["greek:strEq:alpha,numeric:eq:2.2", "greek:strEq:beta,numeric:eq:1.1"],
+            [("alfa", 10), ("alfa", 10)],
+        ),
+        # validating all potential filter types
+        ## strings
+        ({"column": "nato"}, ["nato:strEq:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strContains:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strStartsWith:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strEndsWith:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:matches:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strEqCI:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strContainsCI:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strStartsWithCI:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strEndsWithCI:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:matchesCI:bravo"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotEq:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotContains:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotStartsWith:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotEndsWith:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:notMatches:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotEqCI:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotContainsCI:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotStartsWithCI:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:strNotEndsWithCI:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:notMatchesCI:alfa"], [("bravo", 10)]),
+        ({"column": "nato"}, ["nato:notMatchesCI:alfa"], [("bravo", 10)]),
+        # Date handling
+        ({"column": "nato"}, ["timestamp:sameDay:2022-02-02"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameWeek:2022-02-03"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameMonth:2022-02-21"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameYear:2022-03-03"], [("alfa", 10)]),
+        (
+            {"column": "nato"},
+            ["timestamp:sameDayOrBefore:2022-02-02"],
+            [("alfa", 40), ("alfa", 10), ("bravo", 10)],
+        ),
+        (
+            {"column": "nato"},
+            ["timestamp:sameWeekOrBefore:2022-02-03"],
+            [("alfa", 40), ("alfa", 10), ("bravo", 10)],
+        ),
+        (
+            {"column": "nato"},
+            ["timestamp:sameMonthOrBefore:2022-02-21"],
+            [("alfa", 40), ("alfa", 10), ("bravo", 10)],
+        ),
+        (
+            {"column": "nato"},
+            ["timestamp:sameYearOrBefore:2022-03-03"],
+            [("alfa", 40), ("alfa", 10), ("bravo", 10)],
+        ),
+        ({"column": "nato"}, ["timestamp:sameDayOrAfter:2022-02-02"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameWeekOrAfter:2022-02-03"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameMonthOrAfter:2022-02-21"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:sameYearOrAfter:2022-03-03"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:beforeDay:2022-02-02"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["timestamp:beforeWeek:2022-02-03"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["timestamp:beforeMonth:2022-02-21"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["timestamp:beforeYear:2022-03-03"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["timestamp:afterDay:2022-02-01"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:afterWeek:2022-01-20"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:afterMonth:2022-01-01"], [("alfa", 10)]),
+        ({"column": "nato"}, ["timestamp:afterYear:2021-03-03"], [("alfa", 10)]),
+        # numeric
+        ({"column": "nato"}, ["numeric:eq:2.2"], [("alfa", 10)]),
+        ({"column": "nato"}, ["numeric:ne:1.1"], [("alfa", 10)]),
+        ({"column": "nato"}, ["numeric:gt:2.1"], [("alfa", 10)]),
+        ({"column": "nato"}, ["numeric:gte:2.2"], [("alfa", 10)]),
+        ({"column": "nato"}, ["numeric:lt:2.2"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["numeric:lte:2.2"], [("alfa", 40), ("alfa", 10), ("bravo", 10)]),
+        # Boolean
+        ({"column": "nato"}, ["bool:isTrue"], [("alfa", 10)]),
+        ({"column": "nato"}, ["bool:isNotTrue"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["bool:isNotFalse"], [("alfa", 10)]),
+        ({"column": "nato"}, ["bool:isFalse"], [("alfa", 40), ("bravo", 10)]),
+        ({"column": "nato"}, ["bool:isNull"], [("alfa", 50), ("bravo", 10)]),
+        ({"column": "nato"}, ["bool:isNotNull"], [("alfa", 40), ("alfa", 10), ("bravo", 10)]),
+    ],
+)
+@mock.patch(
+    "src.dashboard.get_chart_data.get_chart_data._get_table_cols", mock_get_table_cols_results
+)
+def test_query_results(mock_db, mock_bucket, query_params, filter_groups, expected):
+    mock_db.execute(f'CREATE SCHEMA "{TEST_GLUE_DB}"')
+    mock_db.execute(
+        'CREATE TABLE "cumulus-aggregator-test-db"."test__cube__001" AS SELECT * FROM '
+        'read_parquet("./tests/test_data/mock_cube_col_types.parquet")'
+    )
+    query, count_col = get_chart_data._build_query(
+        query_params, filter_groups, {"data_package_id": "test__cube__001"}
+    )
+    res = mock_db.execute(query).fetchall()
+
+    assert len(res) == len(expected)
+    for i in range(0, len(res)):
+        assert res[i] == expected[i]