WIP: Feature/fix infra spatialite (#79)

Introduces changes to fix spatialite extension loading issue to support
data fields in sqlachemy model type `POINT`. This change directly
supports the work in #26
- Fixes spatialite extension as loadable with the updated dockerfile
setup and correct implementation using aiosqlite compatibility
- Temporary disables integration test. The repository return type needs
to be refactored and return type changed to list dto facilities results

<!-- Generated by sourcery-ai[bot]: start summary -->

## Summary by Sourcery

Fix SpatiaLite extension loading issue to support spatial data fields in
SQLAlchemy models. Introduce a new `PointType` for handling geopoint
data. Refactor test database setup and temporarily disable integration
tests pending refactoring. Update build scripts and Makefile for
improved Docker handling.

New Features:
- Introduce a new `PointType` class to handle geopoint data types in
SQLAlchemy models, enabling support for spatial data fields like
`POINT`.

Bug Fixes:
- Fix the loading of the SpatiaLite extension in the database setup to
support spatial data operations.

Enhancements:
- Refactor the test database setup to use a new `SetUpTestDatabase`
class, improving the management of test database lifecycle and session
handling.

Build:
- Update the Makefile to include a new `set-docker` target and modify
the `docker-run-server` target to remove orphan containers.

Tests:
- Temporarily disable integration tests due to the need for refactoring
repository return types and changing return types to list DTO facilities
results.

Chores:
- Update the `run.sh` script to include additional logging for file
listing and database removal.

<!-- Generated by sourcery-ai[bot]: end summary -->

---------

Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com>

Dockerfile

@@ -2,6 +2,16 @@
 # Reuse the stage from Dockerfile.build
 FROM xcov19-setup AS run
 
+# Set the working directory
+WORKDIR /app
+
+# Bust cached build if --build CACHEBUST=<some data> is passed 
+# to ensure updated source code is built
+ARG CACHEBUST
+COPY --chown=nonroot:nonroot --chmod=555 xcov19 xcov19/
+COPY --chown=nonroot:nonroot --chmod=555 Makefile .
+COPY --chown=nonroot:nonroot --chmod=555 *.sh .
+
 USER nonroot:nonroot
 
 # Set the start command

Dockerfile.build

@@ -87,14 +87,6 @@ RUN chown -R nonroot:nonroot /app
 RUN mkdir -p /var/cache
 RUN chown -R nonroot:nonroot /var/cache
 
-# Copy the application code
-COPY --chown=nonroot:nonroot --chmod=555 xcov19 xcov19/
-COPY --chown=nonroot:nonroot --chmod=555 Makefile .
-COPY --chown=nonroot:nonroot --chmod=555 pyproject.toml .
-COPY --chown=nonroot:nonroot --chmod=555 poetry.lock .
-COPY --chown=nonroot:nonroot --chmod=555 *.sh .
-COPY --chown=nonroot:nonroot --chmod=555 LICENSE .
-
 ENV POETRY_NO_INTERACTION=1
 ENV POETRY_VIRTUALENVS_CREATE=false
 ENV POETRY_CACHE_DIR='/var/cache/pypoetry'
@@ -107,6 +99,11 @@ RUN curl --proto "=https" --tlsv1.2 -sSf -L https://install.python-poetry.org |
 RUN mkdir -p /var/cache/pypoetry && chown -R nonroot:nonroot /var/cache/pypoetry
 RUN chown -R nonroot:nonroot /usr/local/ && chmod -R 755 /usr/local/
 
+# Copy the application code
+COPY --chown=nonroot:nonroot --chmod=555 pyproject.toml .
+COPY --chown=nonroot:nonroot --chmod=555 poetry.lock .
+COPY --chown=nonroot:nonroot --chmod=555 LICENSE .
+
 # Switch to nonroot user
 USER nonroot:nonroot
 

Dockerfile.test-integration

@@ -2,6 +2,16 @@
 # Reuse the stage from Dockerfile.build
 FROM xcov19-setup AS test-integration
 
+# Set the working directory
+WORKDIR /app
+
+# Bust cached build if --build CACHEBUST=<some data> is passed 
+# to ensure updated source code is built
+ARG CACHEBUST=1
+COPY --chown=nonroot:nonroot --chmod=555 xcov19 xcov19/
+COPY --chown=nonroot:nonroot --chmod=555 Makefile .
+COPY --chown=nonroot:nonroot --chmod=555 *.sh .
+
 # Switch to nonroot user
 USER nonroot:nonroot
 

Makefile

@@ -17,19 +17,22 @@ test:
 	APP_ENV=test APP_DB_ENGINE_URL="sqlite+aiosqlite://" pytest -s xcov19/tests/ -m "not slow and not integration and not api"
 
 test-integration:
-	APP_ENV=test APP_DB_ENGINE_URL="sqlite+aiosqlite://" pytest -s xcov19/tests/ -m "integration"
+	APP_ENV=test PYTHON_CONFIGURE_OPTS="--enable-loadable-sqlite-extensions" APP_DB_ENGINE_URL="sqlite+aiosqlite://" pytest -s xcov19/tests/ -m "integration"
 
 todos:
 	@grep -rn "TODO:" xcov19/ --exclude-dir=node_modules --include="*.py"
 
+set-docker:
+	@bash set_docker.sh
+
 docker-build:
 	docker build --load -f Dockerfile.build -t $(XCOV19_SETUP_IMAGE) .
 
 docker-integration:
 	docker build --load -f Dockerfile.test-integration -t $(XCOV19_TEST_INTEGRATION_SETUP_IMAGE) .
 
 docker-run-server:
-	docker compose -f docker-compose.yml up --build
+	docker compose -f docker-compose.yml up --build --remove-orphans
 
 docker-test-integration:
 	make docker-integration && docker run -it -f Dockerfile.test-integration

docker-compose.yml

@@ -10,8 +10,10 @@ x-shared-config: &shared-config
             max-file: 2
 
 services:
-    xcov19-app:
+    app:
         <<: *shared-config
         build: 
           context: .
-          dockerfile: Dockerfile
+          dockerfile: Dockerfile
+          args:
+            CACHEBUST: ${CACHEBUST:-$(date +%s)}

pyproject.toml

@@ -6,6 +6,7 @@ authors = ["codecakes <[email protected]>"]
 readme = "README.md"
 package-mode = true
 
+
 [tool.poetry.dependencies]
 python = "^3.12"
 pydantic = "^2.9.1"
@@ -20,6 +21,7 @@ alembic = "^1.13.2"
 aiosqlite = "^0.20.0"
 sqlmodel = {version="^0.0.22"}
 rich = {version = "^13.8.0"}
+spatialite = "^0.0.3"
 
 ruff = { version = "^0.6.3", optional = true }
 mypy = { version = "^1.11.2", optional = true }
@@ -66,8 +68,9 @@ include = [
 ]
 exclude = [
     "**/node_modules",
-    "**/__pycache__"]
-venv = "xcov19-7M_0Y8Vx-py3.12"
+    "**/__pycache__"
+    ]
+# venv = "xcov19-7M_0Y8Vx-py3.12"
 reportMissingImports = true
 
 [[tool.pyright.executionEnvironments]]

run.sh

@@ -1,3 +1,10 @@
 #!/bin/bash
 
-if [ -f "xcov19.db" ]; then rm xcov19.db; fi; APP_ENV=dev APP_DB_ENGINE_URL="sqlite+aiosqlite:///xcov19.db" poetry run python3 -m xcov19.dev
+echo "listing all files";
+ls;
+if [ -f "xcov19.db" ]; then 
+    echo "removing database";
+    rm xcov19.db; 
+fi;
+
+APP_ENV=dev APP_DB_ENGINE_URL="sqlite+aiosqlite:///xcov19.db" poetry run python3 -m xcov19.dev

xcov19/app/database.py

@@ -1,6 +1,8 @@
+import asyncio
 from collections.abc import AsyncGenerator
 from contextlib import asynccontextmanager
 import sys
+import aiosqlite
 from rodi import Container
 from xcov19.infra.models import SQLModel
 from sqlmodel import text
@@ -11,9 +13,11 @@
     AsyncEngine,
     async_sessionmaker,
 )
+from sqlalchemy.dialects.sqlite.aiosqlite import AsyncAdapt_aiosqlite_connection
 
 import logging
 from sqlalchemy.pool import AsyncAdaptedQueuePool
+from sqlalchemy import event
 
 db_logger = logging.getLogger(__name__)
 db_fmt = logging.Formatter(
@@ -45,11 +49,49 @@ def __call__(self) -> async_sessionmaker[AsyncSessionWrapper]:
         )
 
 
+async def _load_spatialite(dbapi_conn: AsyncAdapt_aiosqlite_connection) -> None:
+    """Loads spatialite sqlite extension."""
+    conn: aiosqlite.Connection = dbapi_conn.driver_connection
+    await conn.enable_load_extension(True)
+    await conn.load_extension("mod_spatialite")
+    db_logger.info("======= PRAGMA load_extension successful =======")
+    try:
+        async with conn.execute("SELECT spatialite_version() as version") as cursor:
+            result = await cursor.fetchone()
+            db_logger.info(f"==== Spatialite Version: {result} ====")
+            db_logger.info("===== mod_spatialite loaded =====")
+    except (AttributeError, aiosqlite.OperationalError) as e:
+        db_logger.error(e)
+        raise (e)
+
+
+def setup_spatialite(engine: AsyncEngine) -> None:
+    """An event listener hook to setup spatialite using aiosqlite."""
+
+    @event.listens_for(engine.sync_engine, "connect")
+    def load_spatialite(
+        dbapi_conn: AsyncAdapt_aiosqlite_connection, _connection_record
+    ):
+        loop = asyncio.get_running_loop()
+        # Schedule the coroutine in the existing event loop
+        loop.create_task(_load_spatialite(dbapi_conn))
+
+
 async def setup_database(engine: AsyncEngine) -> None:
     """Sets up tables for database."""
+
+    setup_spatialite(engine)
     async with engine.begin() as conn:
+        # Enable extension loading
+        await conn.execute(text("PRAGMA load_extension = 1"))
+        # db_logger.info("SQLAlchemy setup to load the SpatiaLite extension.")
+        # await conn.execute(text("SELECT load_extension('/opt/homebrew/Cellar/libspatialite/5.1.0_1/lib/mod_spatialite.dylib')"))
+        # await conn.execute(text("SELECT load_extension('mod_spatialite')"))
         # see: https://sqlmodel.tiangolo.com/tutorial/relationship-attributes/cascade-delete-relationships/#enable-foreign-key-support-in-sqlite
         await conn.execute(text("PRAGMA foreign_keys=ON"))
+        # test_result = await conn.execute(text("SELECT spatialite_version() as version;"))
+        # print(f"==== Spatialite Version: {test_result.fetchone()} ====")
+
         await conn.run_sync(SQLModel.metadata.create_all)
         await conn.commit()
         db_logger.info("===== Database tables setup. =====")

xcov19/domain/models/provider.py

@@ -28,6 +28,7 @@ class FacilityOwnership(enum.StrEnum):
 type Specialties = List[str]
 type Qualification = List[str]
 type PracticeExpYears = int | float
+type MoneyType = int | float
 
 
 @dataclass
@@ -73,6 +74,7 @@ class Doctor:
     specialties: Specialties
     degree: Qualification
     experience: PracticeExpYears
+    fee: MoneyType
 
 
 @dataclass
@@ -84,5 +86,6 @@ class Provider:
     facility_type: FacilityType
     ownership: FacilityOwnerType
     specialties: Specialties
+    available_doctors: List[Doctor]
     stars: Annotated[int, Stars(min_rating=1, max_rating=5)]
     reviews: Annotated[int, Reviews(value=0)]

xcov19/infra/models.py

@@ -34,23 +34,73 @@
 
 from __future__ import annotations
 
-from typing import List
+import json
+from typing import Annotated, Dict, List, Tuple, Any
+from pydantic import GetCoreSchemaHandler, TypeAdapter
+from pydantic_core import CoreSchema, core_schema
+from sqlalchemy.sql.elements import ColumnElement
+from sqlalchemy.sql.type_api import _BindProcessorType
 from sqlmodel import SQLModel, Field, Relationship
-from sqlalchemy import Column, Text, Float, Index
+from sqlalchemy import BindParameter, Column, Dialect, Text, Float, Index, func
 from sqlalchemy.orm import relationship, Mapped
 import uuid
-from sqlalchemy.dialects.sqlite import TEXT
+from sqlalchemy.dialects.sqlite import TEXT, NUMERIC, JSON, INTEGER
+from sqlalchemy.types import UserDefinedType
 
 
+class PointType(UserDefinedType):
+    """Defines a geopoint type.
+
+    It also sets the type as a pydantic type when plugged into TypeAdapter.
+    """
+
+    def get_col_spec(self):
+        return "POINT"
+
+    def result_processor(self, dialect: Dialect, coltype: Any) -> Any | None:
+        def process(value):
+            if not value:
+                return None
+            parsed_value = value[6:-1].split()
+            return tuple(map(float, parsed_value))
+
+        return process
+
+    def bind_processor(self, dialect: Dialect) -> _BindProcessorType | None:
+        def process(value):
+            if not value:
+                return None
+            lat, lng = value
+            return f"POINT({lat} {lng})"
+
+        return process
+
+    def bind_expression(self, bindvalue: BindParameter) -> ColumnElement | None:
+        return func.GeomFromText(bindvalue, type_=self)
+
+    @classmethod
+    def __get_pydantic_core_schema__(
+        cls, _source_type: Tuple, handler: GetCoreSchemaHandler
+    ) -> CoreSchema:
+        """Pydantic validates the data as a tuple."""
+        return core_schema.no_info_after_validator_function(cls, handler(tuple))
+
+    @classmethod
+    def pydantic_adapter(cls) -> TypeAdapter:
+        return TypeAdapter(cls)
+
+
+def generate_uuid() -> str:
+    return str(uuid.uuid4())
+
+
+### These tables map to the domain models for Patient
 class Patient(SQLModel, table=True):
     patient_id: str = Field(
-        sa_column=Column(
-            TEXT, unique=True, primary_key=True, default=str(uuid.uuid4())
-        ),
+        sa_column=Column(TEXT, unique=True, primary_key=True, default=generate_uuid),
         allow_mutation=False,
     )
     queries: Mapped[List["Query"]] = Relationship(
-        # back_populates="patient",
         passive_deletes="all",
         cascade_delete=True,
         sa_relationship=relationship(back_populates="patient"),
@@ -61,9 +111,7 @@ class Query(SQLModel, table=True):
     """Every Query must have both a Patient and a Location."""
 
     query_id: str = Field(
-        sa_column=Column(
-            TEXT, unique=True, primary_key=True, default=str(uuid.uuid4())
-        ),
+        sa_column=Column(TEXT, unique=True, primary_key=True, default=generate_uuid),
         allow_mutation=False,
     )
     query: str = Field(allow_mutation=False, sa_column=Column(Text))
@@ -80,26 +128,46 @@ class Location(SQLModel, table=True):
         Index("ix_location_composite_lat_lng", "latitude", "longitude", unique=True),
     )
     location_id: str = Field(
-        sa_column=Column(
-            TEXT, unique=True, primary_key=True, default=str(uuid.uuid4())
-        ),
+        sa_column=Column(TEXT, unique=True, primary_key=True, default=generate_uuid),
         allow_mutation=False,
     )
     latitude: float = Field(sa_column=Column(Float))
     longitude: float = Field(sa_column=Column(Float))
     queries: Mapped[List["Query"]] = Relationship(
-        # back_populates="location",
         cascade_delete=True,
         passive_deletes=True,
         sa_relationship=relationship(back_populates="location"),
     )
 
 
-# TODO: Define Provider SQL model fields
-# class Provider(SQLModel, table=True):
-#     # TODO: Compare with Github issue, domain model and noccodb
-#     ...
+###
+
+
+### These tables map to the domain models for Provider
+class Provider(SQLModel, table=True):
+    provider_id: str = Field(
+        sa_column=Column(TEXT, unique=True, primary_key=True, default=generate_uuid),
+        allow_mutation=False,
+    )
+    name: str = Field(
+        sa_column=Column(TEXT, nullable=False),
+    )
+    address: str = Field(sa_column=Column(TEXT, nullable=False), allow_mutation=False)
+    geopoint: Annotated[
+        tuple, lambda geom: PointType.pydantic_adapter().validate_python(geom)
+    ] = Field(sa_column=Column(PointType, nullable=False), allow_mutation=False)
+    contact: int = Field(sa_column=Column(NUMERIC, nullable=False))
+    facility_type: str = Field(sa_column=Column(TEXT, nullable=False))
+    ownership_type: str = Field(sa_column=Column(TEXT, nullable=False))
+    specialties: List[str] = Field(sa_column=Column(JSON, nullable=False))
+    stars: int = Field(sa_column=Column(INTEGER, nullable=False, default=0))
+    reviews: int = Field(sa_column=Column(INTEGER, nullable=False, default=0))
+    available_doctors: List[Dict[str, str | int | float | list]] = Field(
+        sa_column=Column(JSON, nullable=False, default=json.dumps([]))
+    )
+
 
+###
 
 # TODO: Add Model events for database ops during testing
 # @event.listens_for(Query, "after_delete")