✨(api) add database tables auditability

We need to be able to track who changed what when on most of our data
for auditability purpose.

For performance purpose, we decided to use PostgreSQL table triggers to
track table changes over time.

.github/workflows/api.yml

@@ -131,6 +131,8 @@ jobs:
           ref: ${{ github.head_ref }}
       - name: Create postgis extension
         run: psql "postgresql://qualicharge:pass@localhost:5432/qualicharge-api" -c "create extension postgis;"
+      - name: Create btree_gist extension
+        run: psql "postgresql://qualicharge:pass@localhost:5432/qualicharge-api" -c "create extension btree_gist;"
       - name: Install pipenv
         run: pipx install pipenv
       - name: Set up Python 3.12
@@ -307,6 +309,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Create postgis extension
         run: psql "postgresql://qualicharge:pass@localhost:5432/test-qualicharge-api" -c "create extension postgis;"
+      - name: Create btree_gist extension
+        run: psql "postgresql://qualicharge:pass@localhost:5432/test-qualicharge-api" -c "create extension btree_gist;"
       - name: Install pipenv
         run: pipx install pipenv
       - name: Set up Python 3.12
@@ -355,6 +359,8 @@ jobs:
       - uses: actions/checkout@v4
       - name: Create postgis extension
         run: psql "postgresql://qualicharge:pass@localhost:5432/test-qualicharge-api" -c "create extension postgis;"
+      - name: Create btree_gist extension
+        run: psql "postgresql://qualicharge:pass@localhost:5432/test-qualicharge-api" -c "create extension btree_gist;"
       - name: Install pipenv
         run: pipx install pipenv
       - name: Set up Python 3.12

Makefile

@@ -215,6 +215,7 @@ create-api-test-db: ## create API test database
 	@echo "Creating api service test database…"
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "create database \"$${QUALICHARGE_TEST_DB_NAME}\";"' || echo "Duly noted, skipping database creation."
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/$${QUALICHARGE_TEST_DB_NAME}" -c "create extension postgis;"' || echo "Duly noted, skipping extension creation."
+	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/$${QUALICHARGE_TEST_DB_NAME}" -c "create extension btree_gist;"' || echo "Duly noted, skipping extension creation."
 .PHONY: create-api-test-db
 
 create-metabase-db: ## create metabase database
@@ -271,6 +272,7 @@ migrate-api:  ## run alembic database migrations for the api service
 	@echo "Creating api service database…"
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/postgres" -c "create database \"$${QUALICHARGE_DB_NAME}\";"' || echo "Duly noted, skipping database creation."
 	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/$${QUALICHARGE_DB_NAME}" -c "create extension postgis;"' || echo "Duly noted, skipping extension creation."
+	@$(COMPOSE) exec postgresql bash -c 'psql "postgresql://$${POSTGRES_USER}:$${POSTGRES_PASSWORD}@$${QUALICHARGE_DB_HOST}:$${QUALICHARGE_DB_PORT}/$${QUALICHARGE_DB_NAME}" -c "create extension btree_gist;"' || echo "Duly noted, skipping extension creation."
 	@echo "Running migrations for api service…"
 	@bin/alembic upgrade head
 .PHONY: migrate-api

src/api/CHANGELOG.md

@@ -8,6 +8,10 @@ and this project adheres to
 
 ## [Unreleased]
 
+### Added
+
+- Integrate `postgresql_audit` for critical database changes versioning
+
 ### Changed
 
 - Upgrade fastapi to `0.115.7`

src/api/Pipfile

@@ -14,6 +14,7 @@ geopandas = "==1.0.1"
 httpx = {extras = ["cli"], version = "==0.28.1"}
 pandas = "==2.2.3"
 passlib = {extras = ["bcrypt"], version = "==1.7.4"}
+postgresql-audit = "==0.17.1"
 psycopg = {extras = ["pool", "binary"], version = "==3.2.4"}
 pyarrow = "==19.0.0"
 pydantic = "==2.10.6"

src/api/pyproject.toml

@@ -73,7 +73,8 @@ exclude = [
 
 [[tool.mypy.overrides]]
 module = [
+  "postgresql_audit.*",
   "shapely.*",
-  "sqlalchemy_utils.*"
+  "sqlalchemy_utils.*",
 ]
 ignore_missing_imports = true

src/api/qualicharge/api/v1/routers/dynamic.py

@@ -400,7 +400,10 @@ async def create_session(
     # - `session` / `QCSession` / `SessionCreate` refers to qualicharge charging session
     pdc_id = get_pdc_id(session.id_pdc_itinerance, db_session)
 
-    db_qc_session = QCSession(**session.model_dump(exclude={"id_pdc_itinerance"}))
+    db_qc_session = QCSession(
+        **session.model_dump(exclude={"id_pdc_itinerance"}),
+        created_by_id=user.id,
+    )
     # Store session id so that we do not need to perform another request
     db_qc_session_id = db_qc_session.id
     db_qc_session.point_de_charge_id = pdc_id
@@ -446,7 +449,10 @@ async def create_session_bulk(
     db_qc_sessions = []
     db_qc_session_ids = []
     for session in sessions:
-        db_qc_session = QCSession(**session.model_dump(exclude={"id_pdc_itinerance"}))
+        db_qc_session = QCSession(
+            **session.model_dump(exclude={"id_pdc_itinerance"}),
+            created_by_id=user.id,
+        )
         db_qc_session_ids.append(db_qc_session.id)
         db_qc_session.point_de_charge_id = db_pdcs[session.id_pdc_itinerance]
         db_qc_sessions.append(db_qc_session)

src/api/qualicharge/api/v1/routers/static.py

@@ -228,7 +228,7 @@ async def update(
 
     transaction = session.begin_nested()
     try:
-        update = update_statique(session, id_pdc_itinerance, statique)
+        update = update_statique(session, id_pdc_itinerance, statique, author=user)
     except QCIntegrityError as err:
         transaction.rollback()
         raise HTTPException(
@@ -275,7 +275,7 @@ async def create(
 
     transaction = session.begin_nested()
     try:
-        db_statique = save_statique(session, statique)
+        db_statique = save_statique(session, statique, author=user)
     except ObjectDoesNotExist as err:
         transaction.rollback()
         raise HTTPException(
@@ -312,7 +312,7 @@ async def bulk(
     )
 
     transaction = session.begin_nested()
-    importer = StatiqueImporter(df, transaction.session.connection())
+    importer = StatiqueImporter(df, transaction.session.connection(), author=user)
     try:
         importer.save()
     except (

src/api/qualicharge/auth/factories.py

@@ -9,7 +9,7 @@
 from polyfactory.pytest_plugin import register_fixture
 
 from qualicharge.conf import settings
-from qualicharge.factories import FrenchDataclassFactory, TimestampedSQLModelFactory
+from qualicharge.factories import AuditableSQLModelFactory, FrenchDataclassFactory
 
 from .models import IDToken
 from .schemas import Group, ScopesEnum, User
@@ -33,7 +33,7 @@ class IDTokenFactory(ModelFactory[IDToken]):
     email = "[email protected]"
 
 
-class UserFactory(TimestampedSQLModelFactory[User]):
+class UserFactory(AuditableSQLModelFactory[User]):
     """User schema factory."""
 
     username = Use(
@@ -45,7 +45,7 @@ class UserFactory(TimestampedSQLModelFactory[User]):
     scopes = Use(FrenchDataclassFactory.__random__.sample, list(ScopesEnum), 2)
 
 
-class GroupFactory(TimestampedSQLModelFactory[Group]):
+class GroupFactory(AuditableSQLModelFactory[Group]):
     """Group schema factory."""
 
     name = Use(FrenchDataclassFactory.__faker__.company)

src/api/qualicharge/auth/schemas.py

@@ -9,7 +9,7 @@
 from sqlmodel import Field, Relationship, SQLModel
 
 from qualicharge.conf import settings
-from qualicharge.schemas import BaseTimestampedSQLModel
+from qualicharge.schemas import BaseAuditableSQLModel
 from qualicharge.schemas.core import OperationalUnit
 
 
@@ -53,9 +53,13 @@ class ScopesEnum(StrEnum):
 
 
 # -- Core schemas
-class User(BaseTimestampedSQLModel, table=True):
+class User(BaseAuditableSQLModel, table=True):
     """QualiCharge User."""
 
+    __versioned__ = {
+        "exclude": BaseAuditableSQLModel.__versioned__["exclude"] + ["password"]
+    }
+
     id: UUID = Field(default_factory=uuid4, primary_key=True)
     username: str = Field(unique=True, max_length=150)
     email: EmailStr = Field(unique=True, sa_type=String)
@@ -103,7 +107,7 @@ def check_password(self, password: str) -> bool:
         return settings.PASSWORD_CONTEXT.verify(password, self.password)
 
 
-class Group(BaseTimestampedSQLModel, table=True):
+class Group(BaseAuditableSQLModel, table=True):
     """QualiCharge Group."""
 
     id: UUID = Field(default_factory=uuid4, primary_key=True)

src/api/qualicharge/factories/__init__.py

@@ -34,3 +34,24 @@ class TimestampedSQLModelFactory(Generic[T], SQLAlchemyFactory[T]):
     id = Use(uuid4)
     created_at = Use(lambda: datetime.now(timezone.utc) - timedelta(hours=1))
     updated_at = Use(datetime.now, timezone.utc)
+
+
+class AuditableSQLModelFactory(Generic[T], SQLAlchemyFactory[T]):
+    """A base factory for Auditable SQLModel.
+
+    We expect SQLModel to define the following fields:
+
+    - id: UUID
+    - created_at: datetime
+    - updated_at: datetime
+    - created_by_id: UUID
+    - updated_by_id: UUID
+    """
+
+    __is_base_factory__ = True
+
+    id = Use(uuid4)
+    created_at = Use(lambda: datetime.now(timezone.utc) - timedelta(hours=1))
+    updated_at = Use(datetime.now, timezone.utc)
+    created_by_id = None
+    updated_by_id = None

src/api/qualicharge/factories/static.py

@@ -20,7 +20,11 @@
     PointDeCharge,
     Station,
 )
-from . import FrenchDataclassFactory, TimestampedSQLModelFactory
+from . import (
+    AuditableSQLModelFactory,
+    FrenchDataclassFactory,
+    TimestampedSQLModelFactory,
+)
 
 T = TypeVar("T")
 
@@ -83,13 +87,13 @@ def id_station_itinerance(cls, id_pdc_itinerance: str):
         return prefix + FrenchDataclassFactory.__faker__.pystr_format("E######")
 
 
-class AmenageurFactory(TimestampedSQLModelFactory[Amenageur]):
+class AmenageurFactory(AuditableSQLModelFactory[Amenageur]):
     """Amenageur schema factory."""
 
     contact_amenageur = Use(FrenchDataclassFactory.__faker__.ascii_company_email)
 
 
-class EnseigneFactory(TimestampedSQLModelFactory[Enseigne]):
+class EnseigneFactory(AuditableSQLModelFactory[Enseigne]):
     """Enseigne schema factory."""
 
 
@@ -100,7 +104,7 @@ class CoordinateFactory(DataclassFactory[Coordinate]):
     latitude = Use(DataclassFactory.__faker__.pyfloat, min_value=-90, max_value=90)
 
 
-class LocalisationFactory(TimestampedSQLModelFactory[Localisation]):
+class LocalisationFactory(AuditableSQLModelFactory[Localisation]):
     """Localisation schema factory."""
 
     @classmethod
@@ -113,7 +117,7 @@ def get_sqlalchemy_types(cls) -> Dict[Any, Callable[[], Any]]:
         }
 
 
-class OperateurFactory(TimestampedSQLModelFactory[Operateur]):
+class OperateurFactory(AuditableSQLModelFactory[Operateur]):
     """Operateur schema factory."""
 
     contact_operateur = Use(FrenchDataclassFactory.__faker__.ascii_company_email)
@@ -132,7 +136,7 @@ class OperateurFactory(TimestampedSQLModelFactory[Operateur]):
     )
 
 
-class PointDeChargeFactory(TimestampedSQLModelFactory[PointDeCharge]):
+class PointDeChargeFactory(AuditableSQLModelFactory[PointDeCharge]):
     """PointDeCharge schema factory."""
 
     id_pdc_itinerance = Use(
@@ -154,7 +158,7 @@ class OperationalUnitFactory(TimestampedSQLModelFactory[OperationalUnit]):
     name = Use(FrenchDataclassFactory.__faker__.company)
 
 
-class StationFactory(TimestampedSQLModelFactory[Station]):
+class StationFactory(AuditableSQLModelFactory[Station]):
     """Station schema factory."""
 
     id_station_itinerance = Use(

src/api/qualicharge/fixtures/operational_units.py

@@ -15,6 +15,7 @@
 """
 
 from collections import namedtuple
+from typing import List
 
 # Nota bene: we import the GroupOperationalUnit schema so that the MetaData registry
 # is aware of the GroupOperationalUnit table and allows to use this secondary
@@ -27,7 +28,7 @@
 
 # Operational units
 Item = namedtuple("Item", ["code", "name"])
-data = [
+data: List[Item] = [
     Item(
         "FR073",
         "ACELEC CHARGE",