diff --git a/Makefile b/Makefile index c891153c..1ae4c79a 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,7 @@ SHELL := /bin/bash # -- Docker COMPOSE = bin/compose -COMPOSE_UP = $(COMPOSE) up -d +COMPOSE_UP = $(COMPOSE) up -d --remove-orphans COMPOSE_RUN = $(COMPOSE) run --rm --no-deps COMPOSE_RUN_API = $(COMPOSE_RUN) api COMPOSE_RUN_API_PIPENV = $(COMPOSE_RUN_API) pipenv run @@ -333,6 +333,10 @@ reset-db: \ reset-dashboard-db .PHONY: reset-db +refresh-api-static: ## Refresh the API Statique Materialized View + $(COMPOSE) exec api pipenv run python -m qualicharge refresh-static +.PHONY: refresh-api-static + reset-api-db: ## Reset the PostgreSQL API database $(COMPOSE) stop $(COMPOSE) down postgresql diff --git a/src/api/CHANGELOG.md b/src/api/CHANGELOG.md index 792eed96..6a88f767 100644 --- a/src/api/CHANGELOG.md +++ b/src/api/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to `SENTRY_PROFILES_SAMPLE_RATE` configuration - Set request's user (`username`) in Sentry's context - Add `Localisation.coordonneesXY` unique contraint [BC] šŸ’„ +- Implement `Statique` materialized view ### Changed diff --git a/src/api/Pipfile b/src/api/Pipfile index 18b1c43e..455edd9b 100644 --- a/src/api/Pipfile +++ b/src/api/Pipfile @@ -26,6 +26,7 @@ questionary = "==2.1.0" sentry-sdk = {extras = ["fastapi"], version = "==2.20.0"} setuptools = "==75.8.0" sqlalchemy-timescaledb = "==0.4.1" +sqlalchemy-utils = "==0.41.2" sqlmodel = "==0.0.22" typer = "==0.15.1" uvicorn = {extras = ["standard"] } diff --git a/src/api/Pipfile.lock b/src/api/Pipfile.lock index b31fbfbf..3c96292f 100644 --- a/src/api/Pipfile.lock +++ b/src/api/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "6c812bc177a6f0da310185aa53a02103338be8ddc24295c1aa061820f9a30352" + "sha256": "8dda9b1e97aa4c777e193b31fa0da4b39e75107360a5a73d5b167170d1b744af" }, "pipfile-spec": 6, "requires": { @@ -1353,6 +1353,15 @@ "index": "pypi", "version": "==0.4.1" }, + "sqlalchemy-utils": { + "hashes": [ + "sha256:85cf3842da2bf060760f955f8467b87983fb2e30f1764fd0e24a48307dc8ec6e", + "sha256:bc599c8c3b3319e53ce6c5c3c471120bd325d0071fb6f38a10e924e3d07b9990" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==0.41.2" + }, "sqlmodel": { "hashes": [ "sha256:7d37c882a30c43464d143e35e9ecaf945d88035e20117bf5ec2834a23cbe505e", @@ -2109,11 +2118,11 @@ }, "faker": { "hashes": [ - "sha256:49dde3b06a5602177bc2ad013149b6f60a290b7154539180d37b6f876ae79b20", - "sha256:ac4cf2f967ce02c898efa50651c43180bd658a7707cfd676fcc5410ad1482c03" + "sha256:42f2da8cf561e38c72b25e9891168b1e25fec42b6b0b5b0b6cd6041da54af885", + "sha256:926d2301787220e0554c2e39afc4dc535ce4b0a8d0a089657137999f66334ef4" ], "markers": "python_version >= '3.8'", - "version": "==33.3.1" + "version": "==35.0.0" }, "flask": { "hashes": [ diff --git a/src/api/cron.json b/src/api/cron.json new file mode 100644 index 00000000..443ca356 --- /dev/null +++ b/src/api/cron.json @@ -0,0 +1,7 @@ +{ + "jobs": [ + { + "command": "*/10 * * * * python -m qualicharge refresh-static" + } + ] +} diff --git a/src/api/pyproject.toml b/src/api/pyproject.toml index 187ee919..7c60d7aa 100644 --- a/src/api/pyproject.toml +++ b/src/api/pyproject.toml @@ -74,5 +74,6 @@ exclude = [ [[tool.mypy.overrides]] module = [ "shapely.*", + "sqlalchemy_utils.*" ] ignore_missing_imports = true diff --git a/src/api/qualicharge/cli.py b/src/api/qualicharge/cli.py index 8b021447..bb986f06 100644 --- a/src/api/qualicharge/cli.py +++ b/src/api/qualicharge/cli.py @@ -14,6 +14,7 @@ from rich.table import Table from sqlalchemy import Column as SAColumn from sqlalchemy.exc import IntegrityError, OperationalError, ProgrammingError +from sqlalchemy_utils import refresh_materialized_view from sqlmodel import Session as SMSession from sqlmodel import select @@ -23,7 +24,7 @@ from .db import get_session from .exceptions import IntegrityError as QCIntegrityError from .fixtures.operational_units import prefixes -from .schemas.core import OperationalUnit +from .schemas.core import STATIQUE_MV_TABLE_NAME, OperationalUnit from .schemas.sql import StatiqueImporter logging.basicConfig( @@ -456,6 +457,18 @@ def import_static(ctx: typer.Context, input_file: Path): console.log("Saved (or updated) all entries successfully.") +@app.command() +def refresh_static(ctx: typer.Context, concurrently: bool = False): + """Refresh the Statique materialized view.""" + session: SMSession = ctx.obj + + # Refresh the database + console.log("Refreshing databaseā€¦") + refresh_materialized_view( + session, STATIQUE_MV_TABLE_NAME, concurrently=concurrently + ) + + @app.callback() def main(ctx: typer.Context): """Attach database session to the context object.""" diff --git a/src/api/qualicharge/migrations/env.py b/src/api/qualicharge/migrations/env.py index f00bba8d..cbd9bf79 100644 --- a/src/api/qualicharge/migrations/env.py +++ b/src/api/qualicharge/migrations/env.py @@ -3,9 +3,11 @@ from logging.config import fileConfig from alembic import context +from geoalchemy2.alembic_helpers import create_geospatial_index from sqlalchemy import engine_from_config, pool from sqlmodel import SQLModel + from qualicharge.conf import settings # Nota bene: be sure to import all models that need to be migrated here diff --git a/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py b/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py new file mode 100644 index 00000000..82e37aec --- /dev/null +++ b/src/api/qualicharge/migrations/versions/4b99d15436b0_add_statique_materialized_view.py @@ -0,0 +1,47 @@ +"""Add statique materialized view + +Revision ID: 4b99d15436b0 +Revises: 9ae109e209c9 +Create Date: 2025-01-16 15:02:04.004411 + +""" + +from typing import Sequence, Union + +from alembic import op +from geoalchemy2.functions import ST_GeomFromEWKB +from sqlalchemy_utils.view import CreateView, DropView + +from qualicharge.schemas.core import StatiqueMV, _StatiqueMV + +# revision identifiers, used by Alembic. +revision: str = "4b99d15436b0" +down_revision: Union[str, None] = "9ae109e209c9" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Create the Statique Materialized view and related indexes.""" + op.execute( + CreateView( + _StatiqueMV.__table__.fullname, _StatiqueMV.selectable, materialized=True + ) + ) + op.create_geospatial_index( + "idx_statique_coordonneesXY", + _StatiqueMV.__table__.fullname, + [ST_GeomFromEWKB(StatiqueMV.coordonneesXY)], + unique=False, + postgresql_using="gist", + ) + for idx in _StatiqueMV.__table__.indexes: + idx.create(op.get_bind()) + + +def downgrade() -> None: + """Delete the Statique Materialized View.""" + + op.execute( + DropView(_StatiqueMV.__table__.fullname, materialized=True, cascade=True) + ) diff --git a/src/api/qualicharge/schemas/core.py b/src/api/qualicharge/schemas/core.py index 7d1b38fe..9b070169 100644 --- a/src/api/qualicharge/schemas/core.py +++ b/src/api/qualicharge/schemas/core.py @@ -1,7 +1,8 @@ """QualiCharge core statique and dynamique schemas.""" +from datetime import datetime from enum import IntEnum -from typing import TYPE_CHECKING, List, Optional, Union, cast +from typing import TYPE_CHECKING, ClassVar, List, Optional, Union, cast from uuid import UUID, uuid4 from geoalchemy2.shape import to_shape @@ -19,11 +20,15 @@ ) from pydantic_extra_types.coordinate import Coordinate from shapely.geometry import mapping -from sqlalchemy import event +from sqlalchemy import Select, event +from sqlalchemy import cast as SA_cast from sqlalchemy.dialects.postgresql import ENUM as PgEnum +from sqlalchemy.orm import registry from sqlalchemy.schema import Column as SAColumn +from sqlalchemy.schema import Index from sqlalchemy.types import Date, DateTime, String -from sqlmodel import Field, Relationship, UniqueConstraint, select +from sqlalchemy_utils import create_materialized_view +from sqlmodel import Field, Relationship, SQLModel, UniqueConstraint, select from sqlmodel import Session as SMSession from sqlmodel.main import SQLModelConfig @@ -44,12 +49,17 @@ ImplantationStationEnum, NotFutureDate, RaccordementEnum, + Statique, ) from . import BaseTimestampedSQLModel if TYPE_CHECKING: from qualicharge.auth.schemas import Group +mapper_registry = registry() + +STATIQUE_MV_TABLE_NAME: str = "statique" + class OperationalUnitTypeEnum(IntEnum): """Operational unit types.""" @@ -472,3 +482,98 @@ class Status(BaseTimestampedSQLModel, StatusBase, table=True): def id_pdc_itinerance(self) -> str: """Return the PointDeCharge.id_pdc_itinerance (used for serialization only).""" return self.point_de_charge.id_pdc_itinerance + + +class StatiqueMV(Statique, SQLModel): + """Statique Materialized View.""" + + __tablename__ = STATIQUE_MV_TABLE_NAME + + model_config = SQLModel.model_config + + pdc_id: UUID + pdc_updated_at: datetime + + +class _StatiqueMV(SQLModel): + """Statique Materialized view. + + NOTE: This is an internal model used **ONLY** for creating the materialized view. + """ + + selectable: ClassVar[Select] = ( + select( # type: ignore[call-overload, misc] + cast(SAColumn, PointDeCharge.id).label("pdc_id"), + cast(SAColumn, PointDeCharge.updated_at).label("pdc_updated_at"), + Amenageur.nom_amenageur, + Amenageur.siren_amenageur, + Amenageur.contact_amenageur, + Operateur.nom_operateur, + Operateur.contact_operateur, + Operateur.telephone_operateur, + Enseigne.nom_enseigne, + Station.id_station_itinerance, + Station.id_station_local, + Station.nom_station, + Station.implantation_station, + Localisation.adresse_station, + Localisation.code_insee_commune, + SA_cast( + Localisation.coordonneesXY, + Geometry( + geometry_type="POINT", + # WGS84 coordinates system + srid=4326, + spatial_index=False, + ), + ).label("coordonneesXY"), + Station.nbre_pdc, + PointDeCharge.id_pdc_itinerance, + PointDeCharge.id_pdc_local, + PointDeCharge.puissance_nominale, + PointDeCharge.prise_type_ef, + PointDeCharge.prise_type_2, + PointDeCharge.prise_type_combo_ccs, + PointDeCharge.prise_type_chademo, + PointDeCharge.prise_type_autre, + PointDeCharge.gratuit, + PointDeCharge.paiement_acte, + PointDeCharge.paiement_cb, + PointDeCharge.paiement_autre, + PointDeCharge.tarification, + Station.condition_acces, + PointDeCharge.reservation, + Station.horaires, + PointDeCharge.accessibilite_pmr, + PointDeCharge.restriction_gabarit, + Station.station_deux_roues, + Station.raccordement, + Station.num_pdl, + Station.date_mise_en_service, + PointDeCharge.observations, + Station.date_maj, + PointDeCharge.cable_t2_attache, + ) + .select_from(PointDeCharge) + .join(Station) + .join(Amenageur) + .join(Operateur) + .join(Enseigne) + .join(Localisation) + ) + + __table__ = create_materialized_view( + name=STATIQUE_MV_TABLE_NAME, + selectable=selectable, + metadata=SQLModel.metadata, + indexes=[ + Index("idx_statique_id_pdc_itinerance", "id_pdc_itinerance", unique=True), + Index( + "idx_statique_code_insee_commune", + "code_insee_commune", + ), + ], + ) + + +mapper_registry.map_imperatively(StatiqueMV, _StatiqueMV.__table__) diff --git a/src/api/tests/schemas/test_static.py b/src/api/tests/schemas/test_static.py index cef105f4..0d2e606e 100644 --- a/src/api/tests/schemas/test_static.py +++ b/src/api/tests/schemas/test_static.py @@ -2,12 +2,15 @@ import re from datetime import datetime, timedelta, timezone +from uuid import UUID import pytest from geoalchemy2.shape import to_shape +from geoalchemy2.types import WKBElement from pydantic_extra_types.coordinate import Coordinate from shapely.geometry import mapping from sqlalchemy.exc import IntegrityError +from sqlalchemy_utils import refresh_materialized_view from sqlmodel import select from qualicharge.factories.static import ( @@ -18,8 +21,16 @@ OperationalUnitFactory, PointDeChargeFactory, StationFactory, + StatiqueFactory, ) -from qualicharge.schemas.core import Amenageur, Localisation, OperationalUnit, Station +from qualicharge.schemas.core import ( + Amenageur, + Localisation, + OperationalUnit, + Station, + StatiqueMV, +) +from qualicharge.schemas.utils import save_statiques @pytest.mark.parametrize( @@ -392,3 +403,21 @@ def test_operational_unit_create_stations_fk(db_session): select(Station).where(Station.operational_unit_id == operational_unit.id) ).all() assert len(stations) == n_stations + extra_stations + + +def test_statique_materialized_view(db_session): + """Test the StatiqueMV schema.""" + n_pdc = 4 + statiques = StatiqueFactory.batch(n_pdc) + save_statiques(db_session, statiques) + refresh_materialized_view(db_session, "statique") + + db_statiques = db_session.exec(select(StatiqueMV)).all() + assert len(db_statiques) == n_pdc + + assert {s.id_pdc_itinerance for s in statiques} == { + s.id_pdc_itinerance for s in db_statiques + } + assert isinstance(db_statiques[0].coordonneesXY, WKBElement) + assert isinstance(db_statiques[0].pdc_id, UUID) + assert isinstance(db_statiques[0].pdc_updated_at, datetime)