Log data imports

Author: jolpica

jolpica/formula_one/importer/json_models.py

@@ -49,8 +49,7 @@ def to_timedelta(self) -> timedelta:
 
 def mutate_timedelta_from_dict(value: Any) -> Any:
     if isinstance(value, dict) and value.get("_type") == "timedelta":
-        del value["_type"]
-        return TimedeltaModel(**value).to_timedelta()
+        return TimedeltaModel(**{key: val for key, val in value.items() if key != "_type"}).to_timedelta()
     return value
 
 

jolpica_api/data_import/admin.py

@@ -0,0 +1,12 @@
+from django.contrib import admin
+
+from .models import DataImportLog
+
+
+class DataImportLogAdmin(admin.ModelAdmin):
+    def __init__(self, model, admin_site):
+        self.list_display = [field.name for field in model._meta.fields if field.name != "updated_records"]
+        super().__init__(model, admin_site)
+
+
+admin.site.register(DataImportLog, DataImportLogAdmin)

jolpica_api/data_import/apps.py

@@ -3,4 +3,4 @@
 
 class DataImportConfig(AppConfig):
     default_auto_field = "django.db.models.BigAutoField"
-    name = "data_import"
+    name = "jolpica_api.data_import"

jolpica_api/data_import/migrations/0001_initial.py

@@ -0,0 +1,37 @@
+# Generated by Django 5.1.6 on 2025-03-02 16:36
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="DataImportLog",
+            fields=[
+                ("id", models.BigAutoField(primary_key=True, serialize=False)),
+                ("is_success", models.BooleanField(default=False)),
+                ("completed_at", models.DateTimeField(auto_now_add=True)),
+                ("total_records", models.PositiveIntegerField(null=True)),
+                ("updated_records", models.JSONField(null=True)),
+                ("error_type", models.CharField(max_length=255, null=True)),
+                ("errors", models.JSONField(null=True)),
+                (
+                    "user",
+                    models.ForeignKey(
+                        null=True,
+                        on_delete=django.db.models.deletion.SET_NULL,
+                        to=settings.AUTH_USER_MODEL,
+                    ),
+                ),
+            ],
+        ),
+    ]

jolpica_api/data_import/migrations/__init__.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from django.contrib.auth.models import User
+from django.db import models
+
+
+class DataImportLog(models.Model):
+    id = models.BigAutoField(primary_key=True)
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True)
+    is_success = models.BooleanField(default=False)
+    completed_at = models.DateTimeField(auto_now_add=True)
+    total_records = models.PositiveIntegerField(null=True)
+    updated_records = models.JSONField(null=True)
+    error_type = models.CharField(max_length=255, null=True)
+    errors = models.JSONField(null=True)

jolpica_api/data_import/models.py

@@ -3,9 +3,11 @@
 
 import pytest
 from django.contrib.auth.models import User
+from rest_framework import status
 from rest_framework.test import APIClient
 
 import jolpica.formula_one.models as f1
+from jolpica_api.data_import.models import DataImportLog
 
 
 @pytest.fixture(scope="function")
@@ -182,3 +184,126 @@ def test_data_import_2023_18_models_are_imported(client: APIClient):
         ).count()
         > 45
     )
+
+
+@pytest.mark.django_db
+def test_successful_import(client):
+    """Test successful data import."""
+    data = {
+        "dry_run": False,
+        "data": [
+            {
+                "object_type": "Driver",
+                "foreign_keys": {},
+                "objects": [{"reference": "max_verstappen", "forename": "Max"}],
+            }
+        ],
+    }
+    response = client.put("/data/import/", data, format="json")
+
+    assert response.status_code == status.HTTP_200_OK
+    assert DataImportLog.objects.count() == 1
+    log = DataImportLog.objects.first()
+    assert log.completed_at is not None
+    assert log.error_type is None
+    assert log.updated_records == {"Driver": [831]}
+    assert log.is_success
+    assert log.error_type is None
+    assert log.errors is None
+
+
+@pytest.mark.django_db
+def test_validation_error_has_logs(client):
+    """Test import with deserialization errors."""
+    data = {
+        "dry_run": False,
+        "data": [
+            {
+                "object_type": "Driver",
+                "objects": [{"reference": "max_verstappen", "forename": "Max"}],
+            }
+        ],
+    }
+    response = client.put("/data/import/", data, format="json")
+
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    assert DataImportLog.objects.count() == 1
+    log = DataImportLog.objects.first()
+    assert not log.is_success
+    assert log.error_type == "VALIDATION"
+    assert log.errors[0]["type"] == "missing"
+
+
+@pytest.mark.django_db
+def test_deserialisation_error_has_log(client):
+    """Test validation error during request data validation."""
+    data = {
+        "dry_run": False,
+        "data": [
+            {
+                "object_type": "Round",
+                "foreign_keys": {"year": 9999},
+                "objects": [{"number": 5}],
+            }
+        ],
+    }
+    response = client.put("/data/import/", data, format="json")
+
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    assert DataImportLog.objects.count() == 1
+    log = DataImportLog.objects.first()
+    assert not log.is_success
+    assert log.error_type == "DESERIALISATION"
+    assert response.json()["errors"][0] == {
+        "index": 0,
+        "message": ["DoesNotExist('Season matching query does not exist.')"],
+        "type": "Round",
+    }
+
+
+@pytest.mark.django_db
+def test_dry_run(client):
+    """Test dry run."""
+    data = {
+        "dry_run": True,
+        "data": [
+            {
+                "object_type": "Driver",
+                "foreign_keys": {},
+                "objects": [{"reference": "max_verstappen", "forename": "Max"}],
+            }
+        ],
+    }
+    response = client.put("/data/import/", data, format="json")
+
+    assert response.status_code == status.HTTP_200_OK
+    assert DataImportLog.objects.count() == 0  # No log for dry run
+
+
+@pytest.mark.django_db
+def test_db_error(client):
+    """Test database error during import."""
+    data = {
+        "dry_run": False,
+        "legacy_import": False,
+        "data": [
+            {
+                "object_type": "Lap",
+                "foreign_keys": {"year": 2023, "round": 18, "session": "R", "car_number": 1},
+                "objects": [
+                    {"number": 1, "position": 1, "average_speed": 200.0, "is_entry_fastest_lap": True},
+                    {
+                        "number": 2,
+                        "position": 1,
+                        "average_speed": 201.0,
+                    },
+                ],
+            },
+        ],
+    }
+    response = client.put("/data/import/", data, format="json")
+    assert response.status_code == status.HTTP_400_BAD_REQUEST
+    assert DataImportLog.objects.count() == 1
+    log = DataImportLog.objects.first()
+    assert not log.is_success
+    assert log.error_type == "IMPORT"

jolpica_api/data_import/tests/test_views.py

@@ -1,10 +1,16 @@
-from pydantic import BaseModel, ValidationError
-from rest_framework.permissions import IsAdminUser
+from collections import defaultdict
+
+from pydantic import BaseModel, Field, ValidationError
+from rest_framework.permissions import IsAdminUser, IsAuthenticated
 from rest_framework.request import Request
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
+from jolpica.formula_one.importer.deserialisers import DeserialisationResult
 from jolpica.formula_one.importer.importer import JSONModelImporter
+from jolpica.formula_one.importer.json_models import F1Import
+
+from .models import DataImportLog
 
 
 class ImportDataRequestData(BaseModel):
@@ -15,25 +21,53 @@ class ImportDataRequestData(BaseModel):
     # Should never be used for data from 2025 onwards.
     legacy_import: bool = False
 
-    data: list
+    data: list[F1Import] = Field(min_length=1)
 
 
 class ImportData(APIView):
-    permission_classes = [IsAdminUser]
+    permission_classes = [IsAuthenticated, IsAdminUser]
 
     def put(self, request: Request) -> Response:
+        if request.user.is_anonymous:
+            return Response(status=401)
         try:
             request_data = ImportDataRequestData.model_validate(request.data)
         except ValidationError as ex:
-            return Response({"errors": ex.errors(include_url=False, include_input=False)}, status=400)
+            errors = ex.errors(include_url=False, include_input=False)
+            DataImportLog(user=request.user, is_success=False, error_type="VALIDATION", errors=errors).save()
+            return Response({"errors": errors}, status=400)
 
         model_importer = JSONModelImporter(legacy_import=request_data.legacy_import)
-        result = model_importer.deserialise_all(request_data.data)
+        result = model_importer.deserialise_all(request.data["data"])
 
         if not result.success:
+            DataImportLog(
+                user=request.user, is_success=False, error_type="DESERIALISATION", errors=result.errors
+            ).save()
             return Response({"errors": result.errors}, status=400)
 
         if not request_data.dry_run and request.user.is_staff:
-            model_importer.save_deserialisation_result_to_db(result)
+            try:
+                model_importer.save_deserialisation_result_to_db(result)
+            except Exception as ex:
+                DataImportLog(user=request.user, is_success=False, error_type="IMPORT", errors=[repr(ex)]).save()
+                return Response({"errors": [{"type": "import_error", "message": repr(ex)}]}, status=400)
 
+        if not request_data.dry_run:
+            save_successful_import_to_db(request, result)
         return Response({})
+
+
+def save_successful_import_to_db(request: Request, result: DeserialisationResult) -> None:
+    if request.user.is_anonymous:
+        raise ValueError()
+    updated_record_count = 0
+    updated_records = defaultdict(list)
+    for model_import, instances in result.instances.items():
+        instance_pks = [ins.pk for ins in instances]
+        updated_record_count += len(instance_pks)
+        updated_records[model_import.model_class.__name__].extend(instance_pks)
+
+    DataImportLog(
+        user=request.user, is_success=True, total_records=updated_record_count, updated_records=updated_records
+    ).save()

jolpica_api/data_import/views.py

@@ -73,6 +73,7 @@
     "jolpica.formula_one",
     "jolpica_api.authentication",
     "jolpica_api.ergastapi",
+    "jolpica_api.data_import",
 ]
 if DEPLOYMENT_ENV in ("LOCAL", "SANDBOX"):
     INSTALLED_APPS += ["django_dbml", "debug_toolbar"]