Logements should keep the importation order (#1604)

* Logements should keep the importation order

* Add some assertions

conventions/forms/convention_form_logements.py

@@ -220,6 +220,10 @@ class LogementForm(forms.Form):
             "max_digits": "La loyer doit-être inférieur à 10000 €",
         },
     )
+    import_order = forms.IntegerField(
+        label="",
+        required=False,
+    )
 
     def clean_loyer(self):
         """
@@ -457,6 +461,10 @@ class FoyerResidenceLogementForm(forms.Form):
             "max_digits": "La redevance maximale inférieur à 10000 €",
         },
     )
+    import_order = forms.IntegerField(
+        label="",
+        required=False,
+    )
 
 
 class BaseFoyerResidenceLogementFormSet(BaseFormSet):

conventions/services/convention_generator.py

@@ -9,10 +9,6 @@
 import jinja2
 from django.conf import settings
 from django.core.files.storage import default_storage
-from django.db import transaction
-from django.db.models import F, Func, IntegerField, Value
-from django.db.models.functions import Cast
-from django.db.utils import DataError
 from django.forms.models import model_to_dict
 from django.template.defaultfilters import date as template_date
 from docx.shared import Inches
@@ -140,31 +136,8 @@ def generate_convention_doc(convention: Convention, save_data=False) -> DocxTemp
 
     adresse = _get_adresse(convention)
 
-    # Logements are ordered by typologie first, then by designation
-    # Designation is a string, we try to find a number in it and cast it as a number to sort
-    # If the cast fail, we order by designation as a string
-    try:
-        with transaction.atomic():
-            logements = (
-                convention.lot.logements.all()
-                .annotate(
-                    int_designation=Cast(
-                        Func(
-                            F("designation"),
-                            Value(r"\D"),
-                            Value(""),
-                            Value("g"),
-                            function="regexp_replace",
-                        ),
-                        IntegerField(),
-                    )
-                )
-                .order_by("typologie", "int_designation")
-            )
-            # Force queryset execution
-            list(logements)
-    except DataError:
-        logements = convention.lot.logements.all().order_by("typologie", "designation")
+    # Logements should keep the importation order
+    logements = convention.lot.logements.order_by("import_order")
 
     context = {
         **avenant_data,
@@ -672,7 +645,7 @@ def fiche_caf_doc(convention):
         "programme": convention.programme,
         "lot": convention.lot,
         "administration": convention.programme.administration,
-        "logements": convention.lot.logements.all(),
+        "logements": convention.lot.logements.order_by("import_order"),
         "nb_logements_par_type": nb_logements_par_type,
         "lot_num": lot_num,
         "loyer_m2": _get_loyer_par_metre_carre(convention),

conventions/services/logements.py

@@ -17,7 +17,7 @@ class ConventionLogementsService(ConventionService):
 
     def get(self):
         initial = []
-        logements = self.convention.lot.logements.all()
+        logements = self.convention.lot.logements.order_by("import_order")
         for logement in logements:
             initial.append(
                 {
@@ -68,6 +68,7 @@ def _upload_logements(self):
                 Logement,
                 self.convention,
                 "logements.xlsx",
+                import_order=True,
             )
             if result["success"] != utils.ReturnStatus.ERROR:
                 lgts_by_designation = {}
@@ -144,6 +145,9 @@ def _logements_atomic_update(self):
                     f"form-{idx}-loyer": utils.get_form_value(
                         form_logement, logement, "loyer"
                     ),
+                    f"form-{idx}-import_order": utils.get_form_value(
+                        form_logement, logement, "import_order"
+                    ),
                 }
             else:
                 initformset = {
@@ -165,6 +169,7 @@ def _logements_atomic_update(self):
                     ].value(),
                     f"form-{idx}-coeficient": form_logement["coeficient"].value(),
                     f"form-{idx}-loyer": form_logement["loyer"].value(),
+                    f"form-{idx}-import_order": form_logement["import_order"].value(),
                 }
         self.formset = LogementFormSet(initformset)
         self.formset.programme_id = self.convention.programme_id
@@ -217,6 +222,7 @@ def _save_logements(self):
                 ]
                 logement.coeficient = form_logement.cleaned_data["coeficient"]
                 logement.loyer = form_logement.cleaned_data["loyer"]
+                logement.import_order = form_logement.cleaned_data["import_order"]
             else:
                 logement = Logement.objects.create(
                     lot=self.convention.lot,
@@ -233,6 +239,7 @@ def _save_logements(self):
                     ],
                     coeficient=form_logement.cleaned_data["coeficient"],
                     loyer=form_logement.cleaned_data["loyer"],
+                    import_order=form_logement.cleaned_data["import_order"],
                 )
             logement.save()
 
@@ -244,7 +251,7 @@ class ConventionFoyerResidenceLogementsService(ConventionService):
 
     def get(self):
         initial = []
-        logements = self.convention.lot.logements.all()
+        logements = self.convention.lot.logements.order_by("import_order")
         for logement in logements:
             initial.append(
                 {
@@ -286,6 +293,7 @@ def _upload_foyer_residence_logements(self):
                 "foyer_residence_logements.xlsx",
                 class_field_mapping="foyer_residence_import_mapping",
                 class_field_needed_mapping="foyer_residence_needed_in_mapping",
+                import_order=True,
             )
             if result["success"] != utils.ReturnStatus.ERROR:
                 lgts_by_designation = {}
@@ -335,6 +343,9 @@ def _foyer_residence_logements_atomic_update(self):
                     f"form-{idx}-loyer": utils.get_form_value(
                         form_logement, logement, "loyer"
                     ),
+                    f"form-{idx}-import_order": utils.get_form_value(
+                        form_logement, logement, "import_order"
+                    ),
                 }
             else:
                 initformset = {
@@ -345,6 +356,7 @@ def _foyer_residence_logements_atomic_update(self):
                         "surface_habitable"
                     ].value(),
                     f"form-{idx}-loyer": form_logement["loyer"].value(),
+                    f"form-{idx}-import_order": form_logement["import_order"].value(),
                 }
         self.formset = FoyerResidenceLogementFormSet(initformset)
         self.formset.lot_id = self.convention.lot_id
@@ -395,12 +407,14 @@ def _save_foyer_residence_logements(self):
                     "surface_habitable"
                 ]
                 logement.loyer = form_logement.cleaned_data["loyer"]
+                logement.import_order = form_logement.cleaned_data["import_order"]
             else:
                 logement = Logement.objects.create(
                     lot=self.convention.lot,
                     designation=form_logement.cleaned_data["designation"],
                     typologie=form_logement.cleaned_data["typologie"],
                     surface_habitable=form_logement.cleaned_data["surface_habitable"],
                     loyer=form_logement.cleaned_data["loyer"],
+                    import_order=form_logement.cleaned_data["import_order"],
                 )
             logement.save()

conventions/services/upload_objects.py

@@ -27,6 +27,7 @@ def handle_uploaded_xlsx(
     file_name,
     class_field_mapping="import_mapping",
     class_field_needed_mapping="needed_in_mapping",
+    import_order=False,
 ):
     try:
         my_file.seek(0)
@@ -85,6 +86,7 @@ def handle_uploaded_xlsx(
         min_row,
         class_field_mapping=class_field_mapping,
         class_field_needed_mapping=class_field_needed_mapping,
+        import_order=import_order,
     )
 
     return {
@@ -150,16 +152,22 @@ def _get_object_from_worksheet(
     *,
     class_field_mapping,
     class_field_needed_mapping,
+    import_order: bool,
 ):
     my_objects = []
 
-    for row in my_ws.iter_rows(
-        min_row=min_row, max_row=my_ws.max_row, min_col=1, max_col=my_ws.max_column
+    for index, row in enumerate(
+        my_ws.iter_rows(
+            min_row=min_row, max_row=my_ws.max_row, min_col=1, max_col=my_ws.max_column
+        )
     ):
         my_row, empty_line, new_warnings = _extract_row(
             row, column_from_index, my_class, class_field_mapping=class_field_mapping
         )
 
+        if import_order:
+            my_row["import_order"] = index
+
         if hasattr(my_class, class_field_needed_mapping):
             if not empty_line and getattr(my_class, class_field_needed_mapping):
                 for needed_field in getattr(my_class, class_field_needed_mapping):

conventions/tests/fixtures.py

@@ -87,6 +87,7 @@
     "form-0-loyer_par_metre_carre": "4.5",
     "form-0-coeficient": "1.0000",
     "form-0-loyer": "135.00",
+    "form-0-import_order": "0",
     "form-1-uuid": "",
     "form-1-designation": "B2",
     "form-1-typologie": "T1",
@@ -97,6 +98,7 @@
     "form-1-loyer_par_metre_carre": "4.5",
     "form-1-coeficient": "1.0000",
     "form-1-loyer": "135.00",
+    "form-1-import_order": "1",
     "loyer_derogatoire": "10",
     "surface_locaux_collectifs_residentiels": "25",
     "lgts_mixite_sociale_negocies": "2",

conventions/tests/services/test_convention_generator.py

@@ -214,70 +214,35 @@ def test_get_adresse(self):
 
     def test_generate_convention_doc(self):
         convention = Convention.objects.get(numero="0001")
-        Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T2, designation="Logement 2"
-        )
-        Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T2, designation="Logement 1"
-        )
-        Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T1, designation="Logement 3"
-        )
         Logement.objects.create(
             lot=convention.lot,
-            typologie=TypologieLogement.T1BIS,
-            designation="Logement 4",
+            typologie=TypologieLogement.T2,
+            designation="Logement 2",
+            import_order=3,
         )
         Logement.objects.create(
             lot=convention.lot,
-            typologie=TypologieLogement.T1BIS,
-            designation="Logement 34",
-        )
-        convention.programme.nature_logement = NatureLogement.RESISDENCESOCIALE
-
-        with patch(
-            "conventions.services.convention_generator.DocxTemplate.render"
-        ) as mocked_render:
-            generate_convention_doc(convention)
-
-            args, _ = mocked_render.call_args
-            context = args[0]
-
-            mocked_render.assert_called_once()
-            assert set(context.keys()) == convention_context_keys()
-            assert [logement.designation for logement in context["logements"]] == [
-                "Logement 3",
-                "Logement 4",
-                "Logement 34",
-                "Logement 1",
-                "Logement 2",
-            ]
-
-    def test_generate_convention_doc_logements_without_numbers(self):
-        convention = Convention.objects.get(numero="0001")
-        Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T2, designation="Logement 2"
+            typologie=TypologieLogement.T2,
+            designation="Logement 1",
+            import_order=1,
         )
         Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T2, designation="Logement 1"
-        )
-        Logement.objects.create(
-            lot=convention.lot, typologie=TypologieLogement.T1, designation="Logement 3"
+            lot=convention.lot,
+            typologie=TypologieLogement.T1,
+            designation="Logement 3",
+            import_order=2,
         )
         Logement.objects.create(
             lot=convention.lot,
             typologie=TypologieLogement.T1BIS,
             designation="Logement 4",
+            import_order=4,
         )
         Logement.objects.create(
             lot=convention.lot,
             typologie=TypologieLogement.T1BIS,
             designation="Logement 34",
-        )
-        Logement.objects.create(
-            lot=convention.lot,
-            typologie=TypologieLogement.T1BIS,
-            designation="Logement",
+            import_order=0,
         )
         convention.programme.nature_logement = NatureLogement.RESISDENCESOCIALE
 
@@ -292,12 +257,11 @@ def test_generate_convention_doc_logements_without_numbers(self):
             mocked_render.assert_called_once()
             assert set(context.keys()) == convention_context_keys()
             assert [logement.designation for logement in context["logements"]] == [
-                "Logement 3",
-                "Logement",
                 "Logement 34",
-                "Logement 4",
                 "Logement 1",
+                "Logement 3",
                 "Logement 2",
+                "Logement 4",
             ]
 
     def test_get_convention_template_path(self):

conventions/tests/services/test_logements_service.py

@@ -98,6 +98,7 @@ def test_save(self):
                     "loyer_par_metre_carre",
                     "coeficient",
                     "loyer",
+                    "import_order",
                 ],
             ),
             {
@@ -110,6 +111,7 @@ def test_save(self):
                 "loyer_par_metre_carre": Decimal("4.5"),
                 "coeficient": Decimal("1.0000"),
                 "loyer": Decimal("135.00"),
+                "import_order": 0,
             },
         )
         self.assertEqual(logement_b1.lot.loyer_derogatoire, 10.00)

programmes/migrations/0114_logement_import_order.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.1 on 2024-10-10 07:37
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("programmes", "0113_lot_loyer_associations_foncieres"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="logement",
+            name="import_order",
+            field=models.IntegerField(blank=True, null=True),
+        ),
+    ]

programmes/models/models.py

@@ -516,6 +516,10 @@ def bailleur(self):
     def annexes(self):
         return Annexe.objects.filter(logement__lot=self)
 
+    @property
+    def logements_import_ordered(self):
+        return self.logements.order_by("import_order")
+
     def clone(self, cloned_programme):
         parent_id = self.parent_id or self.id
         lot_fields = model_to_dict(
@@ -663,6 +667,7 @@ class Logement(models.Model):
     loyer = models.DecimalField(max_digits=12, decimal_places=2, null=True, blank=True)
     cree_le = models.DateTimeField(auto_now_add=True)
     mis_a_jour_le = models.DateTimeField(auto_now=True)
+    import_order = models.IntegerField(null=True, blank=True)
 
     import_mapping = {
         "Désignation des logements": designation,

templates/conventions/common/form_logements.html

@@ -122,6 +122,12 @@ <h4>Logements associés à la convention</h4>
                                             </td>
                                             <td id="lgt_loyer">
                                                 {% include "common/form/input_number.html" with form_input=form.loyer step="0.01" onchange="compute_total_value('loyer')" parent_object_field=main_object_field editable=editable_after_upload %}
+                                                <input
+                                                    type="hidden"
+                                                    id="{{form.import_order.id_for_label}}"
+                                                    name="{{form.import_order.html_name}}"
+                                                    {% if form.import_order.value is not None %}value="{{ form.import_order.value }}"{% endif %}
+                                                />
                                             </td>
                                         </tr>
                                     {% endwith %}