Merge pull request #1840 from watchdogpolska/dev_ext

v1.5.22

.pre-commit-config.yaml

@@ -1,5 +1,5 @@
 ---
-default_stages: [commit, push]
+default_stages: [pre-commit, pre-push]
 default_language_version:
   # force all unspecified python hooks to run python3
   python: python3
@@ -28,7 +28,7 @@ repos:
           - --skip-glob
           - "**/migrations/*.py"
   - repo: https://github.com/sirosen/check-jsonschema
-    rev: 0.29.2
+    rev: 0.29.3
     hooks:
       - id: check-github-workflows
       - id: check-readthedocs
@@ -39,7 +39,7 @@ repos:
         args: [-c, .yamllint.yml]
   #        exclude: (feder/letters/logs/cassettes/.*|docker-compose.yml)
   - repo: https://github.com/psf/black
-    rev: 24.8.0
+    rev: 24.10.0
     hooks:
       - id: black
         args:

docker-compose.yml

@@ -13,6 +13,8 @@ services:
       --max_allowed_packet=1024M
     ports:
       - "3306:3306"
+    networks:
+      - feder-app-network
     environment:
       MYSQL_ROOT_PASSWORD: "password"
       MYSQL_DATABASE: feder # feder_copy
@@ -25,6 +27,8 @@ services:
       dockerfile: .contrib/docker/Dockerfile.gulp
     ports:
       - "35729:35729"
+    networks:
+      - feder-app-network
     volumes:
       - .:/app
 
@@ -70,9 +74,11 @@ services:
       OPENAI_API_ENGINE_35_MAX_TOKENS: 16000
       OPENAI_API_ENGINE_4_MAX_TOKENS: 100000
       OPENAI_API_TEMPERATURE: 0.0
-      OPENAI_API_VERSION: "2024-05-01-preview"
+      OPENAI_API_VERSION: "2024-09-01-preview"
     ports:
       - "8000:8000"
+    networks:
+      - feder-app-network
     # Following allows to execute `docker attach feder_web_1`
     # necessary when using ipdb in development:
     stdin_open: true
@@ -98,6 +104,8 @@ services:
     ports:
       - "1080:1080"
       - "1025:1025"
+    networks:
+      - feder-app-network
 
   #
   # To use email reception uncomment below section and fill in your dev IMAP
@@ -112,7 +120,14 @@ services:
   #     COMPRESS_EML: "True"
   #   env_file:
   #     - secrets_imap-to-webhook-url.env
+  #   networks:
+  #     - feder-app-network
 
 volumes:
   mysql-data:
   maildump_data:
+
+networks:
+  feder-app-network:
+    driver: bridge
+

feder/__init__.py

@@ -1,5 +1,5 @@
 # PEP 396: The __version__ attribute's value SHOULD be a string.
-__version__ = "1.5.21"
+__version__ = "1.5.22"
 
 
 # Compatibility to eg. django-rest-framework

feder/assets/scss/_milestones.scss

@@ -13,7 +13,7 @@ $milestone-icon-width: $milestone-icon-padding * 2 + $milestone-icon-font-size +
 
   // the <ul .milestones>
   list-style-type: none;
-  padding-left: $milestone-icon-width/2;
+  padding-left: calc($milestone-icon-width / 2);
 
   // the top and bottom borders
   // of the list
@@ -68,11 +68,11 @@ $milestone-icon-width: $milestone-icon-padding * 2 + $milestone-icon-font-size +
     // does the heavy lifting to the icons
     & > .fa:first-child,
     & > .glyphicon:first-child {
-      margin-left: -($milestone-offset-larger + ($milestone-icon-width - 2)/2 + 1);
+      margin-left: -($milestone-offset-larger +  calc(($milestone-icon-width - 2px) / 2) + 1);
       margin-top: -$milestone-offset-smaller;
       float:left;
       position: relative;
-      border-radius: $milestone-icon-width/2;
+      border-radius: calc($milestone-icon-width / 2);
       padding: $milestone-icon-padding;
       font-size: $milestone-icon-font-size;
       min-width: $milestone-icon-width;

feder/cases/models.py

@@ -1,7 +1,7 @@
 import uuid
 from datetime import timedelta
 from email.headerregistry import Address
-from functools import lru_cache
+from functools import cache
 
 from autoslug.fields import AutoSlugField
 from django.apps import apps
@@ -41,7 +41,7 @@ def enforce_quarantined_queryset(queryset, user, path_case):
     return queryset.filter(**{f"{path_case}__in": Case.objects.for_user(user).all()})
 
 
-@lru_cache(maxsize=1)  # TODO: use @functools.cache on python>=3.9
+@cache
 def get_quarantined_perm():
     ctype = ContentType.objects.get_for_model(Monitoring)
     return Permission.objects.get(content_type=ctype, codename="view_quarantined_case")

feder/letters/models.py

@@ -1,4 +1,5 @@
 import email
+import gzip
 import json
 import logging
 import uuid
@@ -107,9 +108,10 @@ def for_user(self, user):
             return self.filter(
                 record__case__is_quarantined=False,
                 record__case__monitoring__is_public=True,
-            )
-        if user.is_superuser or user.is_authenticated:
+            ).exclude_spam()
+        if user.is_superuser:
             return self
+        return self.exclude_spam()
 
 
 class LetterManager(BaseManager.from_queryset(LetterQuerySet)):
@@ -455,10 +457,20 @@ def get_recipients(self):
         Letter eml file.
         """
         if not self.eml:
+            if self.email_to:
+                return [self.email_to]
             return []
 
         with self.eml.open(mode="rb") as f:
-            msg = email.message_from_binary_file(f)
+            # Check if the file is a gzip file by reading its magic number
+            magic_number = f.read(2)
+            f.seek(0)  # Reset file pointer to the beginning
+
+            if magic_number == b"\x1f\x8b":
+                with gzip.open(f, mode="rb") as gz:
+                    msg = email.message_from_binary_file(gz)
+            else:
+                msg = email.message_from_binary_file(f)
 
         to_addrs = msg.get_all("To", [])
         cc_addrs = msg.get_all("Cc", [])
@@ -743,9 +755,11 @@ def _enforce_quarantine(self, user):
 
     def for_user(self, user):
         if not user.is_superuser:
-            return self.filter(
-                letter__is_spam__in=[Letter.SPAM.unknown, Letter.SPAM.non_spam]
-            )._enforce_quarantine(user)
+            # Align attachment visibility with letter visibility
+            # return self.exclude(letter__is_spam=Letter.SPAM.spam)._enforce_quarantine(
+            #     user
+            # )
+            return self.filter(letter__in=Letter.objects.for_user(user))
         return self
 
     def with_scan_result(self):

feder/llm_evaluation/tests.py

@@ -62,6 +62,20 @@
     "example": False,
     "tool_calls": [],
     "invalid_tool_calls": [],
+    "usage_metadata": {
+        "input_tokens": 350,
+        "output_tokens": 240,
+        "total_tokens": 590,
+        "input_token_details": {
+            "audio": 10,
+            "cache_creation": 200,
+            "cache_read": 100,
+        },
+        "output_token_details": {
+            "audio": 10,
+            "reasoning": 200,
+        },
+    },
 }
 
 
@@ -76,14 +90,15 @@ def test_azure_chat_openai_response_schema_format(self):
             deployment_name=settings.OPENAI_API_ENGINE_35,
             temperature=settings.OPENAI_API_TEMPERATURE,
         )
-
-        response_format = model.output_schema.schema()["definitions"]["AIMessage"][
+        # To debug schema format:
+        # schema = model.get_output_jsonschema()
+        response_format = model.get_output_jsonschema()["$defs"]["AIMessage"][
             "properties"
         ]
         model_schema_keys = set(response_format.keys())
         # TODO: check why schem differs in dev and github test environments
         # TEMP fix for the github tests to pass:
-        expected_schema_keys = set(EXPECTED_RESPONSE_DICT.keys()) | {"usage_metadata"}
+        expected_schema_keys = set(EXPECTED_RESPONSE_DICT.keys())
         print("\n", 20 * "-", " settings.DEBUG:\n", settings.DEBUG)
         print("\n", 20 * "-", " model:\n", model)
         print("\n", 20 * "-", "response_format:\n", response_format)

feder/monitorings/forms.py

@@ -61,6 +61,7 @@ def __init__(self, *args, **kwargs):
                         "name",
                         "description",
                         "notify_alert",
+                        "is_public",
                         "hide_new_cases",
                         "use_llm",
                     ),
@@ -82,6 +83,7 @@ class Meta:
             "description",
             "notify_alert",
             "hide_new_cases",
+            "is_public",
             "use_llm",
             "subject",
             "template",